xref: /illumos-gate/usr/src/uts/i86pc/io/rootnex.c (revision 23a869ddcd5330bdcbefc98e81d9de23b83f3182)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 /*
25  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
26  * Copyright (c) 2011 Bayard G. Bell.  All rights reserved.
27  * Copyright 2012 Garrett D'Amore <garrett@damore.org>.  All rights reserved.
28  * Copyright 2017 Joyent, Inc.
29  * Copyright 2020 Ryan Zezeski
30  */
31 
32 /*
33  * x86 root nexus driver
34  */
35 
36 #include <sys/sysmacros.h>
37 #include <sys/conf.h>
38 #include <sys/autoconf.h>
39 #include <sys/sysmacros.h>
40 #include <sys/debug.h>
41 #include <sys/psw.h>
42 #include <sys/ddidmareq.h>
43 #include <sys/promif.h>
44 #include <sys/devops.h>
45 #include <sys/kmem.h>
46 #include <sys/cmn_err.h>
47 #include <vm/seg.h>
48 #include <vm/seg_kmem.h>
49 #include <vm/seg_dev.h>
50 #include <sys/vmem.h>
51 #include <sys/mman.h>
52 #include <vm/hat.h>
53 #include <vm/as.h>
54 #include <vm/page.h>
55 #include <sys/avintr.h>
56 #include <sys/errno.h>
57 #include <sys/modctl.h>
58 #include <sys/ddi_impldefs.h>
59 #include <sys/sunddi.h>
60 #include <sys/sunndi.h>
61 #include <sys/mach_intr.h>
62 #include <sys/psm.h>
63 #include <sys/ontrap.h>
64 #include <sys/atomic.h>
65 #include <sys/sdt.h>
66 #include <sys/rootnex.h>
67 #include <vm/hat_i86.h>
68 #include <sys/ddifm.h>
69 #include <sys/ddi_isa.h>
70 #include <sys/apic.h>
71 
72 #ifdef __xpv
73 #include <sys/bootinfo.h>
74 #include <sys/hypervisor.h>
75 #include <sys/bootconf.h>
76 #include <vm/kboot_mmu.h>
77 #endif
78 
79 #if defined(__amd64) && !defined(__xpv)
80 #include <sys/immu.h>
81 #endif
82 
83 
84 /*
85  * enable/disable extra checking of function parameters. Useful for debugging
86  * drivers.
87  */
88 #ifdef	DEBUG
89 int rootnex_alloc_check_parms = 1;
90 int rootnex_bind_check_parms = 1;
91 int rootnex_bind_check_inuse = 1;
92 int rootnex_unbind_verify_buffer = 0;
93 int rootnex_sync_check_parms = 1;
94 #else
95 int rootnex_alloc_check_parms = 0;
96 int rootnex_bind_check_parms = 0;
97 int rootnex_bind_check_inuse = 0;
98 int rootnex_unbind_verify_buffer = 0;
99 int rootnex_sync_check_parms = 0;
100 #endif
101 
102 boolean_t rootnex_dmar_not_setup;
103 
104 /* Master Abort and Target Abort panic flag */
105 int rootnex_fm_ma_ta_panic_flag = 0;
106 
107 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
108 int rootnex_bind_fail = 1;
109 int rootnex_bind_warn = 1;
110 uint8_t *rootnex_warn_list;
111 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
112 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
113 
114 /*
115  * revert back to old broken behavior of always sync'ing entire copy buffer.
116  * This is useful if be have a buggy driver which doesn't correctly pass in
117  * the offset and size into ddi_dma_sync().
118  */
119 int rootnex_sync_ignore_params = 0;
120 
121 /*
122  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
123  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
124  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
125  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
126  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
127  * (< 8K). We will still need to allocate the copy buffer during bind though
128  * (if we need one). These can only be modified in /etc/system before rootnex
129  * attach.
130  */
131 #if defined(__amd64)
132 int rootnex_prealloc_cookies = 65;
133 int rootnex_prealloc_windows = 4;
134 int rootnex_prealloc_copybuf = 2;
135 #else
136 int rootnex_prealloc_cookies = 33;
137 int rootnex_prealloc_windows = 4;
138 int rootnex_prealloc_copybuf = 2;
139 #endif
140 
141 /* driver global state */
142 static rootnex_state_t *rootnex_state;
143 
144 #ifdef DEBUG
145 /* shortcut to rootnex counters */
146 static uint64_t *rootnex_cnt;
147 #endif
148 
149 /*
150  * XXX - does x86 even need these or are they left over from the SPARC days?
151  */
152 /* statically defined integer/boolean properties for the root node */
153 static rootnex_intprop_t rootnex_intprp[] = {
154 	{ "PAGESIZE",			PAGESIZE },
155 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
156 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
157 	{ DDI_RELATIVE_ADDRESSING,	1 },
158 };
159 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
160 
161 /*
162  * If we're dom0, we're using a real device so we need to load
163  * the cookies with MFNs instead of PFNs.
164  */
165 #ifdef __xpv
166 typedef maddr_t rootnex_addr_t;
167 #define	ROOTNEX_PADDR_TO_RBASE(pa)	\
168 	(DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa))
169 #else
170 typedef paddr_t rootnex_addr_t;
171 #define	ROOTNEX_PADDR_TO_RBASE(pa)	(pa)
172 #endif
173 
174 static struct cb_ops rootnex_cb_ops = {
175 	nodev,		/* open */
176 	nodev,		/* close */
177 	nodev,		/* strategy */
178 	nodev,		/* print */
179 	nodev,		/* dump */
180 	nodev,		/* read */
181 	nodev,		/* write */
182 	nodev,		/* ioctl */
183 	nodev,		/* devmap */
184 	nodev,		/* mmap */
185 	nodev,		/* segmap */
186 	nochpoll,	/* chpoll */
187 	ddi_prop_op,	/* cb_prop_op */
188 	NULL,		/* struct streamtab */
189 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
190 	CB_REV,		/* Rev */
191 	nodev,		/* cb_aread */
192 	nodev		/* cb_awrite */
193 };
194 
195 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
196     off_t offset, off_t len, caddr_t *vaddrp);
197 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
198     struct hat *hat, struct seg *seg, caddr_t addr,
199     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
200 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
201     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
202     ddi_dma_handle_t *handlep);
203 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
204     ddi_dma_handle_t handle);
205 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
206     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
207     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
208 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
209     ddi_dma_handle_t handle);
210 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
211     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
212 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
213     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
214     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
215 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
216     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
217     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
218 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
219     ddi_ctl_enum_t ctlop, void *arg, void *result);
220 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
221     ddi_iblock_cookie_t *ibc);
222 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
223     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
224 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *,
225     void *);
226 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *);
227 
228 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
229     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
230     ddi_dma_handle_t *handlep);
231 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
232     ddi_dma_handle_t handle);
233 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
234     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
235     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
236 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
237     ddi_dma_handle_t handle);
238 #if defined(__amd64) && !defined(__xpv)
239 static void rootnex_coredma_reset_cookies(dev_info_t *dip,
240     ddi_dma_handle_t handle);
241 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
242     ddi_dma_cookie_t **cookiepp, uint_t *ccountp);
243 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
244     ddi_dma_cookie_t *cookiep, uint_t ccount);
245 static int rootnex_coredma_clear_cookies(dev_info_t *dip,
246     ddi_dma_handle_t handle);
247 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle);
248 #endif
249 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip,
250     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
251 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip,
252     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
253     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
254 
255 #if defined(__amd64) && !defined(__xpv)
256 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
257     ddi_dma_handle_t handle, void *v);
258 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
259     ddi_dma_handle_t handle);
260 #endif
261 
262 
263 static struct bus_ops rootnex_bus_ops = {
264 	BUSO_REV,
265 	rootnex_map,
266 	NULL,
267 	NULL,
268 	NULL,
269 	rootnex_map_fault,
270 	0,
271 	rootnex_dma_allochdl,
272 	rootnex_dma_freehdl,
273 	rootnex_dma_bindhdl,
274 	rootnex_dma_unbindhdl,
275 	rootnex_dma_sync,
276 	rootnex_dma_win,
277 	rootnex_dma_mctl,
278 	rootnex_ctlops,
279 	ddi_bus_prop_op,
280 	i_ddi_rootnex_get_eventcookie,
281 	i_ddi_rootnex_add_eventcall,
282 	i_ddi_rootnex_remove_eventcall,
283 	i_ddi_rootnex_post_event,
284 	0,			/* bus_intr_ctl */
285 	0,			/* bus_config */
286 	0,			/* bus_unconfig */
287 	rootnex_fm_init,	/* bus_fm_init */
288 	NULL,			/* bus_fm_fini */
289 	NULL,			/* bus_fm_access_enter */
290 	NULL,			/* bus_fm_access_exit */
291 	NULL,			/* bus_powr */
292 	rootnex_intr_ops	/* bus_intr_op */
293 };
294 
295 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
296 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
297 static int rootnex_quiesce(dev_info_t *dip);
298 
299 static struct dev_ops rootnex_ops = {
300 	DEVO_REV,
301 	0,
302 	ddi_no_info,
303 	nulldev,
304 	nulldev,
305 	rootnex_attach,
306 	rootnex_detach,
307 	nulldev,
308 	&rootnex_cb_ops,
309 	&rootnex_bus_ops,
310 	NULL,
311 	rootnex_quiesce,		/* quiesce */
312 };
313 
314 static struct modldrv rootnex_modldrv = {
315 	&mod_driverops,
316 	"i86pc root nexus",
317 	&rootnex_ops
318 };
319 
320 static struct modlinkage rootnex_modlinkage = {
321 	MODREV_1,
322 	(void *)&rootnex_modldrv,
323 	NULL
324 };
325 
326 #if defined(__amd64) && !defined(__xpv)
327 static iommulib_nexops_t iommulib_nexops = {
328 	IOMMU_NEXOPS_VERSION,
329 	"Rootnex IOMMU ops Vers 1.1",
330 	NULL,
331 	rootnex_coredma_allochdl,
332 	rootnex_coredma_freehdl,
333 	rootnex_coredma_bindhdl,
334 	rootnex_coredma_unbindhdl,
335 	rootnex_coredma_reset_cookies,
336 	rootnex_coredma_get_cookies,
337 	rootnex_coredma_set_cookies,
338 	rootnex_coredma_clear_cookies,
339 	rootnex_coredma_get_sleep_flags,
340 	rootnex_coredma_sync,
341 	rootnex_coredma_win,
342 	rootnex_coredma_hdl_setprivate,
343 	rootnex_coredma_hdl_getprivate
344 };
345 #endif
346 
347 /*
348  *  extern hacks
349  */
350 extern struct seg_ops segdev_ops;
351 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
352 #ifdef	DDI_MAP_DEBUG
353 extern int ddi_map_debug_flag;
354 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
355 #endif
356 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
357 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
358 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
359     psm_intr_op_t, int *);
360 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
361 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
362 
363 /*
364  * Use device arena to use for device control register mappings.
365  * Various kernel memory walkers (debugger, dtrace) need to know
366  * to avoid this address range to prevent undesired device activity.
367  */
368 extern void *device_arena_alloc(size_t size, int vm_flag);
369 extern void device_arena_free(void * vaddr, size_t size);
370 
371 
372 /*
373  *  Internal functions
374  */
375 static int rootnex_dma_init();
376 static void rootnex_add_props(dev_info_t *);
377 static int rootnex_ctl_reportdev(dev_info_t *dip);
378 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
379 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
380 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
381 static int rootnex_map_handle(ddi_map_req_t *mp);
382 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
383 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
384 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
385     ddi_dma_attr_t *attr);
386 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
387     rootnex_sglinfo_t *sglinfo);
388 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object,
389     ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo);
390 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
391     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
392 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
393     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
394 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
395 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
396     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag);
397 static void rootnex_teardown_windows(rootnex_dma_t *dma);
398 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
399     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
400 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
401     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
402     size_t *copybuf_used, page_t **cur_pp);
403 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
404     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
405     ddi_dma_attr_t *attr, off_t cur_offset);
406 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
407     rootnex_dma_t *dma, rootnex_window_t **windowp,
408     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
409 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
410     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
411 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
412     off_t offset, size_t size, uint_t cache_flags);
413 static int rootnex_verify_buffer(rootnex_dma_t *dma);
414 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
415     const void *comp_addr, const void *not_used);
416 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object,
417     rootnex_sglinfo_t *sglinfo);
418 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object);
419 
420 /*
421  * _init()
422  *
423  */
424 int
425 _init(void)
426 {
427 
428 	rootnex_state = NULL;
429 	return (mod_install(&rootnex_modlinkage));
430 }
431 
432 
433 /*
434  * _info()
435  *
436  */
437 int
438 _info(struct modinfo *modinfop)
439 {
440 	return (mod_info(&rootnex_modlinkage, modinfop));
441 }
442 
443 
444 /*
445  * _fini()
446  *
447  */
448 int
449 _fini(void)
450 {
451 	return (EBUSY);
452 }
453 
454 
455 /*
456  * rootnex_attach()
457  *
458  */
459 static int
460 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
461 {
462 	int fmcap;
463 	int e;
464 
465 	switch (cmd) {
466 	case DDI_ATTACH:
467 		break;
468 	case DDI_RESUME:
469 #if defined(__amd64) && !defined(__xpv)
470 		return (immu_unquiesce());
471 #else
472 		return (DDI_SUCCESS);
473 #endif
474 	default:
475 		return (DDI_FAILURE);
476 	}
477 
478 	/*
479 	 * We should only have one instance of rootnex. Save it away since we
480 	 * don't have an easy way to get it back later.
481 	 */
482 	ASSERT(rootnex_state == NULL);
483 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
484 
485 	rootnex_state->r_dip = dip;
486 	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
487 	rootnex_state->r_reserved_msg_printed = B_FALSE;
488 #ifdef DEBUG
489 	rootnex_cnt = &rootnex_state->r_counters[0];
490 #endif
491 
492 	/*
493 	 * Set minimum fm capability level for i86pc platforms and then
494 	 * initialize error handling. Since we're the rootnex, we don't
495 	 * care what's returned in the fmcap field.
496 	 */
497 	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
498 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
499 	fmcap = ddi_system_fmcap;
500 	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
501 
502 	/* initialize DMA related state */
503 	e = rootnex_dma_init();
504 	if (e != DDI_SUCCESS) {
505 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
506 		return (DDI_FAILURE);
507 	}
508 
509 	/* Add static root node properties */
510 	rootnex_add_props(dip);
511 
512 	/* since we can't call ddi_report_dev() */
513 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
514 
515 	/* Initialize rootnex event handle */
516 	i_ddi_rootnex_init_events(dip);
517 
518 #if defined(__amd64) && !defined(__xpv)
519 	e = iommulib_nexus_register(dip, &iommulib_nexops,
520 	    &rootnex_state->r_iommulib_handle);
521 
522 	ASSERT(e == DDI_SUCCESS);
523 #endif
524 
525 	return (DDI_SUCCESS);
526 }
527 
528 
529 /*
530  * rootnex_detach()
531  *
532  */
533 /*ARGSUSED*/
534 static int
535 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
536 {
537 	switch (cmd) {
538 	case DDI_SUSPEND:
539 #if defined(__amd64) && !defined(__xpv)
540 		return (immu_quiesce());
541 #else
542 		return (DDI_SUCCESS);
543 #endif
544 	default:
545 		return (DDI_FAILURE);
546 	}
547 	/*NOTREACHED*/
548 
549 }
550 
551 
552 /*
553  * rootnex_dma_init()
554  *
555  */
556 /*ARGSUSED*/
557 static int
558 rootnex_dma_init()
559 {
560 	size_t bufsize;
561 
562 
563 	/*
564 	 * size of our cookie/window/copybuf state needed in dma bind that we
565 	 * pre-alloc in dma_alloc_handle
566 	 */
567 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
568 	rootnex_state->r_prealloc_size =
569 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
570 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
571 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
572 
573 	/*
574 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
575 	 * allocate 16 extra bytes for struct pointer alignment
576 	 * (p->dmai_private & dma->dp_prealloc_buffer)
577 	 */
578 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
579 	    rootnex_state->r_prealloc_size + 0x10;
580 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
581 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
582 	if (rootnex_state->r_dmahdl_cache == NULL) {
583 		return (DDI_FAILURE);
584 	}
585 
586 	/*
587 	 * allocate array to track which major numbers we have printed warnings
588 	 * for.
589 	 */
590 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
591 	    KM_SLEEP);
592 
593 	return (DDI_SUCCESS);
594 }
595 
596 
597 /*
598  * rootnex_add_props()
599  *
600  */
601 static void
602 rootnex_add_props(dev_info_t *dip)
603 {
604 	rootnex_intprop_t *rpp;
605 	int i;
606 
607 	/* Add static integer/boolean properties to the root node */
608 	rpp = rootnex_intprp;
609 	for (i = 0; i < NROOT_INTPROPS; i++) {
610 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
611 		    rpp[i].prop_name, rpp[i].prop_value);
612 	}
613 }
614 
615 
616 
617 /*
618  * *************************
619  *  ctlops related routines
620  * *************************
621  */
622 
623 /*
624  * rootnex_ctlops()
625  *
626  */
627 /*ARGSUSED*/
628 static int
629 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
630     void *arg, void *result)
631 {
632 	int n, *ptr;
633 	struct ddi_parent_private_data *pdp;
634 
635 	switch (ctlop) {
636 	case DDI_CTLOPS_DMAPMAPC:
637 		/*
638 		 * Return 'partial' to indicate that dma mapping
639 		 * has to be done in the main MMU.
640 		 */
641 		return (DDI_DMA_PARTIAL);
642 
643 	case DDI_CTLOPS_BTOP:
644 		/*
645 		 * Convert byte count input to physical page units.
646 		 * (byte counts that are not a page-size multiple
647 		 * are rounded down)
648 		 */
649 		*(ulong_t *)result = btop(*(ulong_t *)arg);
650 		return (DDI_SUCCESS);
651 
652 	case DDI_CTLOPS_PTOB:
653 		/*
654 		 * Convert size in physical pages to bytes
655 		 */
656 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
657 		return (DDI_SUCCESS);
658 
659 	case DDI_CTLOPS_BTOPR:
660 		/*
661 		 * Convert byte count input to physical page units
662 		 * (byte counts that are not a page-size multiple
663 		 * are rounded up)
664 		 */
665 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
666 		return (DDI_SUCCESS);
667 
668 	case DDI_CTLOPS_INITCHILD:
669 		return (impl_ddi_sunbus_initchild(arg));
670 
671 	case DDI_CTLOPS_UNINITCHILD:
672 		impl_ddi_sunbus_removechild(arg);
673 		return (DDI_SUCCESS);
674 
675 	case DDI_CTLOPS_REPORTDEV:
676 		return (rootnex_ctl_reportdev(rdip));
677 
678 	case DDI_CTLOPS_IOMIN:
679 		/*
680 		 * Nothing to do here but reflect back..
681 		 */
682 		return (DDI_SUCCESS);
683 
684 	case DDI_CTLOPS_REGSIZE:
685 	case DDI_CTLOPS_NREGS:
686 		break;
687 
688 	case DDI_CTLOPS_SIDDEV:
689 		if (ndi_dev_is_prom_node(rdip))
690 			return (DDI_SUCCESS);
691 		if (ndi_dev_is_persistent_node(rdip))
692 			return (DDI_SUCCESS);
693 		return (DDI_FAILURE);
694 
695 	case DDI_CTLOPS_POWER:
696 		return ((*pm_platform_power)((power_req_t *)arg));
697 
698 	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
699 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
700 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
701 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
702 	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
703 	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
704 		if (!rootnex_state->r_reserved_msg_printed) {
705 			rootnex_state->r_reserved_msg_printed = B_TRUE;
706 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
707 			    "1 or more reserved/obsolete operations.");
708 		}
709 		return (DDI_FAILURE);
710 
711 	default:
712 		return (DDI_FAILURE);
713 	}
714 	/*
715 	 * The rest are for "hardware" properties
716 	 */
717 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
718 		return (DDI_FAILURE);
719 
720 	if (ctlop == DDI_CTLOPS_NREGS) {
721 		ptr = (int *)result;
722 		*ptr = pdp->par_nreg;
723 	} else {
724 		off_t *size = (off_t *)result;
725 
726 		ptr = (int *)arg;
727 		n = *ptr;
728 		if (n >= pdp->par_nreg) {
729 			return (DDI_FAILURE);
730 		}
731 		*size = (off_t)pdp->par_reg[n].regspec_size;
732 	}
733 	return (DDI_SUCCESS);
734 }
735 
736 
737 /*
738  * rootnex_ctl_reportdev()
739  *
740  */
741 static int
742 rootnex_ctl_reportdev(dev_info_t *dev)
743 {
744 	int i, n, len, f_len = 0;
745 	char *buf;
746 
747 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
748 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
749 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
750 	len = strlen(buf);
751 
752 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
753 
754 		struct regspec *rp = sparc_pd_getreg(dev, i);
755 
756 		if (i == 0)
757 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
758 			    ": ");
759 		else
760 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
761 			    " and ");
762 		len = strlen(buf);
763 
764 		switch (rp->regspec_bustype) {
765 
766 		case BTEISA:
767 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
768 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
769 			break;
770 
771 		case BTISA:
772 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
773 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
774 			break;
775 
776 		default:
777 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
778 			    "space %x offset %x",
779 			    rp->regspec_bustype, rp->regspec_addr);
780 			break;
781 		}
782 		len = strlen(buf);
783 	}
784 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
785 		int pri;
786 
787 		if (i != 0) {
788 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
789 			    ",");
790 			len = strlen(buf);
791 		}
792 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
793 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
794 		    " sparc ipl %d", pri);
795 		len = strlen(buf);
796 	}
797 #ifdef DEBUG
798 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
799 		cmn_err(CE_NOTE, "next message is truncated: "
800 		    "printed length 1024, real length %d", f_len);
801 	}
802 #endif /* DEBUG */
803 	cmn_err(CE_CONT, "?%s\n", buf);
804 	kmem_free(buf, REPORTDEV_BUFSIZE);
805 	return (DDI_SUCCESS);
806 }
807 
808 
809 /*
810  * ******************
811  *  map related code
812  * ******************
813  */
814 
815 /*
816  * rootnex_map()
817  *
818  */
819 static int
820 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
821     off_t len, caddr_t *vaddrp)
822 {
823 	struct regspec *orp = NULL;
824 	struct regspec64 rp = { 0 };
825 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
826 
827 	mp = &mr;
828 
829 	switch (mp->map_op)  {
830 	case DDI_MO_MAP_LOCKED:
831 	case DDI_MO_UNMAP:
832 	case DDI_MO_MAP_HANDLE:
833 		break;
834 	default:
835 #ifdef	DDI_MAP_DEBUG
836 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
837 		    mp->map_op);
838 #endif	/* DDI_MAP_DEBUG */
839 		return (DDI_ME_UNIMPLEMENTED);
840 	}
841 
842 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
843 #ifdef	DDI_MAP_DEBUG
844 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
845 #endif	/* DDI_MAP_DEBUG */
846 		return (DDI_ME_UNIMPLEMENTED);
847 	}
848 
849 	/*
850 	 * First, we need to get the original regspec out before we convert it
851 	 * to the extended format. If we have a register number, then we need to
852 	 * convert that to a regspec.
853 	 */
854 	if (mp->map_type == DDI_MT_RNUMBER)  {
855 
856 		int rnumber = mp->map_obj.rnumber;
857 #ifdef	DDI_MAP_DEBUG
858 		static char *out_of_range =
859 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
860 #endif	/* DDI_MAP_DEBUG */
861 
862 		orp = i_ddi_rnumber_to_regspec(rdip, rnumber);
863 		if (orp == NULL) {
864 #ifdef	DDI_MAP_DEBUG
865 			cmn_err(CE_WARN, out_of_range, rnumber,
866 			    ddi_get_name(rdip));
867 #endif	/* DDI_MAP_DEBUG */
868 			return (DDI_ME_RNUMBER_RANGE);
869 		}
870 	} else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) {
871 		orp = mp->map_obj.rp;
872 	}
873 
874 	/*
875 	 * Ensure that we are always using a 64-bit extended regspec regardless
876 	 * of what was passed into us. If the child driver is using a 64-bit
877 	 * regspec, then we need to make sure that we copy this to the local
878 	 * regspec64, rp.
879 	 */
880 	if (orp != NULL) {
881 		rp.regspec_bustype = orp->regspec_bustype;
882 		rp.regspec_addr = orp->regspec_addr;
883 		rp.regspec_size = orp->regspec_size;
884 	} else {
885 		struct regspec64 *rp64;
886 		rp64 = (struct regspec64 *)mp->map_obj.rp;
887 		rp = *rp64;
888 	}
889 
890 	mp->map_type = DDI_MT_REGSPEC;
891 	mp->map_flags |= DDI_MF_EXT_REGSPEC;
892 	mp->map_obj.rp = (struct regspec *)&rp;
893 
894 	/*
895 	 * Adjust offset and length correspnding to called values...
896 	 * XXX: A non-zero length means override the one in the regspec
897 	 * XXX: (regardless of what's in the parent's range?)
898 	 */
899 
900 #ifdef	DDI_MAP_DEBUG
901 	cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d "
902 	    "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip),
903 	    rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset,
904 	    len, mp->map_handlep);
905 #endif	/* DDI_MAP_DEBUG */
906 
907 	/*
908 	 * I/O or memory mapping:
909 	 *
910 	 *	<bustype=0, addr=x, len=x>: memory
911 	 *	<bustype=1, addr=x, len=x>: i/o
912 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
913 	 */
914 
915 	if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) {
916 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
917 		    " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">",
918 		    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
919 		    rp.regspec_addr, rp.regspec_size);
920 		return (DDI_ME_INVAL);
921 	}
922 
923 	if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) {
924 		/*
925 		 * compatibility i/o mapping
926 		 */
927 		rp.regspec_bustype += offset;
928 	} else {
929 		/*
930 		 * Normal memory or i/o mapping
931 		 */
932 		rp.regspec_addr += offset;
933 	}
934 
935 	if (len != 0)
936 		rp.regspec_size = len;
937 
938 #ifdef	DDI_MAP_DEBUG
939 	cmn_err(CE_CONT, "             <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64
940 	    ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n",
941 	    ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype,
942 	    rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep);
943 #endif	/* DDI_MAP_DEBUG */
944 
945 
946 	/*
947 	 * The x86 root nexus does not have any notion of valid ranges of
948 	 * addresses. Its children have valid ranges, but because there are none
949 	 * for the nexus, we don't need to call i_ddi_apply_range().  Verify
950 	 * that is the case.
951 	 */
952 	ASSERT0(sparc_pd_getnrng(dip));
953 
954 	switch (mp->map_op)  {
955 	case DDI_MO_MAP_LOCKED:
956 
957 		/*
958 		 * Set up the locked down kernel mapping to the regspec...
959 		 */
960 
961 		return (rootnex_map_regspec(mp, vaddrp));
962 
963 	case DDI_MO_UNMAP:
964 
965 		/*
966 		 * Release mapping...
967 		 */
968 
969 		return (rootnex_unmap_regspec(mp, vaddrp));
970 
971 	case DDI_MO_MAP_HANDLE:
972 
973 		return (rootnex_map_handle(mp));
974 
975 	default:
976 		return (DDI_ME_UNIMPLEMENTED);
977 	}
978 }
979 
980 
981 /*
982  * rootnex_map_fault()
983  *
984  *	fault in mappings for requestors
985  */
986 /*ARGSUSED*/
987 static int
988 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
989     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
990     uint_t lock)
991 {
992 
993 #ifdef	DDI_MAP_DEBUG
994 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
995 	ddi_map_debug(" Seg <%s>\n",
996 	    seg->s_ops == &segdev_ops ? "segdev" :
997 	    seg == &kvseg ? "segkmem" : "NONE!");
998 #endif	/* DDI_MAP_DEBUG */
999 
1000 	/*
1001 	 * This is all terribly broken, but it is a start
1002 	 *
1003 	 * XXX	Note that this test means that segdev_ops
1004 	 *	must be exported from seg_dev.c.
1005 	 * XXX	What about devices with their own segment drivers?
1006 	 */
1007 	if (seg->s_ops == &segdev_ops) {
1008 		struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1009 
1010 		if (hat == NULL) {
1011 			/*
1012 			 * This is one plausible interpretation of
1013 			 * a null hat i.e. use the first hat on the
1014 			 * address space hat list which by convention is
1015 			 * the hat of the system MMU.  At alternative
1016 			 * would be to panic .. this might well be better ..
1017 			 */
1018 			ASSERT(AS_READ_HELD(seg->s_as));
1019 			hat = seg->s_as->a_hat;
1020 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1021 		}
1022 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1023 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1024 	} else if (seg == &kvseg && dp == NULL) {
1025 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1026 		    HAT_LOAD_LOCK);
1027 	} else
1028 		return (DDI_FAILURE);
1029 	return (DDI_SUCCESS);
1030 }
1031 
1032 
1033 static int
1034 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1035 {
1036 	rootnex_addr_t rbase;
1037 	void *cvaddr;
1038 	uint64_t npages, pgoffset;
1039 	struct regspec64 *rp;
1040 	ddi_acc_hdl_t *hp;
1041 	ddi_acc_impl_t *ap;
1042 	uint_t	hat_acc_flags;
1043 	paddr_t pbase;
1044 
1045 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1046 	rp = (struct regspec64 *)mp->map_obj.rp;
1047 	hp = mp->map_handlep;
1048 
1049 #ifdef	DDI_MAP_DEBUG
1050 	ddi_map_debug(
1051 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1052 	    rp->regspec_bustype, rp->regspec_addr,
1053 	    rp->regspec_size, mp->map_handlep);
1054 #endif	/* DDI_MAP_DEBUG */
1055 
1056 	/*
1057 	 * I/O or memory mapping
1058 	 *
1059 	 *	<bustype=0, addr=x, len=x>: memory
1060 	 *	<bustype=1, addr=x, len=x>: i/o
1061 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1062 	 */
1063 
1064 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1065 		cmn_err(CE_WARN, "rootnex: invalid register spec"
1066 		    " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">",
1067 		    rp->regspec_bustype, rp->regspec_addr, rp->regspec_size);
1068 		return (DDI_FAILURE);
1069 	}
1070 
1071 	if (rp->regspec_bustype != 0) {
1072 		/*
1073 		 * I/O space - needs a handle.
1074 		 */
1075 		if (hp == NULL) {
1076 			return (DDI_FAILURE);
1077 		}
1078 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1079 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1080 		impl_acc_hdl_init(hp);
1081 
1082 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1083 #ifdef  DDI_MAP_DEBUG
1084 			ddi_map_debug("rootnex_map_regspec: mmap() "
1085 			    "to I/O space is not supported.\n");
1086 #endif  /* DDI_MAP_DEBUG */
1087 			return (DDI_ME_INVAL);
1088 		} else {
1089 			/*
1090 			 * 1275-compliant vs. compatibility i/o mapping
1091 			 */
1092 			*vaddrp =
1093 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1094 			    ((caddr_t)(uintptr_t)rp->regspec_bustype) :
1095 			    ((caddr_t)(uintptr_t)rp->regspec_addr);
1096 #ifdef __xpv
1097 			if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1098 				hp->ah_pfn = xen_assign_pfn(
1099 				    mmu_btop((ulong_t)rp->regspec_addr &
1100 				    MMU_PAGEMASK));
1101 			} else {
1102 				hp->ah_pfn = mmu_btop(
1103 				    (ulong_t)rp->regspec_addr & MMU_PAGEMASK);
1104 			}
1105 #else
1106 			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
1107 			    MMU_PAGEMASK);
1108 #endif
1109 			hp->ah_pnum = mmu_btopr(rp->regspec_size +
1110 			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
1111 		}
1112 
1113 #ifdef	DDI_MAP_DEBUG
1114 		ddi_map_debug(
1115 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1116 		    rp->regspec_size, *vaddrp);
1117 #endif	/* DDI_MAP_DEBUG */
1118 		return (DDI_SUCCESS);
1119 	}
1120 
1121 	/*
1122 	 * Memory space
1123 	 */
1124 
1125 	if (hp != NULL) {
1126 		/*
1127 		 * hat layer ignores
1128 		 * hp->ah_acc.devacc_attr_endian_flags.
1129 		 */
1130 		switch (hp->ah_acc.devacc_attr_dataorder) {
1131 		case DDI_STRICTORDER_ACC:
1132 			hat_acc_flags = HAT_STRICTORDER;
1133 			break;
1134 		case DDI_UNORDERED_OK_ACC:
1135 			hat_acc_flags = HAT_UNORDERED_OK;
1136 			break;
1137 		case DDI_MERGING_OK_ACC:
1138 			hat_acc_flags = HAT_MERGING_OK;
1139 			break;
1140 		case DDI_LOADCACHING_OK_ACC:
1141 			hat_acc_flags = HAT_LOADCACHING_OK;
1142 			break;
1143 		case DDI_STORECACHING_OK_ACC:
1144 			hat_acc_flags = HAT_STORECACHING_OK;
1145 			break;
1146 		default:
1147 			return (DDI_ME_INVAL);
1148 		}
1149 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1150 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1151 		impl_acc_hdl_init(hp);
1152 		hp->ah_hat_flags = hat_acc_flags;
1153 	} else {
1154 		hat_acc_flags = HAT_STRICTORDER;
1155 	}
1156 
1157 	rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK);
1158 #ifdef __xpv
1159 	/*
1160 	 * If we're dom0, we're using a real device so we need to translate
1161 	 * the MA to a PA.
1162 	 */
1163 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1164 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase)));
1165 	} else {
1166 		pbase = rbase;
1167 	}
1168 #else
1169 	pbase = rbase;
1170 #endif
1171 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1172 
1173 	if (rp->regspec_size == 0) {
1174 #ifdef  DDI_MAP_DEBUG
1175 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1176 #endif  /* DDI_MAP_DEBUG */
1177 		return (DDI_ME_INVAL);
1178 	}
1179 
1180 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1181 		/* extra cast to make gcc happy */
1182 		*vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase));
1183 	} else {
1184 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1185 
1186 #ifdef	DDI_MAP_DEBUG
1187 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages "
1188 		    "physical %llx", npages, pbase);
1189 #endif	/* DDI_MAP_DEBUG */
1190 
1191 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1192 		if (cvaddr == NULL)
1193 			return (DDI_ME_NORESOURCES);
1194 
1195 		/*
1196 		 * Now map in the pages we've allocated...
1197 		 */
1198 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages),
1199 		    mmu_btop(pbase), mp->map_prot | hat_acc_flags,
1200 		    HAT_LOAD_LOCK);
1201 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1202 
1203 		/* save away pfn and npages for FMA */
1204 		hp = mp->map_handlep;
1205 		if (hp) {
1206 			hp->ah_pfn = mmu_btop(pbase);
1207 			hp->ah_pnum = npages;
1208 		}
1209 	}
1210 
1211 #ifdef	DDI_MAP_DEBUG
1212 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1213 #endif	/* DDI_MAP_DEBUG */
1214 	return (DDI_SUCCESS);
1215 }
1216 
1217 
1218 static int
1219 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1220 {
1221 	caddr_t addr = (caddr_t)*vaddrp;
1222 	uint64_t npages, pgoffset;
1223 	struct regspec64 *rp;
1224 
1225 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1226 		return (0);
1227 
1228 	ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC);
1229 	rp = (struct regspec64 *)mp->map_obj.rp;
1230 
1231 	if (rp->regspec_size == 0) {
1232 #ifdef  DDI_MAP_DEBUG
1233 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1234 #endif  /* DDI_MAP_DEBUG */
1235 		return (DDI_ME_INVAL);
1236 	}
1237 
1238 	/*
1239 	 * I/O or memory mapping:
1240 	 *
1241 	 *	<bustype=0, addr=x, len=x>: memory
1242 	 *	<bustype=1, addr=x, len=x>: i/o
1243 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1244 	 */
1245 	if (rp->regspec_bustype != 0) {
1246 		/*
1247 		 * This is I/O space, which requires no particular
1248 		 * processing on unmap since it isn't mapped in the
1249 		 * first place.
1250 		 */
1251 		return (DDI_SUCCESS);
1252 	}
1253 
1254 	/*
1255 	 * Memory space
1256 	 */
1257 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1258 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1259 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1260 	device_arena_free(addr - pgoffset, ptob(npages));
1261 
1262 	/*
1263 	 * Destroy the pointer - the mapping has logically gone
1264 	 */
1265 	*vaddrp = NULL;
1266 
1267 	return (DDI_SUCCESS);
1268 }
1269 
1270 static int
1271 rootnex_map_handle(ddi_map_req_t *mp)
1272 {
1273 	rootnex_addr_t rbase;
1274 	ddi_acc_hdl_t *hp;
1275 	uint64_t pgoffset;
1276 	struct regspec64 *rp;
1277 	paddr_t pbase;
1278 
1279 	rp = (struct regspec64 *)mp->map_obj.rp;
1280 
1281 #ifdef	DDI_MAP_DEBUG
1282 	ddi_map_debug(
1283 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1284 	    rp->regspec_bustype, rp->regspec_addr,
1285 	    rp->regspec_size, mp->map_handlep);
1286 #endif	/* DDI_MAP_DEBUG */
1287 
1288 	/*
1289 	 * I/O or memory mapping:
1290 	 *
1291 	 *	<bustype=0, addr=x, len=x>: memory
1292 	 *	<bustype=1, addr=x, len=x>: i/o
1293 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1294 	 */
1295 	if (rp->regspec_bustype != 0) {
1296 		/*
1297 		 * This refers to I/O space, and we don't support "mapping"
1298 		 * I/O space to a user.
1299 		 */
1300 		return (DDI_FAILURE);
1301 	}
1302 
1303 	/*
1304 	 * Set up the hat_flags for the mapping.
1305 	 */
1306 	hp = mp->map_handlep;
1307 
1308 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1309 	case DDI_NEVERSWAP_ACC:
1310 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1311 		break;
1312 	case DDI_STRUCTURE_LE_ACC:
1313 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1314 		break;
1315 	case DDI_STRUCTURE_BE_ACC:
1316 		return (DDI_FAILURE);
1317 	default:
1318 		return (DDI_REGS_ACC_CONFLICT);
1319 	}
1320 
1321 	switch (hp->ah_acc.devacc_attr_dataorder) {
1322 	case DDI_STRICTORDER_ACC:
1323 		break;
1324 	case DDI_UNORDERED_OK_ACC:
1325 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1326 		break;
1327 	case DDI_MERGING_OK_ACC:
1328 		hp->ah_hat_flags |= HAT_MERGING_OK;
1329 		break;
1330 	case DDI_LOADCACHING_OK_ACC:
1331 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1332 		break;
1333 	case DDI_STORECACHING_OK_ACC:
1334 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1335 		break;
1336 	default:
1337 		return (DDI_FAILURE);
1338 	}
1339 
1340 	rbase = (rootnex_addr_t)rp->regspec_addr &
1341 	    (~(rootnex_addr_t)MMU_PAGEOFFSET);
1342 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET;
1343 
1344 	if (rp->regspec_size == 0)
1345 		return (DDI_ME_INVAL);
1346 
1347 #ifdef __xpv
1348 	/*
1349 	 * If we're dom0, we're using a real device so we need to translate
1350 	 * the MA to a PA.
1351 	 */
1352 	if (DOMAIN_IS_INITDOMAIN(xen_info)) {
1353 		pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) |
1354 		    (rbase & MMU_PAGEOFFSET);
1355 	} else {
1356 		pbase = rbase;
1357 	}
1358 #else
1359 	pbase = rbase;
1360 #endif
1361 
1362 	hp->ah_pfn = mmu_btop(pbase);
1363 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1364 
1365 	return (DDI_SUCCESS);
1366 }
1367 
1368 
1369 
1370 /*
1371  * ************************
1372  *  interrupt related code
1373  * ************************
1374  */
1375 
1376 /*
1377  * rootnex_intr_ops()
1378  *	bus_intr_op() function for interrupt support
1379  */
1380 /* ARGSUSED */
1381 static int
1382 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1383     ddi_intr_handle_impl_t *hdlp, void *result)
1384 {
1385 	struct intrspec			*ispec;
1386 
1387 	DDI_INTR_NEXDBG((CE_CONT,
1388 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1389 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1390 
1391 	/* Process the interrupt operation */
1392 	switch (intr_op) {
1393 	case DDI_INTROP_GETCAP:
1394 		/* First check with pcplusmp */
1395 		if (psm_intr_ops == NULL)
1396 			return (DDI_FAILURE);
1397 
1398 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1399 			*(int *)result = 0;
1400 			return (DDI_FAILURE);
1401 		}
1402 		break;
1403 	case DDI_INTROP_SETCAP:
1404 		if (psm_intr_ops == NULL)
1405 			return (DDI_FAILURE);
1406 
1407 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1408 			return (DDI_FAILURE);
1409 		break;
1410 	case DDI_INTROP_ALLOC:
1411 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1412 		return (rootnex_alloc_intr_fixed(rdip, hdlp, result));
1413 	case DDI_INTROP_FREE:
1414 		ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED);
1415 		return (rootnex_free_intr_fixed(rdip, hdlp));
1416 	case DDI_INTROP_GETPRI:
1417 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1418 			return (DDI_FAILURE);
1419 		*(int *)result = ispec->intrspec_pri;
1420 		break;
1421 	case DDI_INTROP_SETPRI:
1422 		/* Validate the interrupt priority passed to us */
1423 		if (*(int *)result > LOCK_LEVEL)
1424 			return (DDI_FAILURE);
1425 
1426 		/* Ensure that PSM is all initialized and ispec is ok */
1427 		if ((psm_intr_ops == NULL) ||
1428 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1429 			return (DDI_FAILURE);
1430 
1431 		/* Change the priority */
1432 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1433 		    PSM_FAILURE)
1434 			return (DDI_FAILURE);
1435 
1436 		/* update the ispec with the new priority */
1437 		ispec->intrspec_pri =  *(int *)result;
1438 		break;
1439 	case DDI_INTROP_ADDISR:
1440 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1441 			return (DDI_FAILURE);
1442 		ispec->intrspec_func = hdlp->ih_cb_func;
1443 		break;
1444 	case DDI_INTROP_REMISR:
1445 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1446 			return (DDI_FAILURE);
1447 		ispec->intrspec_func = (uint_t (*)()) 0;
1448 		break;
1449 	case DDI_INTROP_ENABLE:
1450 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1451 			return (DDI_FAILURE);
1452 
1453 		/* Call psmi to translate irq with the dip */
1454 		if (psm_intr_ops == NULL)
1455 			return (DDI_FAILURE);
1456 
1457 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1458 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1459 		    (int *)&hdlp->ih_vector) == PSM_FAILURE)
1460 			return (DDI_FAILURE);
1461 
1462 		/* Add the interrupt handler */
1463 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1464 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1465 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1466 			return (DDI_FAILURE);
1467 		break;
1468 	case DDI_INTROP_DISABLE:
1469 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1470 			return (DDI_FAILURE);
1471 
1472 		/* Call psm_ops() to translate irq with the dip */
1473 		if (psm_intr_ops == NULL)
1474 			return (DDI_FAILURE);
1475 
1476 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1477 		(void) (*psm_intr_ops)(rdip, hdlp,
1478 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1479 
1480 		/* Remove the interrupt handler */
1481 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1482 		    hdlp->ih_cb_func, hdlp->ih_vector);
1483 		break;
1484 	case DDI_INTROP_SETMASK:
1485 		if (psm_intr_ops == NULL)
1486 			return (DDI_FAILURE);
1487 
1488 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1489 			return (DDI_FAILURE);
1490 		break;
1491 	case DDI_INTROP_CLRMASK:
1492 		if (psm_intr_ops == NULL)
1493 			return (DDI_FAILURE);
1494 
1495 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1496 			return (DDI_FAILURE);
1497 		break;
1498 	case DDI_INTROP_GETPENDING:
1499 		if (psm_intr_ops == NULL)
1500 			return (DDI_FAILURE);
1501 
1502 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1503 		    result)) {
1504 			*(int *)result = 0;
1505 			return (DDI_FAILURE);
1506 		}
1507 		break;
1508 	case DDI_INTROP_NAVAIL:
1509 	case DDI_INTROP_NINTRS:
1510 		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1511 		if (*(int *)result == 0) {
1512 			/*
1513 			 * Special case for 'pcic' driver' only. This driver
1514 			 * driver is a child of 'isa' and 'rootnex' drivers.
1515 			 *
1516 			 * See detailed comments on this in the function
1517 			 * rootnex_get_ispec().
1518 			 *
1519 			 * Children of 'pcic' send 'NINITR' request all the
1520 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1521 			 * field may not initialized. So, we fake it here
1522 			 * to return 1 (a la what PCMCIA nexus does).
1523 			 */
1524 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1525 				*(int *)result = 1;
1526 			else
1527 				return (DDI_FAILURE);
1528 		}
1529 		break;
1530 	case DDI_INTROP_SUPPORTED_TYPES:
1531 		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1532 		break;
1533 	default:
1534 		return (DDI_FAILURE);
1535 	}
1536 
1537 	return (DDI_SUCCESS);
1538 }
1539 
1540 
1541 /*
1542  * rootnex_get_ispec()
1543  *	convert an interrupt number to an interrupt specification.
1544  *	The interrupt number determines which interrupt spec will be
1545  *	returned if more than one exists.
1546  *
1547  *	Look into the parent private data area of the 'rdip' to find out
1548  *	the interrupt specification.  First check to make sure there is
1549  *	one that matchs "inumber" and then return a pointer to it.
1550  *
1551  *	Return NULL if one could not be found.
1552  *
1553  *	NOTE: This is needed for rootnex_intr_ops()
1554  */
1555 static struct intrspec *
1556 rootnex_get_ispec(dev_info_t *rdip, int inum)
1557 {
1558 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1559 
1560 	/*
1561 	 * Special case handling for drivers that provide their own
1562 	 * intrspec structures instead of relying on the DDI framework.
1563 	 *
1564 	 * A broken hardware driver in ON could potentially provide its
1565 	 * own intrspec structure, instead of relying on the hardware.
1566 	 * If these drivers are children of 'rootnex' then we need to
1567 	 * continue to provide backward compatibility to them here.
1568 	 *
1569 	 * Following check is a special case for 'pcic' driver which
1570 	 * was found to have broken hardwre andby provides its own intrspec.
1571 	 *
1572 	 * Verbatim comments from this driver are shown here:
1573 	 * "Don't use the ddi_add_intr since we don't have a
1574 	 * default intrspec in all cases."
1575 	 *
1576 	 * Since an 'ispec' may not be always created for it,
1577 	 * check for that and create one if so.
1578 	 *
1579 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1580 	 */
1581 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1582 		pdp->par_nintr = 1;
1583 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1584 		    pdp->par_nintr, KM_SLEEP);
1585 	}
1586 
1587 	/* Validate the interrupt number */
1588 	if (inum >= pdp->par_nintr)
1589 		return (NULL);
1590 
1591 	/* Get the interrupt structure pointer and return that */
1592 	return ((struct intrspec *)&pdp->par_intr[inum]);
1593 }
1594 
1595 /*
1596  * Allocate interrupt vector for FIXED (legacy) type.
1597  */
1598 static int
1599 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp,
1600     void *result)
1601 {
1602 	struct intrspec		*ispec;
1603 	ddi_intr_handle_impl_t	info_hdl;
1604 	int			ret;
1605 	int			free_phdl = 0;
1606 	apic_get_type_t		type_info;
1607 
1608 	if (psm_intr_ops == NULL)
1609 		return (DDI_FAILURE);
1610 
1611 	if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1612 		return (DDI_FAILURE);
1613 
1614 	/*
1615 	 * If the PSM module is "APIX" then pass the request for it
1616 	 * to allocate the vector now.
1617 	 */
1618 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1619 	info_hdl.ih_private = &type_info;
1620 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1621 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1622 		if (hdlp->ih_private == NULL) { /* allocate phdl structure */
1623 			free_phdl = 1;
1624 			i_ddi_alloc_intr_phdl(hdlp);
1625 		}
1626 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1627 		ret = (*psm_intr_ops)(rdip, hdlp,
1628 		    PSM_INTR_OP_ALLOC_VECTORS, result);
1629 		if (free_phdl) { /* free up the phdl structure */
1630 			free_phdl = 0;
1631 			i_ddi_free_intr_phdl(hdlp);
1632 			hdlp->ih_private = NULL;
1633 		}
1634 	} else {
1635 		/*
1636 		 * No APIX module; fall back to the old scheme where the
1637 		 * interrupt vector is allocated during ddi_enable_intr() call.
1638 		 */
1639 		hdlp->ih_pri = ispec->intrspec_pri;
1640 		*(int *)result = hdlp->ih_scratch1;
1641 		ret = DDI_SUCCESS;
1642 	}
1643 
1644 	return (ret);
1645 }
1646 
1647 /*
1648  * Free up interrupt vector for FIXED (legacy) type.
1649  */
1650 static int
1651 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp)
1652 {
1653 	struct intrspec			*ispec;
1654 	struct ddi_parent_private_data	*pdp;
1655 	ddi_intr_handle_impl_t		info_hdl;
1656 	int				ret;
1657 	apic_get_type_t			type_info;
1658 
1659 	if (psm_intr_ops == NULL)
1660 		return (DDI_FAILURE);
1661 
1662 	/*
1663 	 * If the PSM module is "APIX" then pass the request for it
1664 	 * to free up the vector now.
1665 	 */
1666 	bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t));
1667 	info_hdl.ih_private = &type_info;
1668 	if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) ==
1669 	    PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) {
1670 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1671 			return (DDI_FAILURE);
1672 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1673 		ret = (*psm_intr_ops)(rdip, hdlp,
1674 		    PSM_INTR_OP_FREE_VECTORS, NULL);
1675 	} else {
1676 		/*
1677 		 * No APIX module; fall back to the old scheme where
1678 		 * the interrupt vector was already freed during
1679 		 * ddi_disable_intr() call.
1680 		 */
1681 		ret = DDI_SUCCESS;
1682 	}
1683 
1684 	pdp = ddi_get_parent_data(rdip);
1685 
1686 	/*
1687 	 * Special case for 'pcic' driver' only.
1688 	 * If an intrspec was created for it, clean it up here
1689 	 * See detailed comments on this in the function
1690 	 * rootnex_get_ispec().
1691 	 */
1692 	if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1693 		kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1694 		    pdp->par_nintr);
1695 		/*
1696 		 * Set it to zero; so that
1697 		 * DDI framework doesn't free it again
1698 		 */
1699 		pdp->par_intr = NULL;
1700 		pdp->par_nintr = 0;
1701 	}
1702 
1703 	return (ret);
1704 }
1705 
1706 
1707 /*
1708  * ******************
1709  *  dma related code
1710  * ******************
1711  */
1712 
1713 /*ARGSUSED*/
1714 static int
1715 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip,
1716     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
1717     ddi_dma_handle_t *handlep)
1718 {
1719 	uint64_t maxsegmentsize_ll;
1720 	uint_t maxsegmentsize;
1721 	ddi_dma_impl_t *hp;
1722 	rootnex_dma_t *dma;
1723 	uint64_t count_max;
1724 	uint64_t seg;
1725 	int kmflag;
1726 	int e;
1727 
1728 
1729 	/* convert our sleep flags */
1730 	if (waitfp == DDI_DMA_SLEEP) {
1731 		kmflag = KM_SLEEP;
1732 	} else {
1733 		kmflag = KM_NOSLEEP;
1734 	}
1735 
1736 	/*
1737 	 * We try to do only one memory allocation here. We'll do a little
1738 	 * pointer manipulation later. If the bind ends up taking more than
1739 	 * our prealloc's space, we'll have to allocate more memory in the
1740 	 * bind operation. Not great, but much better than before and the
1741 	 * best we can do with the current bind interfaces.
1742 	 */
1743 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1744 	if (hp == NULL)
1745 		return (DDI_DMA_NORESOURCES);
1746 
1747 	/* Do our pointer manipulation now, align the structures */
1748 	hp->dmai_private = (void *)(((uintptr_t)hp +
1749 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1750 	dma = (rootnex_dma_t *)hp->dmai_private;
1751 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1752 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1753 
1754 	/* setup the handle */
1755 	rootnex_clean_dmahdl(hp);
1756 	hp->dmai_error.err_fep = NULL;
1757 	hp->dmai_error.err_cf = NULL;
1758 	dma->dp_dip = rdip;
1759 	dma->dp_sglinfo.si_flags = attr->dma_attr_flags;
1760 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1761 
1762 	/*
1763 	 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU
1764 	 * is being used. Set the upper limit to the seg value.
1765 	 * There will be enough DVMA space to always get addresses
1766 	 * that will match the constraints.
1767 	 */
1768 	if (IOMMU_USED(rdip) &&
1769 	    (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) {
1770 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg;
1771 		dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG;
1772 	} else
1773 		dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1774 
1775 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1776 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1777 	hp->dmai_rdip = rdip;
1778 	hp->dmai_attr = *attr;
1779 
1780 	if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr)
1781 		dma->dp_sglinfo.si_cancross = B_FALSE;
1782 	else
1783 		dma->dp_sglinfo.si_cancross = B_TRUE;
1784 
1785 	/* we don't need to worry about the SPL since we do a tryenter */
1786 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1787 
1788 	/*
1789 	 * Figure out our maximum segment size. If the segment size is greater
1790 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1791 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1792 	 * dma_attr_count_max are size-1 type values.
1793 	 *
1794 	 * Maximum segment size is the largest physically contiguous chunk of
1795 	 * memory that we can return from a bind (i.e. the maximum size of a
1796 	 * single cookie).
1797 	 */
1798 
1799 	/* handle the rollover cases */
1800 	seg = attr->dma_attr_seg + 1;
1801 	if (seg < attr->dma_attr_seg) {
1802 		seg = attr->dma_attr_seg;
1803 	}
1804 	count_max = attr->dma_attr_count_max + 1;
1805 	if (count_max < attr->dma_attr_count_max) {
1806 		count_max = attr->dma_attr_count_max;
1807 	}
1808 
1809 	/*
1810 	 * granularity may or may not be a power of two. If it isn't, we can't
1811 	 * use a simple mask.
1812 	 */
1813 	if (!ISP2(attr->dma_attr_granular)) {
1814 		dma->dp_granularity_power_2 = B_FALSE;
1815 	} else {
1816 		dma->dp_granularity_power_2 = B_TRUE;
1817 	}
1818 
1819 	/*
1820 	 * maxxfer should be a whole multiple of granularity. If we're going to
1821 	 * break up a window because we're greater than maxxfer, we might as
1822 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1823 	 * worry about triming the window later on for this case.
1824 	 */
1825 	if (attr->dma_attr_granular > 1) {
1826 		if (dma->dp_granularity_power_2) {
1827 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1828 			    (attr->dma_attr_maxxfer &
1829 			    (attr->dma_attr_granular - 1));
1830 		} else {
1831 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1832 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1833 		}
1834 	} else {
1835 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1836 	}
1837 
1838 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1839 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1840 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1841 		maxsegmentsize = 0xFFFFFFFF;
1842 	} else {
1843 		maxsegmentsize = maxsegmentsize_ll;
1844 	}
1845 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1846 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1847 
1848 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1849 	if (rootnex_alloc_check_parms) {
1850 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1851 		if (e != DDI_SUCCESS) {
1852 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1853 			(void) rootnex_dma_freehdl(dip, rdip,
1854 			    (ddi_dma_handle_t)hp);
1855 			return (e);
1856 		}
1857 	}
1858 
1859 	*handlep = (ddi_dma_handle_t)hp;
1860 
1861 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1862 	ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t,
1863 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1864 
1865 	return (DDI_SUCCESS);
1866 }
1867 
1868 
1869 /*
1870  * rootnex_dma_allochdl()
1871  *    called from ddi_dma_alloc_handle().
1872  */
1873 static int
1874 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1875     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1876 {
1877 	int retval = DDI_SUCCESS;
1878 #if defined(__amd64) && !defined(__xpv)
1879 
1880 	if (IOMMU_UNITIALIZED(rdip)) {
1881 		retval = iommulib_nex_open(dip, rdip);
1882 
1883 		if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP)
1884 			return (retval);
1885 	}
1886 
1887 	if (IOMMU_UNUSED(rdip)) {
1888 		retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1889 		    handlep);
1890 	} else {
1891 		retval = iommulib_nexdma_allochdl(dip, rdip, attr,
1892 		    waitfp, arg, handlep);
1893 	}
1894 #else
1895 	retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg,
1896 	    handlep);
1897 #endif
1898 	switch (retval) {
1899 	case DDI_DMA_NORESOURCES:
1900 		if (waitfp != DDI_DMA_DONTWAIT) {
1901 			ddi_set_callback(waitfp, arg,
1902 			    &rootnex_state->r_dvma_call_list_id);
1903 		}
1904 		break;
1905 	case DDI_SUCCESS:
1906 		ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL);
1907 		break;
1908 	default:
1909 		break;
1910 	}
1911 	return (retval);
1912 }
1913 
1914 /*ARGSUSED*/
1915 static int
1916 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip,
1917     ddi_dma_handle_t handle)
1918 {
1919 	ddi_dma_impl_t *hp;
1920 	rootnex_dma_t *dma;
1921 
1922 
1923 	hp = (ddi_dma_impl_t *)handle;
1924 	dma = (rootnex_dma_t *)hp->dmai_private;
1925 
1926 	/* unbind should have been called first */
1927 	ASSERT(!dma->dp_inuse);
1928 
1929 	mutex_destroy(&dma->dp_mutex);
1930 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1931 
1932 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1933 	ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t,
1934 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1935 
1936 	return (DDI_SUCCESS);
1937 }
1938 
1939 /*
1940  * rootnex_dma_freehdl()
1941  *    called from ddi_dma_free_handle().
1942  */
1943 static int
1944 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1945 {
1946 	int ret;
1947 
1948 	ndi_fmc_remove(rdip, DMA_HANDLE, handle);
1949 #if defined(__amd64) && !defined(__xpv)
1950 	if (IOMMU_USED(rdip))
1951 		ret = iommulib_nexdma_freehdl(dip, rdip, handle);
1952 	else
1953 #endif
1954 	ret = rootnex_coredma_freehdl(dip, rdip, handle);
1955 
1956 	if (rootnex_state->r_dvma_call_list_id)
1957 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1958 
1959 	return (ret);
1960 }
1961 
1962 /*ARGSUSED*/
1963 static int
1964 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
1965     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
1966     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1967 {
1968 	rootnex_sglinfo_t *sinfo;
1969 	ddi_dma_obj_t *dmao;
1970 #if defined(__amd64) && !defined(__xpv)
1971 	struct dvmaseg *dvs;
1972 	ddi_dma_cookie_t *cookie;
1973 #endif
1974 	ddi_dma_attr_t *attr;
1975 	ddi_dma_impl_t *hp;
1976 	rootnex_dma_t *dma;
1977 	int kmflag;
1978 	int e;
1979 	uint_t ncookies;
1980 
1981 	hp = (ddi_dma_impl_t *)handle;
1982 	dma = (rootnex_dma_t *)hp->dmai_private;
1983 	dmao = &dma->dp_dma;
1984 	sinfo = &dma->dp_sglinfo;
1985 	attr = &hp->dmai_attr;
1986 
1987 	/* convert the sleep flags */
1988 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1989 		dma->dp_sleep_flags = kmflag = KM_SLEEP;
1990 	} else {
1991 		dma->dp_sleep_flags = kmflag = KM_NOSLEEP;
1992 	}
1993 
1994 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1995 
1996 	/*
1997 	 * This is useful for debugging a driver. Not as useful in a production
1998 	 * system. The only time this will fail is if you have a driver bug.
1999 	 */
2000 	if (rootnex_bind_check_inuse) {
2001 		/*
2002 		 * No one else should ever have this lock unless someone else
2003 		 * is trying to use this handle. So contention on the lock
2004 		 * is the same as inuse being set.
2005 		 */
2006 		e = mutex_tryenter(&dma->dp_mutex);
2007 		if (e == 0) {
2008 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2009 			return (DDI_DMA_INUSE);
2010 		}
2011 		if (dma->dp_inuse) {
2012 			mutex_exit(&dma->dp_mutex);
2013 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2014 			return (DDI_DMA_INUSE);
2015 		}
2016 		dma->dp_inuse = B_TRUE;
2017 		mutex_exit(&dma->dp_mutex);
2018 	}
2019 
2020 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
2021 	if (rootnex_bind_check_parms) {
2022 		e = rootnex_valid_bind_parms(dmareq, attr);
2023 		if (e != DDI_SUCCESS) {
2024 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2025 			rootnex_clean_dmahdl(hp);
2026 			return (e);
2027 		}
2028 	}
2029 
2030 	/* save away the original bind info */
2031 	dma->dp_dma = dmareq->dmar_object;
2032 
2033 #if defined(__amd64) && !defined(__xpv)
2034 	if (IOMMU_USED(rdip)) {
2035 		dmao = &dma->dp_dvma;
2036 		e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao);
2037 		switch (e) {
2038 		case DDI_SUCCESS:
2039 			if (sinfo->si_cancross ||
2040 			    dmao->dmao_obj.dvma_obj.dv_nseg != 1 ||
2041 			    dmao->dmao_size > sinfo->si_max_cookie_size) {
2042 				dma->dp_dvma_used = B_TRUE;
2043 				break;
2044 			}
2045 			sinfo->si_sgl_size = 1;
2046 			hp->dmai_rflags |= DMP_NOSYNC;
2047 
2048 			dma->dp_dvma_used = B_TRUE;
2049 			dma->dp_need_to_free_cookie = B_FALSE;
2050 
2051 			dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0];
2052 			cookie = hp->dmai_cookie = dma->dp_cookies =
2053 			    (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2054 			cookie->dmac_laddress = dvs->dvs_start +
2055 			    dmao->dmao_obj.dvma_obj.dv_off;
2056 			cookie->dmac_size = dvs->dvs_len;
2057 			cookie->dmac_type = 0;
2058 
2059 			ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *,
2060 			    rdip);
2061 			goto fast;
2062 		case DDI_ENOTSUP:
2063 			break;
2064 		default:
2065 			rootnex_clean_dmahdl(hp);
2066 			return (e);
2067 		}
2068 	}
2069 #endif
2070 
2071 	/*
2072 	 * Figure out a rough estimate of what maximum number of pages
2073 	 * this buffer could use (a high estimate of course).
2074 	 */
2075 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
2076 
2077 	if (dma->dp_dvma_used) {
2078 		/*
2079 		 * The number of physical pages is the worst case.
2080 		 *
2081 		 * For DVMA, the worst case is the length divided
2082 		 * by the maximum cookie length, plus 1. Add to that
2083 		 * the number of segment boundaries potentially crossed, and
2084 		 * the additional number of DVMA segments that was returned.
2085 		 *
2086 		 * In the normal case, for modern devices, si_cancross will
2087 		 * be false, and dv_nseg will be 1, and the fast path will
2088 		 * have been taken above.
2089 		 */
2090 		ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size)
2091 		    + 1;
2092 		if (sinfo->si_cancross)
2093 			ncookies +=
2094 			    (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1;
2095 		ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1);
2096 
2097 		sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies);
2098 	}
2099 
2100 	/*
2101 	 * We'll use the pre-allocated cookies for any bind that will *always*
2102 	 * fit (more important to be consistent, we don't want to create
2103 	 * additional degenerate cases).
2104 	 */
2105 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
2106 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
2107 		dma->dp_need_to_free_cookie = B_FALSE;
2108 		ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
2109 		    uint_t, sinfo->si_max_pages);
2110 
2111 	/*
2112 	 * For anything larger than that, we'll go ahead and allocate the
2113 	 * maximum number of pages we expect to see. Hopefuly, we won't be
2114 	 * seeing this path in the fast path for high performance devices very
2115 	 * frequently.
2116 	 *
2117 	 * a ddi bind interface that allowed the driver to provide storage to
2118 	 * the bind interface would speed this case up.
2119 	 */
2120 	} else {
2121 		/*
2122 		 * Save away how much memory we allocated. If we're doing a
2123 		 * nosleep, the alloc could fail...
2124 		 */
2125 		dma->dp_cookie_size = sinfo->si_max_pages *
2126 		    sizeof (ddi_dma_cookie_t);
2127 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2128 		if (dma->dp_cookies == NULL) {
2129 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2130 			rootnex_clean_dmahdl(hp);
2131 			return (DDI_DMA_NORESOURCES);
2132 		}
2133 		dma->dp_need_to_free_cookie = B_TRUE;
2134 		ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip,
2135 		    uint_t, sinfo->si_max_pages);
2136 	}
2137 	hp->dmai_cookie = dma->dp_cookies;
2138 
2139 	/*
2140 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2141 	 * looking at the constraints in the dma structure. It will then put
2142 	 * some additional state about the sgl in the dma struct (i.e. is
2143 	 * the sgl clean, or do we need to do some munging; how many pages
2144 	 * need to be copied, etc.)
2145 	 */
2146 	if (dma->dp_dvma_used)
2147 		rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2148 	else
2149 		rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo);
2150 
2151 out:
2152 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2153 	/* if we don't need a copy buffer, we don't need to sync */
2154 	if (sinfo->si_copybuf_req == 0) {
2155 		hp->dmai_rflags |= DMP_NOSYNC;
2156 	}
2157 
2158 	/*
2159 	 * if we don't need the copybuf and we don't need to do a partial,  we
2160 	 * hit the fast path. All the high performance devices should be trying
2161 	 * to hit this path. To hit this path, a device should be able to reach
2162 	 * all of memory, shouldn't try to bind more than it can transfer, and
2163 	 * the buffer shouldn't require more cookies than the driver/device can
2164 	 * handle [sgllen]).
2165 	 *
2166 	 * Note that negative values of dma_attr_sgllen are supposed
2167 	 * to mean unlimited, but we just cast them to mean a
2168 	 * "ridiculous large limit".  This saves some extra checks on
2169 	 * hot paths.
2170 	 */
2171 	if ((sinfo->si_copybuf_req == 0) &&
2172 	    (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) &&
2173 	    (dmao->dmao_size <= dma->dp_maxxfer)) {
2174 fast:
2175 		/*
2176 		 * If the driver supports FMA, insert the handle in the FMA DMA
2177 		 * handle cache.
2178 		 */
2179 		if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2180 			hp->dmai_error.err_cf = rootnex_dma_check;
2181 
2182 		/*
2183 		 * copy out the first cookie and ccountp, set the cookie
2184 		 * pointer to the second cookie. The first cookie is passed
2185 		 * back on the stack. Additional cookies are accessed via
2186 		 * ddi_dma_nextcookie()
2187 		 */
2188 		*cookiep = dma->dp_cookies[0];
2189 		*ccountp = sinfo->si_sgl_size;
2190 		hp->dmai_cookie++;
2191 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2192 		hp->dmai_ncookies = *ccountp;
2193 		hp->dmai_curcookie = 1;
2194 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2195 		ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip,
2196 		    uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS],
2197 		    uint_t, dmao->dmao_size, uint_t, *ccountp);
2198 
2199 
2200 		return (DDI_DMA_MAPPED);
2201 	}
2202 
2203 	/*
2204 	 * go to the slow path, we may need to alloc more memory, create
2205 	 * multiple windows, and munge up a sgl to make the device happy.
2206 	 */
2207 
2208 	/*
2209 	 * With the IOMMU mapobject method used, we should never hit
2210 	 * the slow path. If we do, something is seriously wrong.
2211 	 * Clean up and return an error.
2212 	 */
2213 
2214 #if defined(__amd64) && !defined(__xpv)
2215 
2216 	if (dma->dp_dvma_used) {
2217 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2218 		    &dma->dp_dvma);
2219 		e = DDI_DMA_NOMAPPING;
2220 	} else {
2221 #endif
2222 		e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma,
2223 		    kmflag);
2224 #if defined(__amd64) && !defined(__xpv)
2225 	}
2226 #endif
2227 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2228 		if (dma->dp_need_to_free_cookie) {
2229 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2230 		}
2231 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2232 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
2233 		return (e);
2234 	}
2235 
2236 	/*
2237 	 * If the driver supports FMA, insert the handle in the FMA DMA handle
2238 	 * cache.
2239 	 */
2240 	if (attr->dma_attr_flags & DDI_DMA_FLAGERR)
2241 		hp->dmai_error.err_cf = rootnex_dma_check;
2242 
2243 	/* if the first window uses the copy buffer, sync it for the device */
2244 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2245 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
2246 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2247 		    DDI_DMA_SYNC_FORDEV);
2248 	}
2249 
2250 	/*
2251 	 * copy out the first cookie and ccountp, set the cookie pointer to the
2252 	 * second cookie. Make sure the partial flag is set/cleared correctly.
2253 	 * If we have a partial map (i.e. multiple windows), the number of
2254 	 * cookies we return is the number of cookies in the first window.
2255 	 */
2256 	if (e == DDI_DMA_MAPPED) {
2257 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2258 		*ccountp = sinfo->si_sgl_size;
2259 		hp->dmai_nwin = 1;
2260 	} else {
2261 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
2262 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2263 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2264 	}
2265 	*cookiep = dma->dp_cookies[0];
2266 	hp->dmai_cookie++;
2267 	hp->dmai_ncookies = *ccountp;
2268 	hp->dmai_curcookie = 1;
2269 
2270 	ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2271 	ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2272 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2273 	    dmao->dmao_size, uint_t, *ccountp);
2274 	return (e);
2275 }
2276 
2277 /*
2278  * rootnex_dma_bindhdl()
2279  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
2280  */
2281 static int
2282 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
2283     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
2284     ddi_dma_cookie_t *cookiep, uint_t *ccountp)
2285 {
2286 	int ret;
2287 #if defined(__amd64) && !defined(__xpv)
2288 	if (IOMMU_USED(rdip))
2289 		ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq,
2290 		    cookiep, ccountp);
2291 	else
2292 #endif
2293 	ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq,
2294 	    cookiep, ccountp);
2295 
2296 	if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
2297 		ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
2298 		    &rootnex_state->r_dvma_call_list_id);
2299 	}
2300 
2301 	return (ret);
2302 }
2303 
2304 
2305 
2306 /*ARGSUSED*/
2307 static int
2308 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2309     ddi_dma_handle_t handle)
2310 {
2311 	ddi_dma_impl_t *hp;
2312 	rootnex_dma_t *dma;
2313 	int e;
2314 
2315 	hp = (ddi_dma_impl_t *)handle;
2316 	dma = (rootnex_dma_t *)hp->dmai_private;
2317 
2318 	/* make sure the buffer wasn't free'd before calling unbind */
2319 	if (rootnex_unbind_verify_buffer) {
2320 		e = rootnex_verify_buffer(dma);
2321 		if (e != DDI_SUCCESS) {
2322 			ASSERT(0);
2323 			return (DDI_FAILURE);
2324 		}
2325 	}
2326 
2327 	/* sync the current window before unbinding the buffer */
2328 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2329 	    (hp->dmai_rflags & DDI_DMA_READ)) {
2330 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
2331 		    DDI_DMA_SYNC_FORCPU);
2332 	}
2333 
2334 	/*
2335 	 * cleanup and copy buffer or window state. if we didn't use the copy
2336 	 * buffer or windows, there won't be much to do :-)
2337 	 */
2338 	rootnex_teardown_copybuf(dma);
2339 	rootnex_teardown_windows(dma);
2340 
2341 #if defined(__amd64) && !defined(__xpv)
2342 	if (IOMMU_USED(rdip) && dma->dp_dvma_used)
2343 		(void) iommulib_nexdma_unmapobject(dip, rdip, handle,
2344 		    &dma->dp_dvma);
2345 #endif
2346 
2347 	/*
2348 	 * If we had to allocate space to for the worse case sgl (it didn't
2349 	 * fit into our pre-allocate buffer), free that up now
2350 	 */
2351 	if (dma->dp_need_to_free_cookie) {
2352 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2353 	}
2354 
2355 	/*
2356 	 * clean up the handle so it's ready for the next bind (i.e. if the
2357 	 * handle is reused).
2358 	 */
2359 	rootnex_clean_dmahdl(hp);
2360 	hp->dmai_error.err_cf = NULL;
2361 
2362 	ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2363 	ROOTNEX_DPROBE1(rootnex__unbind, uint64_t,
2364 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2365 
2366 	return (DDI_SUCCESS);
2367 }
2368 
2369 /*
2370  * rootnex_dma_unbindhdl()
2371  *    called from ddi_dma_unbind_handle()
2372  */
2373 /*ARGSUSED*/
2374 static int
2375 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2376     ddi_dma_handle_t handle)
2377 {
2378 	int ret;
2379 
2380 #if defined(__amd64) && !defined(__xpv)
2381 	if (IOMMU_USED(rdip))
2382 		ret = iommulib_nexdma_unbindhdl(dip, rdip, handle);
2383 	else
2384 #endif
2385 	ret = rootnex_coredma_unbindhdl(dip, rdip, handle);
2386 
2387 	if (rootnex_state->r_dvma_call_list_id)
2388 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2389 
2390 	return (ret);
2391 }
2392 
2393 #if defined(__amd64) && !defined(__xpv)
2394 
2395 static int
2396 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle)
2397 {
2398 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2399 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2400 
2401 	if (dma->dp_sleep_flags != KM_SLEEP &&
2402 	    dma->dp_sleep_flags != KM_NOSLEEP)
2403 		cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle");
2404 	return (dma->dp_sleep_flags);
2405 }
2406 /*ARGSUSED*/
2407 static void
2408 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2409 {
2410 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2411 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2412 	rootnex_window_t *window;
2413 
2414 	if (dma->dp_window) {
2415 		window = &dma->dp_window[dma->dp_current_win];
2416 		hp->dmai_cookie = window->wd_first_cookie;
2417 	} else {
2418 		hp->dmai_cookie = dma->dp_cookies;
2419 	}
2420 	hp->dmai_cookie++;
2421 	hp->dmai_curcookie = 1;
2422 }
2423 
2424 /*ARGSUSED*/
2425 static int
2426 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2427     ddi_dma_cookie_t **cookiepp, uint_t *ccountp)
2428 {
2429 	int i;
2430 	int km_flags;
2431 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2432 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2433 	rootnex_window_t *window;
2434 	ddi_dma_cookie_t *cp;
2435 	ddi_dma_cookie_t *cookie;
2436 
2437 	ASSERT(*cookiepp == NULL);
2438 	ASSERT(*ccountp == 0);
2439 
2440 	if (dma->dp_window) {
2441 		window = &dma->dp_window[dma->dp_current_win];
2442 		cp = window->wd_first_cookie;
2443 		*ccountp = window->wd_cookie_cnt;
2444 	} else {
2445 		cp = dma->dp_cookies;
2446 		*ccountp = dma->dp_sglinfo.si_sgl_size;
2447 	}
2448 
2449 	km_flags = rootnex_coredma_get_sleep_flags(handle);
2450 	cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags);
2451 	if (cookie == NULL) {
2452 		return (DDI_DMA_NORESOURCES);
2453 	}
2454 
2455 	for (i = 0; i < *ccountp; i++) {
2456 		cookie[i].dmac_notused = cp[i].dmac_notused;
2457 		cookie[i].dmac_type = cp[i].dmac_type;
2458 		cookie[i].dmac_address = cp[i].dmac_address;
2459 		cookie[i].dmac_size = cp[i].dmac_size;
2460 	}
2461 
2462 	*cookiepp = cookie;
2463 
2464 	return (DDI_SUCCESS);
2465 }
2466 
2467 /*ARGSUSED*/
2468 static int
2469 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle,
2470     ddi_dma_cookie_t *cookiep, uint_t ccount)
2471 {
2472 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2473 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2474 	rootnex_window_t *window;
2475 	ddi_dma_cookie_t *cur_cookiep;
2476 
2477 	ASSERT(cookiep);
2478 	ASSERT(ccount != 0);
2479 	ASSERT(dma->dp_need_to_switch_cookies == B_FALSE);
2480 
2481 	if (dma->dp_window) {
2482 		window = &dma->dp_window[dma->dp_current_win];
2483 		dma->dp_saved_cookies = window->wd_first_cookie;
2484 		window->wd_first_cookie = cookiep;
2485 		ASSERT(ccount == window->wd_cookie_cnt);
2486 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2487 		    + window->wd_first_cookie;
2488 	} else {
2489 		dma->dp_saved_cookies = dma->dp_cookies;
2490 		dma->dp_cookies = cookiep;
2491 		ASSERT(ccount == dma->dp_sglinfo.si_sgl_size);
2492 		cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies)
2493 		    + dma->dp_cookies;
2494 	}
2495 
2496 	dma->dp_need_to_switch_cookies = B_TRUE;
2497 	hp->dmai_cookie = cur_cookiep;
2498 
2499 	return (DDI_SUCCESS);
2500 }
2501 
2502 /*ARGSUSED*/
2503 static int
2504 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle)
2505 {
2506 	ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle;
2507 	rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private;
2508 	rootnex_window_t *window;
2509 	ddi_dma_cookie_t *cur_cookiep;
2510 	ddi_dma_cookie_t *cookie_array;
2511 	uint_t ccount;
2512 
2513 	/* check if cookies have not been switched */
2514 	if (dma->dp_need_to_switch_cookies == B_FALSE)
2515 		return (DDI_SUCCESS);
2516 
2517 	ASSERT(dma->dp_saved_cookies);
2518 
2519 	if (dma->dp_window) {
2520 		window = &dma->dp_window[dma->dp_current_win];
2521 		cookie_array = window->wd_first_cookie;
2522 		window->wd_first_cookie = dma->dp_saved_cookies;
2523 		dma->dp_saved_cookies = NULL;
2524 		ccount = window->wd_cookie_cnt;
2525 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2526 		    + window->wd_first_cookie;
2527 	} else {
2528 		cookie_array = dma->dp_cookies;
2529 		dma->dp_cookies = dma->dp_saved_cookies;
2530 		dma->dp_saved_cookies = NULL;
2531 		ccount = dma->dp_sglinfo.si_sgl_size;
2532 		cur_cookiep = (hp->dmai_cookie - cookie_array)
2533 		    + dma->dp_cookies;
2534 	}
2535 
2536 	kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount);
2537 
2538 	hp->dmai_cookie = cur_cookiep;
2539 
2540 	dma->dp_need_to_switch_cookies = B_FALSE;
2541 
2542 	return (DDI_SUCCESS);
2543 }
2544 
2545 #endif
2546 
2547 static struct as *
2548 rootnex_get_as(ddi_dma_obj_t *dmao)
2549 {
2550 	struct as *asp;
2551 
2552 	switch (dmao->dmao_type) {
2553 	case DMA_OTYP_VADDR:
2554 	case DMA_OTYP_BUFVADDR:
2555 		asp = dmao->dmao_obj.virt_obj.v_as;
2556 		if (asp == NULL)
2557 			asp = &kas;
2558 		break;
2559 	default:
2560 		asp = NULL;
2561 		break;
2562 	}
2563 	return (asp);
2564 }
2565 
2566 /*
2567  * rootnex_verify_buffer()
2568  *   verify buffer wasn't free'd
2569  */
2570 static int
2571 rootnex_verify_buffer(rootnex_dma_t *dma)
2572 {
2573 	page_t **pplist;
2574 	caddr_t vaddr;
2575 	uint_t pcnt;
2576 	uint_t poff;
2577 	page_t *pp;
2578 	char b;
2579 	int i;
2580 
2581 	/* Figure out how many pages this buffer occupies */
2582 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2583 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2584 	} else {
2585 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2586 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2587 	}
2588 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2589 
2590 	switch (dma->dp_dma.dmao_type) {
2591 	case DMA_OTYP_PAGES:
2592 		/*
2593 		 * for a linked list of pp's walk through them to make sure
2594 		 * they're locked and not free.
2595 		 */
2596 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2597 		for (i = 0; i < pcnt; i++) {
2598 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2599 				return (DDI_FAILURE);
2600 			}
2601 			pp = pp->p_next;
2602 		}
2603 		break;
2604 
2605 	case DMA_OTYP_VADDR:
2606 	case DMA_OTYP_BUFVADDR:
2607 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2608 		/*
2609 		 * for an array of pp's walk through them to make sure they're
2610 		 * not free. It's possible that they may not be locked.
2611 		 */
2612 		if (pplist) {
2613 			for (i = 0; i < pcnt; i++) {
2614 				if (PP_ISFREE(pplist[i])) {
2615 					return (DDI_FAILURE);
2616 				}
2617 			}
2618 
2619 		/* For a virtual address, try to peek at each page */
2620 		} else {
2621 			if (rootnex_get_as(&dma->dp_dma) == &kas) {
2622 				for (i = 0; i < pcnt; i++) {
2623 					if (ddi_peek8(NULL, vaddr, &b) ==
2624 					    DDI_FAILURE)
2625 						return (DDI_FAILURE);
2626 					vaddr += MMU_PAGESIZE;
2627 				}
2628 			}
2629 		}
2630 		break;
2631 
2632 	default:
2633 		cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object");
2634 		break;
2635 	}
2636 
2637 	return (DDI_SUCCESS);
2638 }
2639 
2640 
2641 /*
2642  * rootnex_clean_dmahdl()
2643  *    Clean the dma handle. This should be called on a handle alloc and an
2644  *    unbind handle. Set the handle state to the default settings.
2645  */
2646 static void
2647 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2648 {
2649 	rootnex_dma_t *dma;
2650 
2651 
2652 	dma = (rootnex_dma_t *)hp->dmai_private;
2653 
2654 	hp->dmai_nwin = 0;
2655 	dma->dp_current_cookie = 0;
2656 	dma->dp_copybuf_size = 0;
2657 	dma->dp_window = NULL;
2658 	dma->dp_cbaddr = NULL;
2659 	dma->dp_inuse = B_FALSE;
2660 	dma->dp_dvma_used = B_FALSE;
2661 	dma->dp_need_to_free_cookie = B_FALSE;
2662 	dma->dp_need_to_switch_cookies = B_FALSE;
2663 	dma->dp_saved_cookies = NULL;
2664 	dma->dp_sleep_flags = KM_PANIC;
2665 	dma->dp_need_to_free_window = B_FALSE;
2666 	dma->dp_partial_required = B_FALSE;
2667 	dma->dp_trim_required = B_FALSE;
2668 	dma->dp_sglinfo.si_copybuf_req = 0;
2669 #if !defined(__amd64)
2670 	dma->dp_cb_remaping = B_FALSE;
2671 	dma->dp_kva = NULL;
2672 #endif
2673 
2674 	/* FMA related initialization */
2675 	hp->dmai_fault = 0;
2676 	hp->dmai_fault_check = NULL;
2677 	hp->dmai_fault_notify = NULL;
2678 	hp->dmai_error.err_ena = 0;
2679 	hp->dmai_error.err_status = DDI_FM_OK;
2680 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2681 	hp->dmai_error.err_ontrap = NULL;
2682 
2683 	/* Cookie tracking */
2684 	hp->dmai_ncookies = 0;
2685 	hp->dmai_curcookie = 0;
2686 }
2687 
2688 
2689 /*
2690  * rootnex_valid_alloc_parms()
2691  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2692  */
2693 static int
2694 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2695 {
2696 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2697 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2698 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2699 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2700 		return (DDI_DMA_BADATTR);
2701 	}
2702 
2703 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2704 		return (DDI_DMA_BADATTR);
2705 	}
2706 
2707 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2708 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2709 	    attr->dma_attr_sgllen == 0) {
2710 		return (DDI_DMA_BADATTR);
2711 	}
2712 
2713 	/* We should be able to DMA into every byte offset in a page */
2714 	if (maxsegmentsize < MMU_PAGESIZE) {
2715 		return (DDI_DMA_BADATTR);
2716 	}
2717 
2718 	/* if we're bouncing on seg, seg must be <= addr_hi */
2719 	if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) &&
2720 	    (attr->dma_attr_seg > attr->dma_attr_addr_hi)) {
2721 		return (DDI_DMA_BADATTR);
2722 	}
2723 	return (DDI_SUCCESS);
2724 }
2725 
2726 /*
2727  * rootnex_valid_bind_parms()
2728  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2729  */
2730 /* ARGSUSED */
2731 static int
2732 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2733 {
2734 #if !defined(__amd64)
2735 	/*
2736 	 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2737 	 * we can track the offset for the obsoleted interfaces.
2738 	 */
2739 	if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2740 		return (DDI_DMA_TOOBIG);
2741 	}
2742 #endif
2743 
2744 	return (DDI_SUCCESS);
2745 }
2746 
2747 
2748 /*
2749  * rootnex_need_bounce_seg()
2750  *    check to see if the buffer lives on both side of the seg.
2751  */
2752 static boolean_t
2753 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo)
2754 {
2755 	ddi_dma_atyp_t buftype;
2756 	rootnex_addr_t raddr;
2757 	boolean_t lower_addr;
2758 	boolean_t upper_addr;
2759 	uint64_t offset;
2760 	page_t **pplist;
2761 	uint64_t paddr;
2762 	uint32_t psize;
2763 	uint32_t size;
2764 	caddr_t vaddr;
2765 	uint_t pcnt;
2766 	page_t *pp;
2767 
2768 	pp = NULL;
2769 	/* shortcuts */
2770 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2771 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2772 	buftype = dmar_object->dmao_type;
2773 	size = dmar_object->dmao_size;
2774 
2775 	lower_addr = B_FALSE;
2776 	upper_addr = B_FALSE;
2777 	pcnt = 0;
2778 
2779 	/*
2780 	 * Process the first page to handle the initial offset of the buffer.
2781 	 * We'll use the base address we get later when we loop through all
2782 	 * the pages.
2783 	 */
2784 	if (buftype == DMA_OTYP_PAGES) {
2785 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2786 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2787 		    MMU_PAGEOFFSET;
2788 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2789 		psize = MIN(size, (MMU_PAGESIZE - offset));
2790 		pp = pp->p_next;
2791 		sglinfo->si_asp = NULL;
2792 	} else if (pplist != NULL) {
2793 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2794 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2795 		if (sglinfo->si_asp == NULL) {
2796 			sglinfo->si_asp = &kas;
2797 		}
2798 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2799 		paddr += offset;
2800 		psize = MIN(size, (MMU_PAGESIZE - offset));
2801 		pcnt++;
2802 	} else {
2803 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2804 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2805 		if (sglinfo->si_asp == NULL) {
2806 			sglinfo->si_asp = &kas;
2807 		}
2808 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2809 		paddr += offset;
2810 		psize = MIN(size, (MMU_PAGESIZE - offset));
2811 		vaddr += psize;
2812 	}
2813 
2814 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2815 
2816 	if ((raddr + psize) > sglinfo->si_segmask) {
2817 		upper_addr = B_TRUE;
2818 	} else {
2819 		lower_addr = B_TRUE;
2820 	}
2821 	size -= psize;
2822 
2823 	/*
2824 	 * Walk through the rest of the pages in the buffer. Track to see
2825 	 * if we have pages on both sides of the segment boundary.
2826 	 */
2827 	while (size > 0) {
2828 		/* partial or full page */
2829 		psize = MIN(size, MMU_PAGESIZE);
2830 
2831 		if (buftype == DMA_OTYP_PAGES) {
2832 			/* get the paddr from the page_t */
2833 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2834 			paddr = pfn_to_pa(pp->p_pagenum);
2835 			pp = pp->p_next;
2836 		} else if (pplist != NULL) {
2837 			/* index into the array of page_t's to get the paddr */
2838 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2839 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2840 			pcnt++;
2841 		} else {
2842 			/* call into the VM to get the paddr */
2843 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
2844 			    vaddr));
2845 			vaddr += psize;
2846 		}
2847 
2848 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2849 
2850 		if ((raddr + psize) > sglinfo->si_segmask) {
2851 			upper_addr = B_TRUE;
2852 		} else {
2853 			lower_addr = B_TRUE;
2854 		}
2855 		/*
2856 		 * if the buffer lives both above and below the segment
2857 		 * boundary, or the current page is the page immediately
2858 		 * after the segment, we will use a copy/bounce buffer for
2859 		 * all pages > seg.
2860 		 */
2861 		if ((lower_addr && upper_addr) ||
2862 		    (raddr == (sglinfo->si_segmask + 1))) {
2863 			return (B_TRUE);
2864 		}
2865 
2866 		size -= psize;
2867 	}
2868 
2869 	return (B_FALSE);
2870 }
2871 
2872 /*
2873  * rootnex_get_sgl()
2874  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2875  *    with a call to the vm layer when vm2.0 comes around...
2876  */
2877 static void
2878 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2879     rootnex_sglinfo_t *sglinfo)
2880 {
2881 	ddi_dma_atyp_t buftype;
2882 	rootnex_addr_t raddr;
2883 	uint64_t last_page;
2884 	uint64_t offset;
2885 	uint64_t addrhi;
2886 	uint64_t addrlo;
2887 	uint64_t maxseg;
2888 	page_t **pplist;
2889 	uint64_t paddr;
2890 	uint32_t psize;
2891 	uint32_t size;
2892 	caddr_t vaddr;
2893 	uint_t pcnt;
2894 	page_t *pp;
2895 	uint_t cnt;
2896 
2897 	pp = NULL;
2898 	/* shortcuts */
2899 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2900 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2901 	maxseg = sglinfo->si_max_cookie_size;
2902 	buftype = dmar_object->dmao_type;
2903 	addrhi = sglinfo->si_max_addr;
2904 	addrlo = sglinfo->si_min_addr;
2905 	size = dmar_object->dmao_size;
2906 
2907 	pcnt = 0;
2908 	cnt = 0;
2909 
2910 
2911 	/*
2912 	 * check to see if we need to use the copy buffer for pages over
2913 	 * the segment attr.
2914 	 */
2915 	sglinfo->si_bounce_on_seg = B_FALSE;
2916 	if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) {
2917 		sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg(
2918 		    dmar_object, sglinfo);
2919 	}
2920 
2921 	/*
2922 	 * if we were passed down a linked list of pages, i.e. pointer to
2923 	 * page_t, use this to get our physical address and buf offset.
2924 	 */
2925 	if (buftype == DMA_OTYP_PAGES) {
2926 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2927 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2928 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2929 		    MMU_PAGEOFFSET;
2930 		paddr = pfn_to_pa(pp->p_pagenum) + offset;
2931 		psize = MIN(size, (MMU_PAGESIZE - offset));
2932 		pp = pp->p_next;
2933 		sglinfo->si_asp = NULL;
2934 
2935 	/*
2936 	 * We weren't passed down a linked list of pages, but if we were passed
2937 	 * down an array of pages, use this to get our physical address and buf
2938 	 * offset.
2939 	 */
2940 	} else if (pplist != NULL) {
2941 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2942 		    (buftype == DMA_OTYP_BUFVADDR));
2943 
2944 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2945 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2946 		if (sglinfo->si_asp == NULL) {
2947 			sglinfo->si_asp = &kas;
2948 		}
2949 
2950 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2951 		paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
2952 		paddr += offset;
2953 		psize = MIN(size, (MMU_PAGESIZE - offset));
2954 		pcnt++;
2955 
2956 	/*
2957 	 * All we have is a virtual address, we'll need to call into the VM
2958 	 * to get the physical address.
2959 	 */
2960 	} else {
2961 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2962 		    (buftype == DMA_OTYP_BUFVADDR));
2963 
2964 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2965 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2966 		if (sglinfo->si_asp == NULL) {
2967 			sglinfo->si_asp = &kas;
2968 		}
2969 
2970 		paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2971 		paddr += offset;
2972 		psize = MIN(size, (MMU_PAGESIZE - offset));
2973 		vaddr += psize;
2974 	}
2975 
2976 	raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
2977 
2978 	/*
2979 	 * Setup the first cookie with the physical address of the page and the
2980 	 * size of the page (which takes into account the initial offset into
2981 	 * the page.
2982 	 */
2983 	sgl[cnt].dmac_laddress = raddr;
2984 	sgl[cnt].dmac_size = psize;
2985 	sgl[cnt].dmac_type = 0;
2986 
2987 	/*
2988 	 * Save away the buffer offset into the page. We'll need this later in
2989 	 * the copy buffer code to help figure out the page index within the
2990 	 * buffer and the offset into the current page.
2991 	 */
2992 	sglinfo->si_buf_offset = offset;
2993 
2994 	/*
2995 	 * If we are using the copy buffer for anything over the segment
2996 	 * boundary, and this page is over the segment boundary.
2997 	 *   OR
2998 	 * if the DMA engine can't reach the physical address.
2999 	 */
3000 	if (((sglinfo->si_bounce_on_seg) &&
3001 	    ((raddr + psize) > sglinfo->si_segmask)) ||
3002 	    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3003 		/*
3004 		 * Increase how much copy buffer we use. We always increase by
3005 		 * pagesize so we don't have to worry about converting offsets.
3006 		 * Set a flag in the cookies dmac_type to indicate that it uses
3007 		 * the copy buffer. If this isn't the last cookie, go to the
3008 		 * next cookie (since we separate each page which uses the copy
3009 		 * buffer in case the copy buffer is not physically contiguous.
3010 		 */
3011 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
3012 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3013 		if ((cnt + 1) < sglinfo->si_max_pages) {
3014 			cnt++;
3015 			sgl[cnt].dmac_laddress = 0;
3016 			sgl[cnt].dmac_size = 0;
3017 			sgl[cnt].dmac_type = 0;
3018 		}
3019 	}
3020 
3021 	/*
3022 	 * save this page's physical address so we can figure out if the next
3023 	 * page is physically contiguous. Keep decrementing size until we are
3024 	 * done with the buffer.
3025 	 */
3026 	last_page = raddr & MMU_PAGEMASK;
3027 	size -= psize;
3028 
3029 	while (size > 0) {
3030 		/* Get the size for this page (i.e. partial or full page) */
3031 		psize = MIN(size, MMU_PAGESIZE);
3032 
3033 		if (buftype == DMA_OTYP_PAGES) {
3034 			/* get the paddr from the page_t */
3035 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
3036 			paddr = pfn_to_pa(pp->p_pagenum);
3037 			pp = pp->p_next;
3038 		} else if (pplist != NULL) {
3039 			/* index into the array of page_t's to get the paddr */
3040 			ASSERT(!PP_ISFREE(pplist[pcnt]));
3041 			paddr = pfn_to_pa(pplist[pcnt]->p_pagenum);
3042 			pcnt++;
3043 		} else {
3044 			/* call into the VM to get the paddr */
3045 			paddr =  pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat,
3046 			    vaddr));
3047 			vaddr += psize;
3048 		}
3049 
3050 		raddr = ROOTNEX_PADDR_TO_RBASE(paddr);
3051 
3052 		/*
3053 		 * If we are using the copy buffer for anything over the
3054 		 * segment boundary, and this page is over the segment
3055 		 * boundary.
3056 		 *   OR
3057 		 * if the DMA engine can't reach the physical address.
3058 		 */
3059 		if (((sglinfo->si_bounce_on_seg) &&
3060 		    ((raddr + psize) > sglinfo->si_segmask)) ||
3061 		    ((raddr < addrlo) || ((raddr + psize) > addrhi))) {
3062 
3063 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
3064 
3065 			/*
3066 			 * if there is something in the current cookie, go to
3067 			 * the next one. We only want one page in a cookie which
3068 			 * uses the copybuf since the copybuf doesn't have to
3069 			 * be physically contiguous.
3070 			 */
3071 			if (sgl[cnt].dmac_size != 0) {
3072 				cnt++;
3073 			}
3074 			sgl[cnt].dmac_laddress = raddr;
3075 			sgl[cnt].dmac_size = psize;
3076 #if defined(__amd64)
3077 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
3078 #else
3079 			/*
3080 			 * save the buf offset for 32-bit kernel. used in the
3081 			 * obsoleted interfaces.
3082 			 */
3083 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
3084 			    (dmar_object->dmao_size - size);
3085 #endif
3086 			/* if this isn't the last cookie, go to the next one */
3087 			if ((cnt + 1) < sglinfo->si_max_pages) {
3088 				cnt++;
3089 				sgl[cnt].dmac_laddress = 0;
3090 				sgl[cnt].dmac_size = 0;
3091 				sgl[cnt].dmac_type = 0;
3092 			}
3093 
3094 		/*
3095 		 * this page didn't need the copy buffer, if it's not physically
3096 		 * contiguous, or it would put us over a segment boundary, or it
3097 		 * puts us over the max cookie size, or the current sgl doesn't
3098 		 * have anything in it.
3099 		 */
3100 		} else if (((last_page + MMU_PAGESIZE) != raddr) ||
3101 		    !(raddr & sglinfo->si_segmask) ||
3102 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3103 		    (sgl[cnt].dmac_size == 0)) {
3104 			/*
3105 			 * if we're not already in a new cookie, go to the next
3106 			 * cookie.
3107 			 */
3108 			if (sgl[cnt].dmac_size != 0) {
3109 				cnt++;
3110 			}
3111 
3112 			/* save the cookie information */
3113 			sgl[cnt].dmac_laddress = raddr;
3114 			sgl[cnt].dmac_size = psize;
3115 #if defined(__amd64)
3116 			sgl[cnt].dmac_type = 0;
3117 #else
3118 			/*
3119 			 * save the buf offset for 32-bit kernel. used in the
3120 			 * obsoleted interfaces.
3121 			 */
3122 			sgl[cnt].dmac_type = dmar_object->dmao_size - size;
3123 #endif
3124 
3125 		/*
3126 		 * this page didn't need the copy buffer, it is physically
3127 		 * contiguous with the last page, and it's <= the max cookie
3128 		 * size.
3129 		 */
3130 		} else {
3131 			sgl[cnt].dmac_size += psize;
3132 
3133 			/*
3134 			 * If this cookie is used up, and more cookies
3135 			 * are available, then move onto the next one.
3136 			 */
3137 			if ((sgl[cnt].dmac_size == maxseg) &&
3138 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3139 				cnt++;
3140 				sgl[cnt].dmac_laddress = 0;
3141 				sgl[cnt].dmac_size = 0;
3142 				sgl[cnt].dmac_type = 0;
3143 			}
3144 		}
3145 
3146 		/*
3147 		 * save this page's physical address so we can figure out if the
3148 		 * next page is physically contiguous. Keep decrementing size
3149 		 * until we are done with the buffer.
3150 		 */
3151 		last_page = raddr;
3152 		size -= psize;
3153 	}
3154 
3155 	/* we're done, save away how many cookies the sgl has */
3156 	if (sgl[cnt].dmac_size == 0) {
3157 		ASSERT(cnt < sglinfo->si_max_pages);
3158 		sglinfo->si_sgl_size = cnt;
3159 	} else {
3160 		sglinfo->si_sgl_size = cnt + 1;
3161 	}
3162 }
3163 
3164 static void
3165 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
3166     rootnex_sglinfo_t *sglinfo)
3167 {
3168 	uint64_t offset;
3169 	uint64_t maxseg;
3170 	uint64_t dvaddr;
3171 	struct dvmaseg *dvs;
3172 	uint64_t paddr;
3173 	uint32_t psize, ssize;
3174 	uint32_t size;
3175 	uint_t cnt;
3176 	int physcontig;
3177 
3178 	ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR);
3179 
3180 	/* shortcuts */
3181 	maxseg = sglinfo->si_max_cookie_size;
3182 	size = dmar_object->dmao_size;
3183 
3184 	cnt = 0;
3185 	sglinfo->si_bounce_on_seg = B_FALSE;
3186 
3187 	dvs = dmar_object->dmao_obj.dvma_obj.dv_seg;
3188 	offset = dmar_object->dmao_obj.dvma_obj.dv_off;
3189 	ssize = dvs->dvs_len;
3190 	paddr = dvs->dvs_start;
3191 	paddr += offset;
3192 	psize = MIN(ssize, (maxseg - offset));
3193 	dvaddr = paddr + psize;
3194 	ssize -= psize;
3195 
3196 	sgl[cnt].dmac_laddress = paddr;
3197 	sgl[cnt].dmac_size = psize;
3198 	sgl[cnt].dmac_type = 0;
3199 
3200 	size -= psize;
3201 	while (size > 0) {
3202 		if (ssize == 0) {
3203 			dvs++;
3204 			ssize = dvs->dvs_len;
3205 			dvaddr = dvs->dvs_start;
3206 			physcontig = 0;
3207 		} else
3208 			physcontig = 1;
3209 
3210 		paddr = dvaddr;
3211 		psize = MIN(ssize, maxseg);
3212 		dvaddr += psize;
3213 		ssize -= psize;
3214 
3215 		if (!physcontig || !(paddr & sglinfo->si_segmask) ||
3216 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
3217 		    (sgl[cnt].dmac_size == 0)) {
3218 			/*
3219 			 * if we're not already in a new cookie, go to the next
3220 			 * cookie.
3221 			 */
3222 			if (sgl[cnt].dmac_size != 0) {
3223 				cnt++;
3224 			}
3225 
3226 			/* save the cookie information */
3227 			sgl[cnt].dmac_laddress = paddr;
3228 			sgl[cnt].dmac_size = psize;
3229 			sgl[cnt].dmac_type = 0;
3230 		} else {
3231 			sgl[cnt].dmac_size += psize;
3232 
3233 			/*
3234 			 * If this cookie is used up, and more cookies
3235 			 * are available, then move onto the next one.
3236 			 */
3237 			if ((sgl[cnt].dmac_size == maxseg) &&
3238 			    ((cnt + 1) < sglinfo->si_max_pages)) {
3239 				cnt++;
3240 				sgl[cnt].dmac_laddress = 0;
3241 				sgl[cnt].dmac_size = 0;
3242 				sgl[cnt].dmac_type = 0;
3243 			}
3244 		}
3245 		size -= psize;
3246 	}
3247 
3248 	/* we're done, save away how many cookies the sgl has */
3249 	if (sgl[cnt].dmac_size == 0) {
3250 		sglinfo->si_sgl_size = cnt;
3251 	} else {
3252 		sglinfo->si_sgl_size = cnt + 1;
3253 	}
3254 }
3255 
3256 /*
3257  * rootnex_bind_slowpath()
3258  *    Call in the bind path if the calling driver can't use the sgl without
3259  *    modifying it. We either need to use the copy buffer and/or we will end up
3260  *    with a partial bind.
3261  */
3262 static int
3263 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3264     rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3265 {
3266 	rootnex_sglinfo_t *sinfo;
3267 	rootnex_window_t *window;
3268 	ddi_dma_cookie_t *cookie;
3269 	size_t copybuf_used;
3270 	size_t dmac_size;
3271 	boolean_t partial;
3272 	off_t cur_offset;
3273 	page_t *cur_pp;
3274 	major_t mnum;
3275 	int e;
3276 	int i;
3277 
3278 
3279 	sinfo = &dma->dp_sglinfo;
3280 	copybuf_used = 0;
3281 	partial = B_FALSE;
3282 
3283 	/*
3284 	 * If we're using the copybuf, set the copybuf state in dma struct.
3285 	 * Needs to be first since it sets the copy buffer size.
3286 	 */
3287 	if (sinfo->si_copybuf_req != 0) {
3288 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
3289 		if (e != DDI_SUCCESS) {
3290 			return (e);
3291 		}
3292 	} else {
3293 		dma->dp_copybuf_size = 0;
3294 	}
3295 
3296 	/*
3297 	 * Figure out if we need to do a partial mapping. If so, figure out
3298 	 * if we need to trim the buffers when we munge the sgl.
3299 	 */
3300 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
3301 	    (dmao->dmao_size > dma->dp_maxxfer) ||
3302 	    ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
3303 		dma->dp_partial_required = B_TRUE;
3304 		if (attr->dma_attr_granular != 1) {
3305 			dma->dp_trim_required = B_TRUE;
3306 		}
3307 	} else {
3308 		dma->dp_partial_required = B_FALSE;
3309 		dma->dp_trim_required = B_FALSE;
3310 	}
3311 
3312 	/* If we need to do a partial bind, make sure the driver supports it */
3313 	if (dma->dp_partial_required &&
3314 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
3315 
3316 		mnum = ddi_driver_major(dma->dp_dip);
3317 		/*
3318 		 * patchable which allows us to print one warning per major
3319 		 * number.
3320 		 */
3321 		if ((rootnex_bind_warn) &&
3322 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
3323 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
3324 			cmn_err(CE_WARN, "!%s: coding error detected, the "
3325 			    "driver is using ddi_dma_attr(9S) incorrectly. "
3326 			    "There is a small risk of data corruption in "
3327 			    "particular with large I/Os. The driver should be "
3328 			    "replaced with a corrected version for proper "
3329 			    "system operation. To disable this warning, add "
3330 			    "'set rootnex:rootnex_bind_warn=0' to "
3331 			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
3332 		}
3333 		return (DDI_DMA_TOOBIG);
3334 	}
3335 
3336 	/*
3337 	 * we might need multiple windows, setup state to handle them. In this
3338 	 * code path, we will have at least one window.
3339 	 */
3340 	e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag);
3341 	if (e != DDI_SUCCESS) {
3342 		rootnex_teardown_copybuf(dma);
3343 		return (e);
3344 	}
3345 
3346 	window = &dma->dp_window[0];
3347 	cookie = &dma->dp_cookies[0];
3348 	cur_offset = 0;
3349 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
3350 	if (dmao->dmao_type == DMA_OTYP_PAGES) {
3351 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
3352 	}
3353 
3354 	/* loop though all the cookies we got back from get_sgl() */
3355 	for (i = 0; i < sinfo->si_sgl_size; i++) {
3356 		/*
3357 		 * If we're using the copy buffer, check this cookie and setup
3358 		 * its associated copy buffer state. If this cookie uses the
3359 		 * copy buffer, make sure we sync this window during dma_sync.
3360 		 */
3361 		if (dma->dp_copybuf_size > 0) {
3362 			rootnex_setup_cookie(dmao, dma, cookie,
3363 			    cur_offset, &copybuf_used, &cur_pp);
3364 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3365 				window->wd_dosync = B_TRUE;
3366 			}
3367 		}
3368 
3369 		/*
3370 		 * save away the cookie size, since it could be modified in
3371 		 * the windowing code.
3372 		 */
3373 		dmac_size = cookie->dmac_size;
3374 
3375 		/* if we went over max copybuf size */
3376 		if (dma->dp_copybuf_size &&
3377 		    (copybuf_used > dma->dp_copybuf_size)) {
3378 			partial = B_TRUE;
3379 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
3380 			    cookie, cur_offset, &copybuf_used);
3381 			if (e != DDI_SUCCESS) {
3382 				rootnex_teardown_copybuf(dma);
3383 				rootnex_teardown_windows(dma);
3384 				return (e);
3385 			}
3386 
3387 			/*
3388 			 * if the coookie uses the copy buffer, make sure the
3389 			 * new window we just moved to is set to sync.
3390 			 */
3391 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3392 				window->wd_dosync = B_TRUE;
3393 			}
3394 			ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *,
3395 			    dma->dp_dip);
3396 
3397 		/* if the cookie cnt == max sgllen, move to the next window */
3398 		} else if (window->wd_cookie_cnt >=
3399 		    (unsigned)attr->dma_attr_sgllen) {
3400 			partial = B_TRUE;
3401 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
3402 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
3403 			    cookie, attr, cur_offset);
3404 			if (e != DDI_SUCCESS) {
3405 				rootnex_teardown_copybuf(dma);
3406 				rootnex_teardown_windows(dma);
3407 				return (e);
3408 			}
3409 
3410 			/*
3411 			 * if the coookie uses the copy buffer, make sure the
3412 			 * new window we just moved to is set to sync.
3413 			 */
3414 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3415 				window->wd_dosync = B_TRUE;
3416 			}
3417 			ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *,
3418 			    dma->dp_dip);
3419 
3420 		/* else if we will be over maxxfer */
3421 		} else if ((window->wd_size + dmac_size) >
3422 		    dma->dp_maxxfer) {
3423 			partial = B_TRUE;
3424 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
3425 			    cookie);
3426 			if (e != DDI_SUCCESS) {
3427 				rootnex_teardown_copybuf(dma);
3428 				rootnex_teardown_windows(dma);
3429 				return (e);
3430 			}
3431 
3432 			/*
3433 			 * if the coookie uses the copy buffer, make sure the
3434 			 * new window we just moved to is set to sync.
3435 			 */
3436 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3437 				window->wd_dosync = B_TRUE;
3438 			}
3439 			ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *,
3440 			    dma->dp_dip);
3441 
3442 		/* else this cookie fits in the current window */
3443 		} else {
3444 			window->wd_cookie_cnt++;
3445 			window->wd_size += dmac_size;
3446 		}
3447 
3448 		/* track our offset into the buffer, go to the next cookie */
3449 		ASSERT(dmac_size <= dmao->dmao_size);
3450 		ASSERT(cookie->dmac_size <= dmac_size);
3451 		cur_offset += dmac_size;
3452 		cookie++;
3453 	}
3454 
3455 	/* if we ended up with a zero sized window in the end, clean it up */
3456 	if (window->wd_size == 0) {
3457 		hp->dmai_nwin--;
3458 		window--;
3459 	}
3460 
3461 	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
3462 
3463 	if (!partial) {
3464 		return (DDI_DMA_MAPPED);
3465 	}
3466 
3467 	ASSERT(dma->dp_partial_required);
3468 	return (DDI_DMA_PARTIAL_MAP);
3469 }
3470 
3471 /*
3472  * rootnex_setup_copybuf()
3473  *    Called in bind slowpath. Figures out if we're going to use the copy
3474  *    buffer, and if we do, sets up the basic state to handle it.
3475  */
3476 static int
3477 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
3478     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
3479 {
3480 	rootnex_sglinfo_t *sinfo;
3481 	ddi_dma_attr_t lattr;
3482 	size_t max_copybuf;
3483 	int cansleep;
3484 	int e;
3485 #if !defined(__amd64)
3486 	int vmflag;
3487 #endif
3488 
3489 	ASSERT(!dma->dp_dvma_used);
3490 
3491 	sinfo = &dma->dp_sglinfo;
3492 
3493 	/* read this first so it's consistent through the routine  */
3494 	max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK;
3495 
3496 	/* We need to call into the rootnex on ddi_dma_sync() */
3497 	hp->dmai_rflags &= ~DMP_NOSYNC;
3498 
3499 	/* make sure the copybuf size <= the max size */
3500 	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
3501 	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
3502 
3503 #if !defined(__amd64)
3504 	/*
3505 	 * if we don't have kva space to copy to/from, allocate the KVA space
3506 	 * now. We only do this for the 32-bit kernel. We use seg kpm space for
3507 	 * the 64-bit kernel.
3508 	 */
3509 	if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
3510 	    (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
3511 
3512 		/* convert the sleep flags */
3513 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3514 			vmflag = VM_SLEEP;
3515 		} else {
3516 			vmflag = VM_NOSLEEP;
3517 		}
3518 
3519 		/* allocate Kernel VA space that we can bcopy to/from */
3520 		dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
3521 		    vmflag);
3522 		if (dma->dp_kva == NULL) {
3523 			return (DDI_DMA_NORESOURCES);
3524 		}
3525 	}
3526 #endif
3527 
3528 	/* convert the sleep flags */
3529 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
3530 		cansleep = 1;
3531 	} else {
3532 		cansleep = 0;
3533 	}
3534 
3535 	/*
3536 	 * Allocate the actual copy buffer. This needs to fit within the DMA
3537 	 * engine limits, so we can't use kmem_alloc... We don't need
3538 	 * contiguous memory (sgllen) since we will be forcing windows on
3539 	 * sgllen anyway.
3540 	 */
3541 	lattr = *attr;
3542 	lattr.dma_attr_align = MMU_PAGESIZE;
3543 	lattr.dma_attr_sgllen = -1;	/* no limit */
3544 	/*
3545 	 * if we're using the copy buffer because of seg, use that for our
3546 	 * upper address limit.
3547 	 */
3548 	if (sinfo->si_bounce_on_seg) {
3549 		lattr.dma_attr_addr_hi = lattr.dma_attr_seg;
3550 	}
3551 	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
3552 	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
3553 	if (e != DDI_SUCCESS) {
3554 #if !defined(__amd64)
3555 		if (dma->dp_kva != NULL) {
3556 			vmem_free(heap_arena, dma->dp_kva,
3557 			    dma->dp_copybuf_size);
3558 		}
3559 #endif
3560 		return (DDI_DMA_NORESOURCES);
3561 	}
3562 
3563 	ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
3564 	    size_t, dma->dp_copybuf_size);
3565 
3566 	return (DDI_SUCCESS);
3567 }
3568 
3569 
3570 /*
3571  * rootnex_setup_windows()
3572  *    Called in bind slowpath to setup the window state. We always have windows
3573  *    in the slowpath. Even if the window count = 1.
3574  */
3575 static int
3576 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3577     ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag)
3578 {
3579 	rootnex_window_t *windowp;
3580 	rootnex_sglinfo_t *sinfo;
3581 	size_t copy_state_size;
3582 	size_t win_state_size;
3583 	size_t state_available;
3584 	size_t space_needed;
3585 	uint_t copybuf_win;
3586 	uint_t maxxfer_win;
3587 	size_t space_used;
3588 	uint_t sglwin;
3589 
3590 
3591 	sinfo = &dma->dp_sglinfo;
3592 
3593 	dma->dp_current_win = 0;
3594 	hp->dmai_nwin = 0;
3595 
3596 	/* If we don't need to do a partial, we only have one window */
3597 	if (!dma->dp_partial_required) {
3598 		dma->dp_max_win = 1;
3599 
3600 	/*
3601 	 * we need multiple windows, need to figure out the worse case number
3602 	 * of windows.
3603 	 */
3604 	} else {
3605 		/*
3606 		 * if we need windows because we need more copy buffer that
3607 		 * we allow, the worse case number of windows we could need
3608 		 * here would be (copybuf space required / copybuf space that
3609 		 * we have) plus one for remainder, and plus 2 to handle the
3610 		 * extra pages on the trim for the first and last pages of the
3611 		 * buffer (a page is the minimum window size so under the right
3612 		 * attr settings, you could have a window for each page).
3613 		 * The last page will only be hit here if the size is not a
3614 		 * multiple of the granularity (which theoretically shouldn't
3615 		 * be the case but never has been enforced, so we could have
3616 		 * broken things without it).
3617 		 */
3618 		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
3619 			ASSERT(dma->dp_copybuf_size > 0);
3620 			copybuf_win = (sinfo->si_copybuf_req /
3621 			    dma->dp_copybuf_size) + 1 + 2;
3622 		} else {
3623 			copybuf_win = 0;
3624 		}
3625 
3626 		/*
3627 		 * if we need windows because we have more cookies than the H/W
3628 		 * can handle, the number of windows we would need here would
3629 		 * be (cookie count / cookies count H/W supports minus 1[for
3630 		 * trim]) plus one for remainder.
3631 		 */
3632 		if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) {
3633 			sglwin = (sinfo->si_sgl_size /
3634 			    (attr->dma_attr_sgllen - 1)) + 1;
3635 		} else {
3636 			sglwin = 0;
3637 		}
3638 
3639 		/*
3640 		 * if we need windows because we're binding more memory than the
3641 		 * H/W can transfer at once, the number of windows we would need
3642 		 * here would be (xfer count / max xfer H/W supports) plus one
3643 		 * for remainder, and plus 2 to handle the extra pages on the
3644 		 * trim (see above comment about trim)
3645 		 */
3646 		if (dmao->dmao_size > dma->dp_maxxfer) {
3647 			maxxfer_win = (dmao->dmao_size /
3648 			    dma->dp_maxxfer) + 1 + 2;
3649 		} else {
3650 			maxxfer_win = 0;
3651 		}
3652 		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3653 		ASSERT(dma->dp_max_win > 0);
3654 	}
3655 	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3656 
3657 	/*
3658 	 * Get space for window and potential copy buffer state. Before we
3659 	 * go and allocate memory, see if we can get away with using what's
3660 	 * left in the pre-allocted state or the dynamically allocated sgl.
3661 	 */
3662 	space_used = (uintptr_t)(sinfo->si_sgl_size *
3663 	    sizeof (ddi_dma_cookie_t));
3664 
3665 	/* if we dynamically allocated space for the cookies */
3666 	if (dma->dp_need_to_free_cookie) {
3667 		/* if we have more space in the pre-allocted buffer, use it */
3668 		ASSERT(space_used <= dma->dp_cookie_size);
3669 		if ((dma->dp_cookie_size - space_used) <=
3670 		    rootnex_state->r_prealloc_size) {
3671 			state_available = rootnex_state->r_prealloc_size;
3672 			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3673 
3674 		/*
3675 		 * else, we have more free space in the dynamically allocated
3676 		 * buffer, i.e. the buffer wasn't worse case fragmented so we
3677 		 * didn't need a lot of cookies.
3678 		 */
3679 		} else {
3680 			state_available = dma->dp_cookie_size - space_used;
3681 			windowp = (rootnex_window_t *)
3682 			    &dma->dp_cookies[sinfo->si_sgl_size];
3683 		}
3684 
3685 	/* we used the pre-alloced buffer */
3686 	} else {
3687 		ASSERT(space_used <= rootnex_state->r_prealloc_size);
3688 		state_available = rootnex_state->r_prealloc_size - space_used;
3689 		windowp = (rootnex_window_t *)
3690 		    &dma->dp_cookies[sinfo->si_sgl_size];
3691 	}
3692 
3693 	/*
3694 	 * figure out how much state we need to track the copy buffer. Add an
3695 	 * addition 8 bytes for pointer alignemnt later.
3696 	 */
3697 	if (dma->dp_copybuf_size > 0) {
3698 		copy_state_size = sinfo->si_max_pages *
3699 		    sizeof (rootnex_pgmap_t);
3700 	} else {
3701 		copy_state_size = 0;
3702 	}
3703 	/* add an additional 8 bytes for pointer alignment */
3704 	space_needed = win_state_size + copy_state_size + 0x8;
3705 
3706 	/* if we have enough space already, use it */
3707 	if (state_available >= space_needed) {
3708 		dma->dp_window = windowp;
3709 		dma->dp_need_to_free_window = B_FALSE;
3710 
3711 	/* not enough space, need to allocate more. */
3712 	} else {
3713 		dma->dp_window = kmem_alloc(space_needed, kmflag);
3714 		if (dma->dp_window == NULL) {
3715 			return (DDI_DMA_NORESOURCES);
3716 		}
3717 		dma->dp_need_to_free_window = B_TRUE;
3718 		dma->dp_window_size = space_needed;
3719 		ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3720 		    dma->dp_dip, size_t, space_needed);
3721 	}
3722 
3723 	/*
3724 	 * we allocate copy buffer state and window state at the same time.
3725 	 * setup our copy buffer state pointers. Make sure it's aligned.
3726 	 */
3727 	if (dma->dp_copybuf_size > 0) {
3728 		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3729 		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3730 
3731 #if !defined(__amd64)
3732 		/*
3733 		 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3734 		 * false/NULL. Should be quicker to bzero vs loop and set.
3735 		 */
3736 		bzero(dma->dp_pgmap, copy_state_size);
3737 #endif
3738 	} else {
3739 		dma->dp_pgmap = NULL;
3740 	}
3741 
3742 	return (DDI_SUCCESS);
3743 }
3744 
3745 
3746 /*
3747  * rootnex_teardown_copybuf()
3748  *    cleans up after rootnex_setup_copybuf()
3749  */
3750 static void
3751 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3752 {
3753 #if !defined(__amd64)
3754 	int i;
3755 
3756 	/*
3757 	 * if we allocated kernel heap VMEM space, go through all the pages and
3758 	 * map out any of the ones that we're mapped into the kernel heap VMEM
3759 	 * arena. Then free the VMEM space.
3760 	 */
3761 	if (dma->dp_kva != NULL) {
3762 		for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3763 			if (dma->dp_pgmap[i].pm_mapped) {
3764 				hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3765 				    MMU_PAGESIZE, HAT_UNLOAD);
3766 				dma->dp_pgmap[i].pm_mapped = B_FALSE;
3767 			}
3768 		}
3769 
3770 		vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3771 	}
3772 
3773 #endif
3774 
3775 	/* if we allocated a copy buffer, free it */
3776 	if (dma->dp_cbaddr != NULL) {
3777 		i_ddi_mem_free(dma->dp_cbaddr, NULL);
3778 	}
3779 }
3780 
3781 
3782 /*
3783  * rootnex_teardown_windows()
3784  *    cleans up after rootnex_setup_windows()
3785  */
3786 static void
3787 rootnex_teardown_windows(rootnex_dma_t *dma)
3788 {
3789 	/*
3790 	 * if we had to allocate window state on the last bind (because we
3791 	 * didn't have enough pre-allocated space in the handle), free it.
3792 	 */
3793 	if (dma->dp_need_to_free_window) {
3794 		kmem_free(dma->dp_window, dma->dp_window_size);
3795 	}
3796 }
3797 
3798 
3799 /*
3800  * rootnex_init_win()
3801  *    Called in bind slow path during creation of a new window. Initializes
3802  *    window state to default values.
3803  */
3804 /*ARGSUSED*/
3805 static void
3806 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3807     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3808 {
3809 	hp->dmai_nwin++;
3810 	window->wd_dosync = B_FALSE;
3811 	window->wd_offset = cur_offset;
3812 	window->wd_size = 0;
3813 	window->wd_first_cookie = cookie;
3814 	window->wd_cookie_cnt = 0;
3815 	window->wd_trim.tr_trim_first = B_FALSE;
3816 	window->wd_trim.tr_trim_last = B_FALSE;
3817 	window->wd_trim.tr_first_copybuf_win = B_FALSE;
3818 	window->wd_trim.tr_last_copybuf_win = B_FALSE;
3819 #if !defined(__amd64)
3820 	window->wd_remap_copybuf = dma->dp_cb_remaping;
3821 #endif
3822 }
3823 
3824 
3825 /*
3826  * rootnex_setup_cookie()
3827  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3828  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3829  *    out if it uses the copy buffer, and if it does, save away everything we'll
3830  *    need during sync.
3831  */
3832 static void
3833 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3834     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3835     page_t **cur_pp)
3836 {
3837 	boolean_t copybuf_sz_power_2;
3838 	rootnex_sglinfo_t *sinfo;
3839 	paddr_t paddr;
3840 	uint_t pidx;
3841 	uint_t pcnt;
3842 	off_t poff;
3843 #if defined(__amd64)
3844 	pfn_t pfn;
3845 #else
3846 	page_t **pplist;
3847 #endif
3848 
3849 	ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR);
3850 
3851 	sinfo = &dma->dp_sglinfo;
3852 
3853 	/*
3854 	 * Calculate the page index relative to the start of the buffer. The
3855 	 * index to the current page for our buffer is the offset into the
3856 	 * first page of the buffer plus our current offset into the buffer
3857 	 * itself, shifted of course...
3858 	 */
3859 	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3860 	ASSERT(pidx < sinfo->si_max_pages);
3861 
3862 	/* if this cookie uses the copy buffer */
3863 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3864 		/*
3865 		 * NOTE: we know that since this cookie uses the copy buffer, it
3866 		 * is <= MMU_PAGESIZE.
3867 		 */
3868 
3869 		/*
3870 		 * get the offset into the page. For the 64-bit kernel, get the
3871 		 * pfn which we'll use with seg kpm.
3872 		 */
3873 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
3874 #if defined(__amd64)
3875 		/* mfn_to_pfn() is a NOP on i86pc */
3876 		pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT);
3877 #endif /* __amd64 */
3878 
3879 		/* figure out if the copybuf size is a power of 2 */
3880 		if (!ISP2(dma->dp_copybuf_size)) {
3881 			copybuf_sz_power_2 = B_FALSE;
3882 		} else {
3883 			copybuf_sz_power_2 = B_TRUE;
3884 		}
3885 
3886 		/* This page uses the copy buffer */
3887 		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3888 
3889 		/*
3890 		 * save the copy buffer KVA that we'll use with this page.
3891 		 * if we still fit within the copybuf, it's a simple add.
3892 		 * otherwise, we need to wrap over using & or % accordingly.
3893 		 */
3894 		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3895 			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3896 			    *copybuf_used;
3897 		} else {
3898 			if (copybuf_sz_power_2) {
3899 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3900 				    (uintptr_t)dma->dp_cbaddr +
3901 				    (*copybuf_used &
3902 				    (dma->dp_copybuf_size - 1)));
3903 			} else {
3904 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3905 				    (uintptr_t)dma->dp_cbaddr +
3906 				    (*copybuf_used % dma->dp_copybuf_size));
3907 			}
3908 		}
3909 
3910 		/*
3911 		 * over write the cookie physical address with the address of
3912 		 * the physical address of the copy buffer page that we will
3913 		 * use.
3914 		 */
3915 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
3916 		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3917 
3918 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
3919 
3920 		/* if we have a kernel VA, it's easy, just save that address */
3921 		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3922 		    (sinfo->si_asp == &kas)) {
3923 			/*
3924 			 * save away the page aligned virtual address of the
3925 			 * driver buffer. Offsets are handled in the sync code.
3926 			 */
3927 			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3928 			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3929 			    & MMU_PAGEMASK);
3930 #if !defined(__amd64)
3931 			/*
3932 			 * we didn't need to, and will never need to map this
3933 			 * page.
3934 			 */
3935 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3936 #endif
3937 
3938 		/* we don't have a kernel VA. We need one for the bcopy. */
3939 		} else {
3940 #if defined(__amd64)
3941 			/*
3942 			 * for the 64-bit kernel, it's easy. We use seg kpm to
3943 			 * get a Kernel VA for the corresponding pfn.
3944 			 */
3945 			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3946 #else
3947 			/*
3948 			 * for the 32-bit kernel, this is a pain. First we'll
3949 			 * save away the page_t or user VA for this page. This
3950 			 * is needed in rootnex_dma_win() when we switch to a
3951 			 * new window which requires us to re-map the copy
3952 			 * buffer.
3953 			 */
3954 			pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3955 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3956 				dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3957 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3958 			} else if (pplist != NULL) {
3959 				dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3960 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3961 			} else {
3962 				dma->dp_pgmap[pidx].pm_pp = NULL;
3963 				dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3964 				    (((uintptr_t)
3965 				    dmar_object->dmao_obj.virt_obj.v_addr +
3966 				    cur_offset) & MMU_PAGEMASK);
3967 			}
3968 
3969 			/*
3970 			 * save away the page aligned virtual address which was
3971 			 * allocated from the kernel heap arena (taking into
3972 			 * account if we need more copy buffer than we alloced
3973 			 * and use multiple windows to handle this, i.e. &,%).
3974 			 * NOTE: there isn't and physical memory backing up this
3975 			 * virtual address space currently.
3976 			 */
3977 			if ((*copybuf_used + MMU_PAGESIZE) <=
3978 			    dma->dp_copybuf_size) {
3979 				dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3980 				    (((uintptr_t)dma->dp_kva + *copybuf_used) &
3981 				    MMU_PAGEMASK);
3982 			} else {
3983 				if (copybuf_sz_power_2) {
3984 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3985 					    (((uintptr_t)dma->dp_kva +
3986 					    (*copybuf_used &
3987 					    (dma->dp_copybuf_size - 1))) &
3988 					    MMU_PAGEMASK);
3989 				} else {
3990 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3991 					    (((uintptr_t)dma->dp_kva +
3992 					    (*copybuf_used %
3993 					    dma->dp_copybuf_size)) &
3994 					    MMU_PAGEMASK);
3995 				}
3996 			}
3997 
3998 			/*
3999 			 * if we haven't used up the available copy buffer yet,
4000 			 * map the kva to the physical page.
4001 			 */
4002 			if (!dma->dp_cb_remaping && ((*copybuf_used +
4003 			    MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
4004 				dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
4005 				if (dma->dp_pgmap[pidx].pm_pp != NULL) {
4006 					i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
4007 					    dma->dp_pgmap[pidx].pm_kaddr);
4008 				} else {
4009 					i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
4010 					    sinfo->si_asp,
4011 					    dma->dp_pgmap[pidx].pm_kaddr);
4012 				}
4013 
4014 			/*
4015 			 * we've used up the available copy buffer, this page
4016 			 * will have to be mapped during rootnex_dma_win() when
4017 			 * we switch to a new window which requires a re-map
4018 			 * the copy buffer. (32-bit kernel only)
4019 			 */
4020 			} else {
4021 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4022 			}
4023 #endif
4024 			/* go to the next page_t */
4025 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4026 				*cur_pp = (*cur_pp)->p_next;
4027 			}
4028 		}
4029 
4030 		/* add to the copy buffer count */
4031 		*copybuf_used += MMU_PAGESIZE;
4032 
4033 	/*
4034 	 * This cookie doesn't use the copy buffer. Walk through the pages this
4035 	 * cookie occupies to reflect this.
4036 	 */
4037 	} else {
4038 		/*
4039 		 * figure out how many pages the cookie occupies. We need to
4040 		 * use the original page offset of the buffer and the cookies
4041 		 * offset in the buffer to do this.
4042 		 */
4043 		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
4044 		pcnt = mmu_btopr(cookie->dmac_size + poff);
4045 
4046 		while (pcnt > 0) {
4047 #if !defined(__amd64)
4048 			/*
4049 			 * the 32-bit kernel doesn't have seg kpm, so we need
4050 			 * to map in the driver buffer (if it didn't come down
4051 			 * with a kernel VA) on the fly. Since this page doesn't
4052 			 * use the copy buffer, it's not, or will it ever, have
4053 			 * to be mapped in.
4054 			 */
4055 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4056 #endif
4057 			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
4058 
4059 			/*
4060 			 * we need to update pidx and cur_pp or we'll loose
4061 			 * track of where we are.
4062 			 */
4063 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
4064 				*cur_pp = (*cur_pp)->p_next;
4065 			}
4066 			pidx++;
4067 			pcnt--;
4068 		}
4069 	}
4070 }
4071 
4072 
4073 /*
4074  * rootnex_sgllen_window_boundary()
4075  *    Called in the bind slow path when the next cookie causes us to exceed (in
4076  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
4077  *    length supported by the DMA H/W.
4078  */
4079 static int
4080 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4081     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
4082     off_t cur_offset)
4083 {
4084 	off_t new_offset;
4085 	size_t trim_sz;
4086 	off_t coffset;
4087 
4088 
4089 	/*
4090 	 * if we know we'll never have to trim, it's pretty easy. Just move to
4091 	 * the next window and init it. We're done.
4092 	 */
4093 	if (!dma->dp_trim_required) {
4094 		(*windowp)++;
4095 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4096 		(*windowp)->wd_cookie_cnt++;
4097 		(*windowp)->wd_size = cookie->dmac_size;
4098 		return (DDI_SUCCESS);
4099 	}
4100 
4101 	/* figure out how much we need to trim from the window */
4102 	ASSERT(attr->dma_attr_granular != 0);
4103 	if (dma->dp_granularity_power_2) {
4104 		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
4105 	} else {
4106 		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
4107 	}
4108 
4109 	/* The window's a whole multiple of granularity. We're done */
4110 	if (trim_sz == 0) {
4111 		(*windowp)++;
4112 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4113 		(*windowp)->wd_cookie_cnt++;
4114 		(*windowp)->wd_size = cookie->dmac_size;
4115 		return (DDI_SUCCESS);
4116 	}
4117 
4118 	/*
4119 	 * The window's not a whole multiple of granularity, since we know this
4120 	 * is due to the sgllen, we need to go back to the last cookie and trim
4121 	 * that one, add the left over part of the old cookie into the new
4122 	 * window, and then add in the new cookie into the new window.
4123 	 */
4124 
4125 	/*
4126 	 * make sure the driver isn't making us do something bad... Trimming and
4127 	 * sgllen == 1 don't go together.
4128 	 */
4129 	if (attr->dma_attr_sgllen == 1) {
4130 		return (DDI_DMA_NOMAPPING);
4131 	}
4132 
4133 	/*
4134 	 * first, setup the current window to account for the trim. Need to go
4135 	 * back to the last cookie for this.
4136 	 */
4137 	cookie--;
4138 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4139 	(*windowp)->wd_trim.tr_last_cookie = cookie;
4140 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4141 	ASSERT(cookie->dmac_size > trim_sz);
4142 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4143 	(*windowp)->wd_size -= trim_sz;
4144 
4145 	/* save the buffer offsets for the next window */
4146 	coffset = cookie->dmac_size - trim_sz;
4147 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4148 
4149 	/*
4150 	 * set this now in case this is the first window. all other cases are
4151 	 * set in dma_win()
4152 	 */
4153 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4154 
4155 	/*
4156 	 * initialize the next window using what's left over in the previous
4157 	 * cookie.
4158 	 */
4159 	(*windowp)++;
4160 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4161 	(*windowp)->wd_cookie_cnt++;
4162 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4163 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4164 	(*windowp)->wd_trim.tr_first_size = trim_sz;
4165 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4166 		(*windowp)->wd_dosync = B_TRUE;
4167 	}
4168 
4169 	/*
4170 	 * now go back to the current cookie and add it to the new window. set
4171 	 * the new window size to the what was left over from the previous
4172 	 * cookie and what's in the current cookie.
4173 	 */
4174 	cookie++;
4175 	(*windowp)->wd_cookie_cnt++;
4176 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
4177 
4178 	/*
4179 	 * trim plus the next cookie could put us over maxxfer (a cookie can be
4180 	 * a max size of maxxfer). Handle that case.
4181 	 */
4182 	if ((*windowp)->wd_size > dma->dp_maxxfer) {
4183 		/*
4184 		 * maxxfer is already a whole multiple of granularity, and this
4185 		 * trim will be <= the previous trim (since a cookie can't be
4186 		 * larger than maxxfer). Make things simple here.
4187 		 */
4188 		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
4189 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4190 		(*windowp)->wd_trim.tr_last_cookie = cookie;
4191 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4192 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4193 		(*windowp)->wd_size -= trim_sz;
4194 		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
4195 
4196 		/* save the buffer offsets for the next window */
4197 		coffset = cookie->dmac_size - trim_sz;
4198 		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4199 
4200 		/* setup the next window */
4201 		(*windowp)++;
4202 		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4203 		(*windowp)->wd_cookie_cnt++;
4204 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4205 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4206 		    coffset;
4207 		(*windowp)->wd_trim.tr_first_size = trim_sz;
4208 	}
4209 
4210 	return (DDI_SUCCESS);
4211 }
4212 
4213 
4214 /*
4215  * rootnex_copybuf_window_boundary()
4216  *    Called in bind slowpath when we get to a window boundary because we used
4217  *    up all the copy buffer that we have.
4218  */
4219 static int
4220 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4221     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
4222     size_t *copybuf_used)
4223 {
4224 	rootnex_sglinfo_t *sinfo;
4225 	off_t new_offset;
4226 	size_t trim_sz;
4227 	paddr_t paddr;
4228 	off_t coffset;
4229 	uint_t pidx;
4230 	off_t poff;
4231 
4232 	pidx = 0;
4233 	sinfo = &dma->dp_sglinfo;
4234 
4235 	/*
4236 	 * the copy buffer should be a whole multiple of page size. We know that
4237 	 * this cookie is <= MMU_PAGESIZE.
4238 	 */
4239 	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
4240 
4241 	/*
4242 	 * from now on, all new windows in this bind need to be re-mapped during
4243 	 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
4244 	 * space...
4245 	 */
4246 #if !defined(__amd64)
4247 	dma->dp_cb_remaping = B_TRUE;
4248 #endif
4249 
4250 	/* reset copybuf used */
4251 	*copybuf_used = 0;
4252 
4253 	/*
4254 	 * if we don't have to trim (since granularity is set to 1), go to the
4255 	 * next window and add the current cookie to it. We know the current
4256 	 * cookie uses the copy buffer since we're in this code path.
4257 	 */
4258 	if (!dma->dp_trim_required) {
4259 		(*windowp)++;
4260 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4261 
4262 		/* Add this cookie to the new window */
4263 		(*windowp)->wd_cookie_cnt++;
4264 		(*windowp)->wd_size += cookie->dmac_size;
4265 		*copybuf_used += MMU_PAGESIZE;
4266 		return (DDI_SUCCESS);
4267 	}
4268 
4269 	/*
4270 	 * *** may need to trim, figure it out.
4271 	 */
4272 
4273 	/* figure out how much we need to trim from the window */
4274 	if (dma->dp_granularity_power_2) {
4275 		trim_sz = (*windowp)->wd_size &
4276 		    (hp->dmai_attr.dma_attr_granular - 1);
4277 	} else {
4278 		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
4279 	}
4280 
4281 	/*
4282 	 * if the window's a whole multiple of granularity, go to the next
4283 	 * window, init it, then add in the current cookie. We know the current
4284 	 * cookie uses the copy buffer since we're in this code path.
4285 	 */
4286 	if (trim_sz == 0) {
4287 		(*windowp)++;
4288 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
4289 
4290 		/* Add this cookie to the new window */
4291 		(*windowp)->wd_cookie_cnt++;
4292 		(*windowp)->wd_size += cookie->dmac_size;
4293 		*copybuf_used += MMU_PAGESIZE;
4294 		return (DDI_SUCCESS);
4295 	}
4296 
4297 	/*
4298 	 * *** We figured it out, we definitly need to trim
4299 	 */
4300 
4301 	/*
4302 	 * make sure the driver isn't making us do something bad...
4303 	 * Trimming and sgllen == 1 don't go together.
4304 	 */
4305 	if (hp->dmai_attr.dma_attr_sgllen == 1) {
4306 		return (DDI_DMA_NOMAPPING);
4307 	}
4308 
4309 	/*
4310 	 * first, setup the current window to account for the trim. Need to go
4311 	 * back to the last cookie for this. Some of the last cookie will be in
4312 	 * the current window, and some of the last cookie will be in the new
4313 	 * window. All of the current cookie will be in the new window.
4314 	 */
4315 	cookie--;
4316 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4317 	(*windowp)->wd_trim.tr_last_cookie = cookie;
4318 	(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4319 	ASSERT(cookie->dmac_size > trim_sz);
4320 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4321 	(*windowp)->wd_size -= trim_sz;
4322 
4323 	/*
4324 	 * we're trimming the last cookie (not the current cookie). So that
4325 	 * last cookie may have or may not have been using the copy buffer (
4326 	 * we know the cookie passed in uses the copy buffer since we're in
4327 	 * this code path).
4328 	 *
4329 	 * If the last cookie doesn't use the copy buffer, nothing special to
4330 	 * do. However, if it does uses the copy buffer, it will be both the
4331 	 * last page in the current window and the first page in the next
4332 	 * window. Since we are reusing the copy buffer (and KVA space on the
4333 	 * 32-bit kernel), this page will use the end of the copy buffer in the
4334 	 * current window, and the start of the copy buffer in the next window.
4335 	 * Track that info... The cookie physical address was already set to
4336 	 * the copy buffer physical address in setup_cookie..
4337 	 */
4338 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4339 		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
4340 		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
4341 		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
4342 		(*windowp)->wd_trim.tr_last_pidx = pidx;
4343 		(*windowp)->wd_trim.tr_last_cbaddr =
4344 		    dma->dp_pgmap[pidx].pm_cbaddr;
4345 #if !defined(__amd64)
4346 		(*windowp)->wd_trim.tr_last_kaddr =
4347 		    dma->dp_pgmap[pidx].pm_kaddr;
4348 #endif
4349 	}
4350 
4351 	/* save the buffer offsets for the next window */
4352 	coffset = cookie->dmac_size - trim_sz;
4353 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4354 
4355 	/*
4356 	 * set this now in case this is the first window. all other cases are
4357 	 * set in dma_win()
4358 	 */
4359 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4360 
4361 	/*
4362 	 * initialize the next window using what's left over in the previous
4363 	 * cookie.
4364 	 */
4365 	(*windowp)++;
4366 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4367 	(*windowp)->wd_cookie_cnt++;
4368 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4369 	(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset;
4370 	(*windowp)->wd_trim.tr_first_size = trim_sz;
4371 
4372 	/*
4373 	 * again, we're tracking if the last cookie uses the copy buffer.
4374 	 * read the comment above for more info on why we need to track
4375 	 * additional state.
4376 	 *
4377 	 * For the first cookie in the new window, we need reset the physical
4378 	 * address to DMA into to the start of the copy buffer plus any
4379 	 * initial page offset which may be present.
4380 	 */
4381 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
4382 		(*windowp)->wd_dosync = B_TRUE;
4383 		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
4384 		(*windowp)->wd_trim.tr_first_pidx = pidx;
4385 		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
4386 		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
4387 
4388 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) +
4389 		    poff;
4390 		(*windowp)->wd_trim.tr_first_paddr =
4391 		    ROOTNEX_PADDR_TO_RBASE(paddr);
4392 
4393 #if !defined(__amd64)
4394 		(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
4395 #endif
4396 		/* account for the cookie copybuf usage in the new window */
4397 		*copybuf_used += MMU_PAGESIZE;
4398 
4399 		/*
4400 		 * every piece of code has to have a hack, and here is this
4401 		 * ones :-)
4402 		 *
4403 		 * There is a complex interaction between setup_cookie and the
4404 		 * copybuf window boundary. The complexity had to be in either
4405 		 * the maxxfer window, or the copybuf window, and I chose the
4406 		 * copybuf code.
4407 		 *
4408 		 * So in this code path, we have taken the last cookie,
4409 		 * virtually broken it in half due to the trim, and it happens
4410 		 * to use the copybuf which further complicates life. At the
4411 		 * same time, we have already setup the current cookie, which
4412 		 * is now wrong. More background info: the current cookie uses
4413 		 * the copybuf, so it is only a page long max. So we need to
4414 		 * fix the current cookies copy buffer address, physical
4415 		 * address, and kva for the 32-bit kernel. We due this by
4416 		 * bumping them by page size (of course, we can't due this on
4417 		 * the physical address since the copy buffer may not be
4418 		 * physically contiguous).
4419 		 */
4420 		cookie++;
4421 		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
4422 		poff = cookie->dmac_laddress & MMU_PAGEOFFSET;
4423 
4424 		paddr = pfn_to_pa(hat_getpfnum(kas.a_hat,
4425 		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
4426 		cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr);
4427 
4428 #if !defined(__amd64)
4429 		ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
4430 		dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
4431 #endif
4432 	} else {
4433 		/* go back to the current cookie */
4434 		cookie++;
4435 	}
4436 
4437 	/*
4438 	 * add the current cookie to the new window. set the new window size to
4439 	 * the what was left over from the previous cookie and what's in the
4440 	 * current cookie.
4441 	 */
4442 	(*windowp)->wd_cookie_cnt++;
4443 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
4444 	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
4445 
4446 	/*
4447 	 * we know that the cookie passed in always uses the copy buffer. We
4448 	 * wouldn't be here if it didn't.
4449 	 */
4450 	*copybuf_used += MMU_PAGESIZE;
4451 
4452 	return (DDI_SUCCESS);
4453 }
4454 
4455 
4456 /*
4457  * rootnex_maxxfer_window_boundary()
4458  *    Called in bind slowpath when we get to a window boundary because we will
4459  *    go over maxxfer.
4460  */
4461 static int
4462 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
4463     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
4464 {
4465 	size_t dmac_size;
4466 	off_t new_offset;
4467 	size_t trim_sz;
4468 	off_t coffset;
4469 
4470 
4471 	/*
4472 	 * calculate how much we have to trim off of the current cookie to equal
4473 	 * maxxfer. We don't have to account for granularity here since our
4474 	 * maxxfer already takes that into account.
4475 	 */
4476 	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
4477 	ASSERT(trim_sz <= cookie->dmac_size);
4478 	ASSERT(trim_sz <= dma->dp_maxxfer);
4479 
4480 	/* save cookie size since we need it later and we might change it */
4481 	dmac_size = cookie->dmac_size;
4482 
4483 	/*
4484 	 * if we're not trimming the entire cookie, setup the current window to
4485 	 * account for the trim.
4486 	 */
4487 	if (trim_sz < cookie->dmac_size) {
4488 		(*windowp)->wd_cookie_cnt++;
4489 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
4490 		(*windowp)->wd_trim.tr_last_cookie = cookie;
4491 		(*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress;
4492 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
4493 		(*windowp)->wd_size = dma->dp_maxxfer;
4494 
4495 		/*
4496 		 * set the adjusted cookie size now in case this is the first
4497 		 * window. All other windows are taken care of in get win
4498 		 */
4499 		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
4500 	}
4501 
4502 	/*
4503 	 * coffset is the current offset within the cookie, new_offset is the
4504 	 * current offset with the entire buffer.
4505 	 */
4506 	coffset = dmac_size - trim_sz;
4507 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
4508 
4509 	/* initialize the next window */
4510 	(*windowp)++;
4511 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
4512 	(*windowp)->wd_cookie_cnt++;
4513 	(*windowp)->wd_size = trim_sz;
4514 	if (trim_sz < dmac_size) {
4515 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
4516 		(*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress +
4517 		    coffset;
4518 		(*windowp)->wd_trim.tr_first_size = trim_sz;
4519 	}
4520 
4521 	return (DDI_SUCCESS);
4522 }
4523 
4524 
4525 /*ARGSUSED*/
4526 static int
4527 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4528     off_t off, size_t len, uint_t cache_flags)
4529 {
4530 	rootnex_sglinfo_t *sinfo;
4531 	rootnex_pgmap_t *cbpage;
4532 	rootnex_window_t *win;
4533 	ddi_dma_impl_t *hp;
4534 	rootnex_dma_t *dma;
4535 	caddr_t fromaddr;
4536 	caddr_t toaddr;
4537 	uint_t psize;
4538 	off_t offset;
4539 	uint_t pidx;
4540 	size_t size;
4541 	off_t poff;
4542 	int e;
4543 
4544 
4545 	hp = (ddi_dma_impl_t *)handle;
4546 	dma = (rootnex_dma_t *)hp->dmai_private;
4547 	sinfo = &dma->dp_sglinfo;
4548 
4549 	/*
4550 	 * if we don't have any windows, we don't need to sync. A copybuf
4551 	 * will cause us to have at least one window.
4552 	 */
4553 	if (dma->dp_window == NULL) {
4554 		return (DDI_SUCCESS);
4555 	}
4556 
4557 	/* This window may not need to be sync'd */
4558 	win = &dma->dp_window[dma->dp_current_win];
4559 	if (!win->wd_dosync) {
4560 		return (DDI_SUCCESS);
4561 	}
4562 
4563 	/* handle off and len special cases */
4564 	if ((off == 0) || (rootnex_sync_ignore_params)) {
4565 		offset = win->wd_offset;
4566 	} else {
4567 		offset = off;
4568 	}
4569 	if ((len == 0) || (rootnex_sync_ignore_params)) {
4570 		size = win->wd_size;
4571 	} else {
4572 		size = len;
4573 	}
4574 
4575 	/* check the sync args to make sure they make a little sense */
4576 	if (rootnex_sync_check_parms) {
4577 		e = rootnex_valid_sync_parms(hp, win, offset, size,
4578 		    cache_flags);
4579 		if (e != DDI_SUCCESS) {
4580 			ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
4581 			return (DDI_FAILURE);
4582 		}
4583 	}
4584 
4585 	/*
4586 	 * special case the first page to handle the offset into the page. The
4587 	 * offset to the current page for our buffer is the offset into the
4588 	 * first page of the buffer plus our current offset into the buffer
4589 	 * itself, masked of course.
4590 	 */
4591 	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
4592 	psize = MIN((MMU_PAGESIZE - poff), size);
4593 
4594 	/* go through all the pages that we want to sync */
4595 	while (size > 0) {
4596 		/*
4597 		 * Calculate the page index relative to the start of the buffer.
4598 		 * The index to the current page for our buffer is the offset
4599 		 * into the first page of the buffer plus our current offset
4600 		 * into the buffer itself, shifted of course...
4601 		 */
4602 		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
4603 		ASSERT(pidx < sinfo->si_max_pages);
4604 
4605 		/*
4606 		 * if this page uses the copy buffer, we need to sync it,
4607 		 * otherwise, go on to the next page.
4608 		 */
4609 		cbpage = &dma->dp_pgmap[pidx];
4610 		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
4611 		    (cbpage->pm_uses_copybuf == B_FALSE));
4612 		if (cbpage->pm_uses_copybuf) {
4613 			/* cbaddr and kaddr should be page aligned */
4614 			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
4615 			    MMU_PAGEOFFSET) == 0);
4616 			ASSERT(((uintptr_t)cbpage->pm_kaddr &
4617 			    MMU_PAGEOFFSET) == 0);
4618 
4619 			/*
4620 			 * if we're copying for the device, we are going to
4621 			 * copy from the drivers buffer and to the rootnex
4622 			 * allocated copy buffer.
4623 			 */
4624 			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
4625 				fromaddr = cbpage->pm_kaddr + poff;
4626 				toaddr = cbpage->pm_cbaddr + poff;
4627 				ROOTNEX_DPROBE2(rootnex__sync__dev,
4628 				    dev_info_t *, dma->dp_dip, size_t, psize);
4629 
4630 			/*
4631 			 * if we're copying for the cpu/kernel, we are going to
4632 			 * copy from the rootnex allocated copy buffer to the
4633 			 * drivers buffer.
4634 			 */
4635 			} else {
4636 				fromaddr = cbpage->pm_cbaddr + poff;
4637 				toaddr = cbpage->pm_kaddr + poff;
4638 				ROOTNEX_DPROBE2(rootnex__sync__cpu,
4639 				    dev_info_t *, dma->dp_dip, size_t, psize);
4640 			}
4641 
4642 			bcopy(fromaddr, toaddr, psize);
4643 		}
4644 
4645 		/*
4646 		 * decrement size until we're done, update our offset into the
4647 		 * buffer, and get the next page size.
4648 		 */
4649 		size -= psize;
4650 		offset += psize;
4651 		psize = MIN(MMU_PAGESIZE, size);
4652 
4653 		/* page offset is zero for the rest of this loop */
4654 		poff = 0;
4655 	}
4656 
4657 	return (DDI_SUCCESS);
4658 }
4659 
4660 /*
4661  * rootnex_dma_sync()
4662  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
4663  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
4664  *    is set, ddi_dma_sync() returns immediately passing back success.
4665  */
4666 /*ARGSUSED*/
4667 static int
4668 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4669     off_t off, size_t len, uint_t cache_flags)
4670 {
4671 #if defined(__amd64) && !defined(__xpv)
4672 	if (IOMMU_USED(rdip)) {
4673 		return (iommulib_nexdma_sync(dip, rdip, handle, off, len,
4674 		    cache_flags));
4675 	}
4676 #endif
4677 	return (rootnex_coredma_sync(dip, rdip, handle, off, len,
4678 	    cache_flags));
4679 }
4680 
4681 /*
4682  * rootnex_valid_sync_parms()
4683  *    checks the parameters passed to sync to verify they are correct.
4684  */
4685 static int
4686 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4687     off_t offset, size_t size, uint_t cache_flags)
4688 {
4689 	off_t woffset;
4690 
4691 
4692 	/*
4693 	 * the first part of the test to make sure the offset passed in is
4694 	 * within the window.
4695 	 */
4696 	if (offset < win->wd_offset) {
4697 		return (DDI_FAILURE);
4698 	}
4699 
4700 	/*
4701 	 * second and last part of the test to make sure the offset and length
4702 	 * passed in is within the window.
4703 	 */
4704 	woffset = offset - win->wd_offset;
4705 	if ((woffset + size) > win->wd_size) {
4706 		return (DDI_FAILURE);
4707 	}
4708 
4709 	/*
4710 	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4711 	 * be set too.
4712 	 */
4713 	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4714 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
4715 		return (DDI_SUCCESS);
4716 	}
4717 
4718 	/*
4719 	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4720 	 * should be set. Also DDI_DMA_READ should be set in the flags.
4721 	 */
4722 	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4723 	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4724 	    (hp->dmai_rflags & DDI_DMA_READ)) {
4725 		return (DDI_SUCCESS);
4726 	}
4727 
4728 	return (DDI_FAILURE);
4729 }
4730 
4731 
4732 /*ARGSUSED*/
4733 static int
4734 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4735     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4736     uint_t *ccountp)
4737 {
4738 	rootnex_window_t *window;
4739 	rootnex_trim_t *trim;
4740 	ddi_dma_impl_t *hp;
4741 	rootnex_dma_t *dma;
4742 	ddi_dma_obj_t *dmao;
4743 #if !defined(__amd64)
4744 	rootnex_sglinfo_t *sinfo;
4745 	rootnex_pgmap_t *pmap;
4746 	uint_t pidx;
4747 	uint_t pcnt;
4748 	off_t poff;
4749 	int i;
4750 #endif
4751 
4752 
4753 	hp = (ddi_dma_impl_t *)handle;
4754 	dma = (rootnex_dma_t *)hp->dmai_private;
4755 #if !defined(__amd64)
4756 	sinfo = &dma->dp_sglinfo;
4757 #endif
4758 
4759 	/* If we try and get a window which doesn't exist, return failure */
4760 	if (win >= hp->dmai_nwin) {
4761 		ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4762 		return (DDI_FAILURE);
4763 	}
4764 
4765 	dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma;
4766 
4767 	/*
4768 	 * if we don't have any windows, and they're asking for the first
4769 	 * window, setup the cookie pointer to the first cookie in the bind.
4770 	 * setup our return values, then increment the cookie since we return
4771 	 * the first cookie on the stack.
4772 	 */
4773 	if (dma->dp_window == NULL) {
4774 		if (win != 0) {
4775 			ROOTNEX_DPROF_INC(
4776 			    &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4777 			return (DDI_FAILURE);
4778 		}
4779 		hp->dmai_cookie = dma->dp_cookies;
4780 		*offp = 0;
4781 		*lenp = dmao->dmao_size;
4782 		*ccountp = dma->dp_sglinfo.si_sgl_size;
4783 		*cookiep = hp->dmai_cookie[0];
4784 		hp->dmai_cookie++;
4785 		hp->dmai_ncookies = *ccountp;
4786 		hp->dmai_curcookie = 1;
4787 		return (DDI_SUCCESS);
4788 	}
4789 
4790 	/* sync the old window before moving on to the new one */
4791 	window = &dma->dp_window[dma->dp_current_win];
4792 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4793 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4794 		    DDI_DMA_SYNC_FORCPU);
4795 	}
4796 
4797 #if !defined(__amd64)
4798 	/*
4799 	 * before we move to the next window, if we need to re-map, unmap all
4800 	 * the pages in this window.
4801 	 */
4802 	if (dma->dp_cb_remaping) {
4803 		/*
4804 		 * If we switch to this window again, we'll need to map in
4805 		 * on the fly next time.
4806 		 */
4807 		window->wd_remap_copybuf = B_TRUE;
4808 
4809 		/*
4810 		 * calculate the page index into the buffer where this window
4811 		 * starts, and the number of pages this window takes up.
4812 		 */
4813 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4814 		    MMU_PAGESHIFT;
4815 		poff = (sinfo->si_buf_offset + window->wd_offset) &
4816 		    MMU_PAGEOFFSET;
4817 		pcnt = mmu_btopr(window->wd_size + poff);
4818 		ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4819 
4820 		/* unmap pages which are currently mapped in this window */
4821 		for (i = 0; i < pcnt; i++) {
4822 			if (dma->dp_pgmap[pidx].pm_mapped) {
4823 				hat_unload(kas.a_hat,
4824 				    dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4825 				    HAT_UNLOAD);
4826 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4827 			}
4828 			pidx++;
4829 		}
4830 	}
4831 #endif
4832 
4833 	/*
4834 	 * Move to the new window.
4835 	 * NOTE: current_win must be set for sync to work right
4836 	 */
4837 	dma->dp_current_win = win;
4838 	window = &dma->dp_window[win];
4839 
4840 	/* if needed, adjust the first and/or last cookies for trim */
4841 	trim = &window->wd_trim;
4842 	if (trim->tr_trim_first) {
4843 		window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr;
4844 		window->wd_first_cookie->dmac_size = trim->tr_first_size;
4845 #if !defined(__amd64)
4846 		window->wd_first_cookie->dmac_type =
4847 		    (window->wd_first_cookie->dmac_type &
4848 		    ROOTNEX_USES_COPYBUF) + window->wd_offset;
4849 #endif
4850 		if (trim->tr_first_copybuf_win) {
4851 			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4852 			    trim->tr_first_cbaddr;
4853 #if !defined(__amd64)
4854 			dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4855 			    trim->tr_first_kaddr;
4856 #endif
4857 		}
4858 	}
4859 	if (trim->tr_trim_last) {
4860 		trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr;
4861 		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4862 		if (trim->tr_last_copybuf_win) {
4863 			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4864 			    trim->tr_last_cbaddr;
4865 #if !defined(__amd64)
4866 			dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4867 			    trim->tr_last_kaddr;
4868 #endif
4869 		}
4870 	}
4871 
4872 	/*
4873 	 * setup the cookie pointer to the first cookie in the window. setup
4874 	 * our return values, then increment the cookie since we return the
4875 	 * first cookie on the stack.
4876 	 */
4877 	hp->dmai_cookie = window->wd_first_cookie;
4878 	*offp = window->wd_offset;
4879 	*lenp = window->wd_size;
4880 	*ccountp = window->wd_cookie_cnt;
4881 	*cookiep = hp->dmai_cookie[0];
4882 	hp->dmai_ncookies = *ccountp;
4883 	hp->dmai_curcookie = 1;
4884 	hp->dmai_cookie++;
4885 
4886 #if !defined(__amd64)
4887 	/* re-map copybuf if required for this window */
4888 	if (dma->dp_cb_remaping) {
4889 		/*
4890 		 * calculate the page index into the buffer where this
4891 		 * window starts.
4892 		 */
4893 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4894 		    MMU_PAGESHIFT;
4895 		ASSERT(pidx < sinfo->si_max_pages);
4896 
4897 		/*
4898 		 * the first page can get unmapped if it's shared with the
4899 		 * previous window. Even if the rest of this window is already
4900 		 * mapped in, we need to still check this one.
4901 		 */
4902 		pmap = &dma->dp_pgmap[pidx];
4903 		if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4904 			if (pmap->pm_pp != NULL) {
4905 				pmap->pm_mapped = B_TRUE;
4906 				i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4907 			} else if (pmap->pm_vaddr != NULL) {
4908 				pmap->pm_mapped = B_TRUE;
4909 				i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4910 				    pmap->pm_kaddr);
4911 			}
4912 		}
4913 		pidx++;
4914 
4915 		/* map in the rest of the pages if required */
4916 		if (window->wd_remap_copybuf) {
4917 			window->wd_remap_copybuf = B_FALSE;
4918 
4919 			/* figure out many pages this window takes up */
4920 			poff = (sinfo->si_buf_offset + window->wd_offset) &
4921 			    MMU_PAGEOFFSET;
4922 			pcnt = mmu_btopr(window->wd_size + poff);
4923 			ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4924 
4925 			/* map pages which require it */
4926 			for (i = 1; i < pcnt; i++) {
4927 				pmap = &dma->dp_pgmap[pidx];
4928 				if (pmap->pm_uses_copybuf) {
4929 					ASSERT(pmap->pm_mapped == B_FALSE);
4930 					if (pmap->pm_pp != NULL) {
4931 						pmap->pm_mapped = B_TRUE;
4932 						i86_pp_map(pmap->pm_pp,
4933 						    pmap->pm_kaddr);
4934 					} else if (pmap->pm_vaddr != NULL) {
4935 						pmap->pm_mapped = B_TRUE;
4936 						i86_va_map(pmap->pm_vaddr,
4937 						    sinfo->si_asp,
4938 						    pmap->pm_kaddr);
4939 					}
4940 				}
4941 				pidx++;
4942 			}
4943 		}
4944 	}
4945 #endif
4946 
4947 	/* if the new window uses the copy buffer, sync it for the device */
4948 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4949 		(void) rootnex_coredma_sync(dip, rdip, handle, 0, 0,
4950 		    DDI_DMA_SYNC_FORDEV);
4951 	}
4952 
4953 	return (DDI_SUCCESS);
4954 }
4955 
4956 /*
4957  * rootnex_dma_win()
4958  *    called from ddi_dma_getwin()
4959  */
4960 /*ARGSUSED*/
4961 static int
4962 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4963     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4964     uint_t *ccountp)
4965 {
4966 #if defined(__amd64) && !defined(__xpv)
4967 	if (IOMMU_USED(rdip)) {
4968 		return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp,
4969 		    cookiep, ccountp));
4970 	}
4971 #endif
4972 
4973 	return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp,
4974 	    cookiep, ccountp));
4975 }
4976 
4977 #if defined(__amd64) && !defined(__xpv)
4978 /*ARGSUSED*/
4979 static int
4980 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip,
4981     ddi_dma_handle_t handle, void *v)
4982 {
4983 	ddi_dma_impl_t *hp;
4984 	rootnex_dma_t *dma;
4985 
4986 	hp = (ddi_dma_impl_t *)handle;
4987 	dma = (rootnex_dma_t *)hp->dmai_private;
4988 	dma->dp_iommu_private = v;
4989 
4990 	return (DDI_SUCCESS);
4991 }
4992 
4993 /*ARGSUSED*/
4994 static void *
4995 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip,
4996     ddi_dma_handle_t handle)
4997 {
4998 	ddi_dma_impl_t *hp;
4999 	rootnex_dma_t *dma;
5000 
5001 	hp = (ddi_dma_impl_t *)handle;
5002 	dma = (rootnex_dma_t *)hp->dmai_private;
5003 
5004 	return (dma->dp_iommu_private);
5005 }
5006 #endif
5007 
5008 /*
5009  * ************************
5010  *  obsoleted dma routines
5011  * ************************
5012  */
5013 
5014 /*
5015  * rootnex_dma_mctl()
5016  *
5017  * We don't support this legacy interface any more on x86.
5018  */
5019 /* ARGSUSED */
5020 static int
5021 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
5022     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
5023     uint_t cache_flags)
5024 {
5025 	/*
5026 	 * The only thing dma_mctl is usef for anymore is legacy SPARC
5027 	 * dvma and sbus-specific routines.
5028 	 */
5029 	return (DDI_FAILURE);
5030 }
5031 
5032 /*
5033  * *********
5034  *  FMA Code
5035  * *********
5036  */
5037 
5038 /*
5039  * rootnex_fm_init()
5040  *    FMA init busop
5041  */
5042 /* ARGSUSED */
5043 static int
5044 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
5045     ddi_iblock_cookie_t *ibc)
5046 {
5047 	*ibc = rootnex_state->r_err_ibc;
5048 
5049 	return (ddi_system_fmcap);
5050 }
5051 
5052 /*
5053  * rootnex_dma_check()
5054  *    Function called after a dma fault occurred to find out whether the
5055  *    fault address is associated with a driver that is able to handle faults
5056  *    and recover from faults.
5057  */
5058 /* ARGSUSED */
5059 static int
5060 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
5061     const void *not_used)
5062 {
5063 	rootnex_window_t *window;
5064 	uint64_t start_addr;
5065 	uint64_t fault_addr;
5066 	ddi_dma_impl_t *hp;
5067 	rootnex_dma_t *dma;
5068 	uint64_t end_addr;
5069 	size_t csize;
5070 	int i;
5071 	int j;
5072 
5073 
5074 	/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
5075 	hp = (ddi_dma_impl_t *)handle;
5076 	ASSERT(hp);
5077 
5078 	dma = (rootnex_dma_t *)hp->dmai_private;
5079 
5080 	/* Get the address that we need to search for */
5081 	fault_addr = *(uint64_t *)addr;
5082 
5083 	/*
5084 	 * if we don't have any windows, we can just walk through all the
5085 	 * cookies.
5086 	 */
5087 	if (dma->dp_window == NULL) {
5088 		/* for each cookie */
5089 		for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
5090 			/*
5091 			 * if the faulted address is within the physical address
5092 			 * range of the cookie, return DDI_FM_NONFATAL.
5093 			 */
5094 			if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
5095 			    (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
5096 			    dma->dp_cookies[i].dmac_size))) {
5097 				return (DDI_FM_NONFATAL);
5098 			}
5099 		}
5100 
5101 		/* fault_addr not within this DMA handle */
5102 		return (DDI_FM_UNKNOWN);
5103 	}
5104 
5105 	/* we have mutiple windows, walk through each window */
5106 	for (i = 0; i < hp->dmai_nwin; i++) {
5107 		window = &dma->dp_window[i];
5108 
5109 		/* Go through all the cookies in the window */
5110 		for (j = 0; j < window->wd_cookie_cnt; j++) {
5111 
5112 			start_addr = window->wd_first_cookie[j].dmac_laddress;
5113 			csize = window->wd_first_cookie[j].dmac_size;
5114 
5115 			/*
5116 			 * if we are trimming the first cookie in the window,
5117 			 * and this is the first cookie, adjust the start
5118 			 * address and size of the cookie to account for the
5119 			 * trim.
5120 			 */
5121 			if (window->wd_trim.tr_trim_first && (j == 0)) {
5122 				start_addr = window->wd_trim.tr_first_paddr;
5123 				csize = window->wd_trim.tr_first_size;
5124 			}
5125 
5126 			/*
5127 			 * if we are trimming the last cookie in the window,
5128 			 * and this is the last cookie, adjust the start
5129 			 * address and size of the cookie to account for the
5130 			 * trim.
5131 			 */
5132 			if (window->wd_trim.tr_trim_last &&
5133 			    (j == (window->wd_cookie_cnt - 1))) {
5134 				start_addr = window->wd_trim.tr_last_paddr;
5135 				csize = window->wd_trim.tr_last_size;
5136 			}
5137 
5138 			end_addr = start_addr + csize;
5139 
5140 			/*
5141 			 * if the faulted address is within the physical
5142 			 * address of the cookie, return DDI_FM_NONFATAL.
5143 			 */
5144 			if ((fault_addr >= start_addr) &&
5145 			    (fault_addr <= end_addr)) {
5146 				return (DDI_FM_NONFATAL);
5147 			}
5148 		}
5149 	}
5150 
5151 	/* fault_addr not within this DMA handle */
5152 	return (DDI_FM_UNKNOWN);
5153 }
5154 
5155 /*ARGSUSED*/
5156 static int
5157 rootnex_quiesce(dev_info_t *dip)
5158 {
5159 #if defined(__amd64) && !defined(__xpv)
5160 	return (immu_quiesce());
5161 #else
5162 	return (DDI_SUCCESS);
5163 #endif
5164 }
5165 
5166 #if defined(__xpv)
5167 void
5168 immu_init(void)
5169 {
5170 	;
5171 }
5172 
5173 void
5174 immu_startup(void)
5175 {
5176 	;
5177 }
5178 /*ARGSUSED*/
5179 void
5180 immu_physmem_update(uint64_t addr, uint64_t size)
5181 {
5182 	;
5183 }
5184 #endif
5185