xref: /illumos-gate/usr/src/uts/i86pc/io/rootnex.c (revision 032624d56c174c5c55126582b32e314a6af15522)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License, Version 1.0 only
6  * (the "License").  You may not use this file except in compliance
7  * with the License.
8  *
9  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10  * or http://www.opensolaris.org/os/licensing.
11  * See the License for the specific language governing permissions
12  * and limitations under the License.
13  *
14  * When distributing Covered Code, include this CDDL HEADER in each
15  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16  * If applicable, add the following below this CDDL HEADER, with the
17  * fields enclosed by brackets "[]" replaced with your own identifying
18  * information: Portions Copyright [yyyy] [name of copyright owner]
19  *
20  * CDDL HEADER END
21  */
22 /*
23  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
24  * Use is subject to license terms.
25  */
26 
27 #pragma ident	"%Z%%M%	%I%	%E% SMI"
28 
29 /*
30  * x86 root nexus driver
31  */
32 
33 #include <sys/sysmacros.h>
34 #include <sys/conf.h>
35 #include <sys/autoconf.h>
36 #include <sys/sysmacros.h>
37 #include <sys/debug.h>
38 #include <sys/psw.h>
39 #include <sys/ddidmareq.h>
40 #include <sys/promif.h>
41 #include <sys/devops.h>
42 #include <sys/kmem.h>
43 #include <sys/cmn_err.h>
44 #include <vm/seg.h>
45 #include <vm/seg_kmem.h>
46 #include <vm/seg_dev.h>
47 #include <sys/vmem.h>
48 #include <sys/mman.h>
49 #include <vm/hat.h>
50 #include <vm/as.h>
51 #include <vm/page.h>
52 #include <sys/avintr.h>
53 #include <sys/errno.h>
54 #include <sys/modctl.h>
55 #include <sys/ddi_impldefs.h>
56 #include <sys/sunddi.h>
57 #include <sys/sunndi.h>
58 #include <sys/psm.h>
59 #include <sys/ontrap.h>
60 #include <sys/atomic.h>
61 #include <sys/sdt.h>
62 #include <sys/rootnex.h>
63 #include <vm/hat_i86.h>
64 
65 
66 /*
67  * enable/disable extra checking of function parameters. Useful for debugging
68  * drivers.
69  */
70 #ifdef	DEBUG
71 int rootnex_alloc_check_parms = 1;
72 int rootnex_bind_check_parms = 1;
73 int rootnex_bind_check_inuse = 1;
74 int rootnex_unbind_verify_buffer = 0;
75 int rootnex_sync_check_parms = 1;
76 #else
77 int rootnex_alloc_check_parms = 0;
78 int rootnex_bind_check_parms = 0;
79 int rootnex_bind_check_inuse = 0;
80 int rootnex_unbind_verify_buffer = 0;
81 int rootnex_sync_check_parms = 0;
82 #endif
83 
84 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
85 int rootnex_bind_fail = 1;
86 int rootnex_bind_warn = 1;
87 uint8_t *rootnex_warn_list;
88 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
89 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
90 
91 /*
92  * revert back to old broken behavior of always sync'ing entire copy buffer.
93  * This is useful if be have a buggy driver which doesn't correctly pass in
94  * the offset and size into ddi_dma_sync().
95  */
96 int rootnex_sync_ignore_params = 0;
97 
98 /*
99  * maximum size that we will allow for a copy buffer. Can be patched on the
100  * fly
101  */
102 size_t rootnex_max_copybuf_size = 0x100000;
103 
104 /*
105  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
106  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
107  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
108  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
109  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
110  * (< 8K). We will still need to allocate the copy buffer during bind though
111  * (if we need one). These can only be modified in /etc/system before rootnex
112  * attach.
113  */
114 #if defined(__amd64)
115 int rootnex_prealloc_cookies = 65;
116 int rootnex_prealloc_windows = 4;
117 int rootnex_prealloc_copybuf = 2;
118 #else
119 int rootnex_prealloc_cookies = 33;
120 int rootnex_prealloc_windows = 4;
121 int rootnex_prealloc_copybuf = 2;
122 #endif
123 
124 /* driver global state */
125 static rootnex_state_t *rootnex_state;
126 
127 /* shortcut to rootnex counters */
128 static uint64_t *rootnex_cnt;
129 
130 /*
131  * XXX - does x86 even need these or are they left over from the SPARC days?
132  */
133 /* statically defined integer/boolean properties for the root node */
134 static rootnex_intprop_t rootnex_intprp[] = {
135 	{ "PAGESIZE",			PAGESIZE },
136 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
137 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
138 	{ DDI_RELATIVE_ADDRESSING,	1 },
139 };
140 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
141 
142 
143 static struct cb_ops rootnex_cb_ops = {
144 	nodev,		/* open */
145 	nodev,		/* close */
146 	nodev,		/* strategy */
147 	nodev,		/* print */
148 	nodev,		/* dump */
149 	nodev,		/* read */
150 	nodev,		/* write */
151 	nodev,		/* ioctl */
152 	nodev,		/* devmap */
153 	nodev,		/* mmap */
154 	nodev,		/* segmap */
155 	nochpoll,	/* chpoll */
156 	ddi_prop_op,	/* cb_prop_op */
157 	NULL,		/* struct streamtab */
158 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
159 	CB_REV,		/* Rev */
160 	nodev,		/* cb_aread */
161 	nodev		/* cb_awrite */
162 };
163 
164 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
165     off_t offset, off_t len, caddr_t *vaddrp);
166 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
167     struct hat *hat, struct seg *seg, caddr_t addr,
168     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
169 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip,
170     struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep);
171 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
172     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
173     ddi_dma_handle_t *handlep);
174 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
175     ddi_dma_handle_t handle);
176 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
177     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
178     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
179 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
180     ddi_dma_handle_t handle);
181 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
182     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
183 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
184     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
185     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
186 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
187     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
188     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
189 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
190     ddi_ctl_enum_t ctlop, void *arg, void *result);
191 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
192     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
193 
194 
195 static struct bus_ops rootnex_bus_ops = {
196 	BUSO_REV,
197 	rootnex_map,
198 	NULL,
199 	NULL,
200 	NULL,
201 	rootnex_map_fault,
202 	rootnex_dma_map,
203 	rootnex_dma_allochdl,
204 	rootnex_dma_freehdl,
205 	rootnex_dma_bindhdl,
206 	rootnex_dma_unbindhdl,
207 	rootnex_dma_sync,
208 	rootnex_dma_win,
209 	rootnex_dma_mctl,
210 	rootnex_ctlops,
211 	ddi_bus_prop_op,
212 	i_ddi_rootnex_get_eventcookie,
213 	i_ddi_rootnex_add_eventcall,
214 	i_ddi_rootnex_remove_eventcall,
215 	i_ddi_rootnex_post_event,
216 	0,			/* bus_intr_ctl */
217 	0,			/* bus_config */
218 	0,			/* bus_unconfig */
219 	NULL,			/* bus_fm_init */
220 	NULL,			/* bus_fm_fini */
221 	NULL,			/* bus_fm_access_enter */
222 	NULL,			/* bus_fm_access_exit */
223 	NULL,			/* bus_powr */
224 	rootnex_intr_ops	/* bus_intr_op */
225 };
226 
227 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
228 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
229 
230 static struct dev_ops rootnex_ops = {
231 	DEVO_REV,
232 	0,
233 	ddi_no_info,
234 	nulldev,
235 	nulldev,
236 	rootnex_attach,
237 	rootnex_detach,
238 	nulldev,
239 	&rootnex_cb_ops,
240 	&rootnex_bus_ops
241 };
242 
243 static struct modldrv rootnex_modldrv = {
244 	&mod_driverops,
245 	"i86pc root nexus %I%",
246 	&rootnex_ops
247 };
248 
249 static struct modlinkage rootnex_modlinkage = {
250 	MODREV_1,
251 	(void *)&rootnex_modldrv,
252 	NULL
253 };
254 
255 
256 /*
257  *  extern hacks
258  */
259 extern struct seg_ops segdev_ops;
260 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
261 #ifdef	DDI_MAP_DEBUG
262 extern int ddi_map_debug_flag;
263 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
264 #endif
265 #define	ptob64(x)	(((uint64_t)(x)) << MMU_PAGESHIFT)
266 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
267 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
268 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
269     psm_intr_op_t, int *);
270 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
271 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
272 /*
273  * Use device arena to use for device control register mappings.
274  * Various kernel memory walkers (debugger, dtrace) need to know
275  * to avoid this address range to prevent undesired device activity.
276  */
277 extern void *device_arena_alloc(size_t size, int vm_flag);
278 extern void device_arena_free(void * vaddr, size_t size);
279 
280 
281 /*
282  *  Internal functions
283  */
284 static int rootnex_dma_init();
285 static void rootnex_add_props(dev_info_t *);
286 static int rootnex_ctl_reportdev(dev_info_t *dip);
287 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
288 static int rootnex_xlate_intrs(dev_info_t *dip, dev_info_t *rdip, int *in,
289     struct ddi_parent_private_data *pdptr);
290 static int rootnex_ctlops_poke(peekpoke_ctlops_t *in_args);
291 static int rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result);
292 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
293 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
294 static int rootnex_map_handle(ddi_map_req_t *mp);
295 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
296 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
297 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
298     ddi_dma_attr_t *attr);
299 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
300     rootnex_sglinfo_t *sglinfo);
301 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
302     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag);
303 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
304     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
305 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
306 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
307     ddi_dma_attr_t *attr, int kmflag);
308 static void rootnex_teardown_windows(rootnex_dma_t *dma);
309 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
310     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
311 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
312     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
313     size_t *copybuf_used, page_t **cur_pp);
314 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
315     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
316     ddi_dma_attr_t *attr, off_t cur_offset);
317 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
318     rootnex_dma_t *dma, rootnex_window_t **windowp,
319     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
320 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
321     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
322 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
323     off_t offset, size_t size, uint_t cache_flags);
324 static int rootnex_verify_buffer(rootnex_dma_t *dma);
325 
326 
327 /*
328  * _init()
329  *
330  */
331 int
332 _init(void)
333 {
334 
335 	rootnex_state = NULL;
336 	return (mod_install(&rootnex_modlinkage));
337 }
338 
339 
340 /*
341  * _info()
342  *
343  */
344 int
345 _info(struct modinfo *modinfop)
346 {
347 	return (mod_info(&rootnex_modlinkage, modinfop));
348 }
349 
350 
351 /*
352  * _fini()
353  *
354  */
355 int
356 _fini(void)
357 {
358 	return (EBUSY);
359 }
360 
361 
362 /*
363  * rootnex_attach()
364  *
365  */
366 static int
367 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
368 {
369 	int e;
370 
371 
372 	switch (cmd) {
373 	case DDI_ATTACH:
374 		break;
375 	case DDI_RESUME:
376 		return (DDI_SUCCESS);
377 	default:
378 		return (DDI_FAILURE);
379 	}
380 
381 	/*
382 	 * We should only have one instance of rootnex. Save it away since we
383 	 * don't have an easy way to get it back later.
384 	 */
385 	ASSERT(rootnex_state == NULL);
386 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
387 
388 	rootnex_state->r_dip = dip;
389 	rootnex_state->r_reserved_msg_printed = B_FALSE;
390 	rootnex_cnt = &rootnex_state->r_counters[0];
391 
392 	mutex_init(&rootnex_state->r_peekpoke_mutex, NULL, MUTEX_SPIN,
393 	    (void *)ipltospl(15));
394 
395 	/* initialize DMA related state */
396 	e = rootnex_dma_init();
397 	if (e != DDI_SUCCESS) {
398 		mutex_destroy(&rootnex_state->r_peekpoke_mutex);
399 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
400 		return (DDI_FAILURE);
401 	}
402 
403 	/* Add static root node properties */
404 	rootnex_add_props(dip);
405 
406 	/* since we can't call ddi_report_dev() */
407 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
408 
409 	/* Initialize rootnex event handle */
410 	i_ddi_rootnex_init_events(dip);
411 
412 	return (DDI_SUCCESS);
413 }
414 
415 
416 /*
417  * rootnex_detach()
418  *
419  */
420 /*ARGSUSED*/
421 static int
422 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
423 {
424 	switch (cmd) {
425 	case DDI_SUSPEND:
426 		break;
427 	default:
428 		return (DDI_FAILURE);
429 	}
430 
431 	return (DDI_SUCCESS);
432 }
433 
434 
435 /*
436  * rootnex_dma_init()
437  *
438  */
439 /*ARGSUSED*/
440 static int
441 rootnex_dma_init()
442 {
443 	size_t bufsize;
444 
445 
446 	/*
447 	 * size of our cookie/window/copybuf state needed in dma bind that we
448 	 * pre-alloc in dma_alloc_handle
449 	 */
450 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
451 	rootnex_state->r_prealloc_size =
452 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
453 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
454 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
455 
456 	/*
457 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
458 	 * allocate 16 extra bytes for struct pointer alignment
459 	 * (p->dmai_private & dma->dp_prealloc_buffer)
460 	 */
461 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
462 	    rootnex_state->r_prealloc_size + 0x10;
463 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
464 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
465 	if (rootnex_state->r_dmahdl_cache == NULL) {
466 		return (DDI_FAILURE);
467 	}
468 
469 	/*
470 	 * allocate array to track which major numbers we have printed warnings
471 	 * for.
472 	 */
473 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
474 	    KM_SLEEP);
475 
476 	return (DDI_SUCCESS);
477 }
478 
479 
480 /*
481  * rootnex_add_props()
482  *
483  */
484 static void
485 rootnex_add_props(dev_info_t *dip)
486 {
487 	rootnex_intprop_t *rpp;
488 	int i;
489 
490 	/* Add static integer/boolean properties to the root node */
491 	rpp = rootnex_intprp;
492 	for (i = 0; i < NROOT_INTPROPS; i++) {
493 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
494 		    rpp[i].prop_name, rpp[i].prop_value);
495 	}
496 }
497 
498 
499 
500 /*
501  * *************************
502  *  ctlops related routines
503  * *************************
504  */
505 
506 /*
507  * rootnex_ctlops()
508  *
509  */
510 static int
511 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
512     void *arg, void *result)
513 {
514 	int n, *ptr;
515 	struct ddi_parent_private_data *pdp;
516 
517 
518 	switch (ctlop) {
519 	case DDI_CTLOPS_DMAPMAPC:
520 		/*
521 		 * Return 'partial' to indicate that dma mapping
522 		 * has to be done in the main MMU.
523 		 */
524 		return (DDI_DMA_PARTIAL);
525 
526 	case DDI_CTLOPS_BTOP:
527 		/*
528 		 * Convert byte count input to physical page units.
529 		 * (byte counts that are not a page-size multiple
530 		 * are rounded down)
531 		 */
532 		*(ulong_t *)result = btop(*(ulong_t *)arg);
533 		return (DDI_SUCCESS);
534 
535 	case DDI_CTLOPS_PTOB:
536 		/*
537 		 * Convert size in physical pages to bytes
538 		 */
539 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
540 		return (DDI_SUCCESS);
541 
542 	case DDI_CTLOPS_BTOPR:
543 		/*
544 		 * Convert byte count input to physical page units
545 		 * (byte counts that are not a page-size multiple
546 		 * are rounded up)
547 		 */
548 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
549 		return (DDI_SUCCESS);
550 
551 	case DDI_CTLOPS_POKE:
552 		return (rootnex_ctlops_poke((peekpoke_ctlops_t *)arg));
553 
554 	case DDI_CTLOPS_PEEK:
555 		return (rootnex_ctlops_peek((peekpoke_ctlops_t *)arg, result));
556 
557 	case DDI_CTLOPS_INITCHILD:
558 		return (impl_ddi_sunbus_initchild(arg));
559 
560 	case DDI_CTLOPS_UNINITCHILD:
561 		impl_ddi_sunbus_removechild(arg);
562 		return (DDI_SUCCESS);
563 
564 	case DDI_CTLOPS_REPORTDEV:
565 		return (rootnex_ctl_reportdev(rdip));
566 
567 	case DDI_CTLOPS_IOMIN:
568 		/*
569 		 * Nothing to do here but reflect back..
570 		 */
571 		return (DDI_SUCCESS);
572 
573 	case DDI_CTLOPS_REGSIZE:
574 	case DDI_CTLOPS_NREGS:
575 	case DDI_CTLOPS_NINTRS:
576 		break;
577 
578 	case DDI_CTLOPS_SIDDEV:
579 		if (ndi_dev_is_prom_node(rdip))
580 			return (DDI_SUCCESS);
581 		if (ndi_dev_is_persistent_node(rdip))
582 			return (DDI_SUCCESS);
583 		return (DDI_FAILURE);
584 
585 	case DDI_CTLOPS_INTR_HILEVEL:
586 		/*
587 		 * Indicate whether the interrupt specified is to be handled
588 		 * above lock level.  In other words, above the level that
589 		 * cv_signal and default type mutexes can be used.
590 		 */
591 		*(int *)result =
592 		    (INT_IPL(((struct intrspec *)arg)->intrspec_pri)
593 		    > LOCK_LEVEL);
594 		return (DDI_SUCCESS);
595 
596 	case DDI_CTLOPS_XLATE_INTRS:
597 		return (rootnex_xlate_intrs(dip, rdip, arg, result));
598 
599 	case DDI_CTLOPS_POWER:
600 		return ((*pm_platform_power)((power_req_t *)arg));
601 
602 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
603 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
604 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
605 		if (!rootnex_state->r_reserved_msg_printed) {
606 			rootnex_state->r_reserved_msg_printed = B_TRUE;
607 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
608 			    "1 or more reserved/obsolete operations.");
609 		}
610 		return (DDI_FAILURE);
611 
612 	default:
613 		return (DDI_FAILURE);
614 	}
615 	/*
616 	 * The rest are for "hardware" properties
617 	 */
618 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
619 		return (DDI_FAILURE);
620 
621 	if (ctlop == DDI_CTLOPS_NREGS) {
622 		ptr = (int *)result;
623 		*ptr = pdp->par_nreg;
624 	} else if (ctlop == DDI_CTLOPS_NINTRS) {
625 		ptr = (int *)result;
626 		*ptr = pdp->par_nintr;
627 	} else {
628 		off_t *size = (off_t *)result;
629 
630 		ptr = (int *)arg;
631 		n = *ptr;
632 		if (n >= pdp->par_nreg) {
633 			return (DDI_FAILURE);
634 		}
635 		*size = (off_t)pdp->par_reg[n].regspec_size;
636 	}
637 	return (DDI_SUCCESS);
638 }
639 
640 
641 /*
642  * rootnex_ctl_reportdev()
643  *
644  */
645 static int
646 rootnex_ctl_reportdev(dev_info_t *dev)
647 {
648 	int i, n, len, f_len = 0;
649 	char *buf;
650 
651 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
652 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
653 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
654 	len = strlen(buf);
655 
656 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
657 
658 		struct regspec *rp = sparc_pd_getreg(dev, i);
659 
660 		if (i == 0)
661 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
662 			    ": ");
663 		else
664 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
665 			    " and ");
666 		len = strlen(buf);
667 
668 		switch (rp->regspec_bustype) {
669 
670 		case BTEISA:
671 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
672 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
673 			break;
674 
675 		case BTISA:
676 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
677 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
678 			break;
679 
680 		default:
681 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
682 			    "space %x offset %x",
683 			    rp->regspec_bustype, rp->regspec_addr);
684 			break;
685 		}
686 		len = strlen(buf);
687 	}
688 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
689 		int pri;
690 
691 		if (i != 0) {
692 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
693 			    ",");
694 			len = strlen(buf);
695 		}
696 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
697 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
698 		    " sparc ipl %d", pri);
699 		len = strlen(buf);
700 	}
701 #ifdef DEBUG
702 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
703 		cmn_err(CE_NOTE, "next message is truncated: "
704 		    "printed length 1024, real length %d", f_len);
705 	}
706 #endif /* DEBUG */
707 	cmn_err(CE_CONT, "?%s\n", buf);
708 	kmem_free(buf, REPORTDEV_BUFSIZE);
709 	return (DDI_SUCCESS);
710 }
711 
712 
713 /*
714  * rootnex_ctlops_poke()
715  *
716  */
717 static int
718 rootnex_ctlops_poke(peekpoke_ctlops_t *in_args)
719 {
720 	int err = DDI_SUCCESS;
721 	on_trap_data_t otd;
722 
723 	/* Cautious access not supported. */
724 	if (in_args->handle != NULL)
725 		return (DDI_FAILURE);
726 
727 	mutex_enter(&rootnex_state->r_peekpoke_mutex);
728 
729 	/* Set up protected environment. */
730 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
731 		switch (in_args->size) {
732 		case sizeof (uint8_t):
733 			*(uint8_t *)in_args->dev_addr = *(uint8_t *)
734 			    in_args->host_addr;
735 			break;
736 
737 		case sizeof (uint16_t):
738 			*(uint16_t *)in_args->dev_addr =
739 			    *(uint16_t *)in_args->host_addr;
740 			break;
741 
742 		case sizeof (uint32_t):
743 			*(uint32_t *)in_args->dev_addr =
744 			    *(uint32_t *)in_args->host_addr;
745 			break;
746 
747 		case sizeof (uint64_t):
748 			*(uint64_t *)in_args->dev_addr =
749 			    *(uint64_t *)in_args->host_addr;
750 			break;
751 
752 		default:
753 			err = DDI_FAILURE;
754 			break;
755 		}
756 	} else
757 		err = DDI_FAILURE;
758 
759 	/* Take down protected environment. */
760 	no_trap();
761 	mutex_exit(&rootnex_state->r_peekpoke_mutex);
762 
763 	return (err);
764 }
765 
766 
767 /*
768  * rootnex_ctlops_peek()
769  *
770  */
771 static int
772 rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result)
773 {
774 	int err = DDI_SUCCESS;
775 	on_trap_data_t otd;
776 
777 	/* Cautious access not supported. */
778 	if (in_args->handle != NULL)
779 		return (DDI_FAILURE);
780 
781 	mutex_enter(&rootnex_state->r_peekpoke_mutex);
782 
783 	if (!on_trap(&otd, OT_DATA_ACCESS)) {
784 		switch (in_args->size) {
785 		case sizeof (uint8_t):
786 			*(uint8_t *)in_args->host_addr =
787 			    *(uint8_t *)in_args->dev_addr;
788 			break;
789 
790 		case sizeof (uint16_t):
791 			*(uint16_t *)in_args->host_addr =
792 			    *(uint16_t *)in_args->dev_addr;
793 			break;
794 
795 		case sizeof (uint32_t):
796 			*(uint32_t *)in_args->host_addr =
797 			    *(uint32_t *)in_args->dev_addr;
798 			break;
799 
800 		case sizeof (uint64_t):
801 			*(uint64_t *)in_args->host_addr =
802 			    *(uint64_t *)in_args->dev_addr;
803 			break;
804 
805 		default:
806 			err = DDI_FAILURE;
807 			break;
808 		}
809 		result = (void *)in_args->host_addr;
810 	} else
811 		err = DDI_FAILURE;
812 
813 	no_trap();
814 	mutex_exit(&rootnex_state->r_peekpoke_mutex);
815 
816 	return (err);
817 }
818 
819 
820 
821 /*
822  * ******************
823  *  map related code
824  * ******************
825  */
826 
827 /*
828  * rootnex_map()
829  *
830  */
831 static int
832 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
833     off_t len, caddr_t *vaddrp)
834 {
835 	struct regspec *rp, tmp_reg;
836 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
837 	int error;
838 
839 	mp = &mr;
840 
841 	switch (mp->map_op)  {
842 	case DDI_MO_MAP_LOCKED:
843 	case DDI_MO_UNMAP:
844 	case DDI_MO_MAP_HANDLE:
845 		break;
846 	default:
847 #ifdef	DDI_MAP_DEBUG
848 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
849 		    mp->map_op);
850 #endif	/* DDI_MAP_DEBUG */
851 		return (DDI_ME_UNIMPLEMENTED);
852 	}
853 
854 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
855 #ifdef	DDI_MAP_DEBUG
856 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
857 #endif	/* DDI_MAP_DEBUG */
858 		return (DDI_ME_UNIMPLEMENTED);
859 	}
860 
861 	/*
862 	 * First, if given an rnumber, convert it to a regspec...
863 	 * (Presumably, this is on behalf of a child of the root node?)
864 	 */
865 
866 	if (mp->map_type == DDI_MT_RNUMBER)  {
867 
868 		int rnumber = mp->map_obj.rnumber;
869 #ifdef	DDI_MAP_DEBUG
870 		static char *out_of_range =
871 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
872 #endif	/* DDI_MAP_DEBUG */
873 
874 		rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
875 		if (rp == NULL)  {
876 #ifdef	DDI_MAP_DEBUG
877 			cmn_err(CE_WARN, out_of_range, rnumber,
878 			    ddi_get_name(rdip));
879 #endif	/* DDI_MAP_DEBUG */
880 			return (DDI_ME_RNUMBER_RANGE);
881 		}
882 
883 		/*
884 		 * Convert the given ddi_map_req_t from rnumber to regspec...
885 		 */
886 
887 		mp->map_type = DDI_MT_REGSPEC;
888 		mp->map_obj.rp = rp;
889 	}
890 
891 	/*
892 	 * Adjust offset and length correspnding to called values...
893 	 * XXX: A non-zero length means override the one in the regspec
894 	 * XXX: (regardless of what's in the parent's range?)
895 	 */
896 
897 	tmp_reg = *(mp->map_obj.rp);		/* Preserve underlying data */
898 	rp = mp->map_obj.rp = &tmp_reg;		/* Use tmp_reg in request */
899 
900 #ifdef	DDI_MAP_DEBUG
901 	cmn_err(CE_CONT,
902 		"rootnex: <%s,%s> <0x%x, 0x%x, 0x%d>"
903 		" offset %d len %d handle 0x%x\n",
904 		ddi_get_name(dip), ddi_get_name(rdip),
905 		rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
906 		offset, len, mp->map_handlep);
907 #endif	/* DDI_MAP_DEBUG */
908 
909 	/*
910 	 * I/O or memory mapping:
911 	 *
912 	 *	<bustype=0, addr=x, len=x>: memory
913 	 *	<bustype=1, addr=x, len=x>: i/o
914 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
915 	 */
916 
917 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
918 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
919 		    " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
920 		    ddi_get_name(rdip), rp->regspec_bustype,
921 		    rp->regspec_addr, rp->regspec_size);
922 		return (DDI_ME_INVAL);
923 	}
924 
925 	if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
926 		/*
927 		 * compatibility i/o mapping
928 		 */
929 		rp->regspec_bustype += (uint_t)offset;
930 	} else {
931 		/*
932 		 * Normal memory or i/o mapping
933 		 */
934 		rp->regspec_addr += (uint_t)offset;
935 	}
936 
937 	if (len != 0)
938 		rp->regspec_size = (uint_t)len;
939 
940 #ifdef	DDI_MAP_DEBUG
941 	cmn_err(CE_CONT,
942 		"             <%s,%s> <0x%x, 0x%x, 0x%d>"
943 		" offset %d len %d handle 0x%x\n",
944 		ddi_get_name(dip), ddi_get_name(rdip),
945 		rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
946 		offset, len, mp->map_handlep);
947 #endif	/* DDI_MAP_DEBUG */
948 
949 	/*
950 	 * Apply any parent ranges at this level, if applicable.
951 	 * (This is where nexus specific regspec translation takes place.
952 	 * Use of this function is implicit agreement that translation is
953 	 * provided via ddi_apply_range.)
954 	 */
955 
956 #ifdef	DDI_MAP_DEBUG
957 	ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
958 	    ddi_get_name(dip), ddi_get_name(rdip));
959 #endif	/* DDI_MAP_DEBUG */
960 
961 	if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
962 		return (error);
963 
964 	switch (mp->map_op)  {
965 	case DDI_MO_MAP_LOCKED:
966 
967 		/*
968 		 * Set up the locked down kernel mapping to the regspec...
969 		 */
970 
971 		return (rootnex_map_regspec(mp, vaddrp));
972 
973 	case DDI_MO_UNMAP:
974 
975 		/*
976 		 * Release mapping...
977 		 */
978 
979 		return (rootnex_unmap_regspec(mp, vaddrp));
980 
981 	case DDI_MO_MAP_HANDLE:
982 
983 		return (rootnex_map_handle(mp));
984 
985 	default:
986 		return (DDI_ME_UNIMPLEMENTED);
987 	}
988 }
989 
990 
991 /*
992  * rootnex_map_fault()
993  *
994  *	fault in mappings for requestors
995  */
996 /*ARGSUSED*/
997 static int
998 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
999     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
1000     uint_t lock)
1001 {
1002 
1003 #ifdef	DDI_MAP_DEBUG
1004 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
1005 	ddi_map_debug(" Seg <%s>\n",
1006 	    seg->s_ops == &segdev_ops ? "segdev" :
1007 	    seg == &kvseg ? "segkmem" : "NONE!");
1008 #endif	/* DDI_MAP_DEBUG */
1009 
1010 	/*
1011 	 * This is all terribly broken, but it is a start
1012 	 *
1013 	 * XXX	Note that this test means that segdev_ops
1014 	 *	must be exported from seg_dev.c.
1015 	 * XXX	What about devices with their own segment drivers?
1016 	 */
1017 	if (seg->s_ops == &segdev_ops) {
1018 		struct segdev_data *sdp =
1019 			(struct segdev_data *)seg->s_data;
1020 
1021 		if (hat == NULL) {
1022 			/*
1023 			 * This is one plausible interpretation of
1024 			 * a null hat i.e. use the first hat on the
1025 			 * address space hat list which by convention is
1026 			 * the hat of the system MMU.  At alternative
1027 			 * would be to panic .. this might well be better ..
1028 			 */
1029 			ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock));
1030 			hat = seg->s_as->a_hat;
1031 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
1032 		}
1033 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
1034 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
1035 	} else if (seg == &kvseg && dp == NULL) {
1036 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
1037 		    HAT_LOAD_LOCK);
1038 	} else
1039 		return (DDI_FAILURE);
1040 	return (DDI_SUCCESS);
1041 }
1042 
1043 
1044 /*
1045  * rootnex_map_regspec()
1046  *     we don't support mapping of I/O cards above 4Gb
1047  */
1048 static int
1049 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1050 {
1051 	ulong_t base;
1052 	void *cvaddr;
1053 	uint_t npages, pgoffset;
1054 	struct regspec *rp;
1055 	ddi_acc_hdl_t *hp;
1056 	ddi_acc_impl_t *ap;
1057 	uint_t	hat_acc_flags;
1058 
1059 	rp = mp->map_obj.rp;
1060 	hp = mp->map_handlep;
1061 
1062 #ifdef	DDI_MAP_DEBUG
1063 	ddi_map_debug(
1064 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
1065 	    rp->regspec_bustype, rp->regspec_addr,
1066 	    rp->regspec_size, mp->map_handlep);
1067 #endif	/* DDI_MAP_DEBUG */
1068 
1069 	/*
1070 	 * I/O or memory mapping
1071 	 *
1072 	 *	<bustype=0, addr=x, len=x>: memory
1073 	 *	<bustype=1, addr=x, len=x>: i/o
1074 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1075 	 */
1076 
1077 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
1078 		cmn_err(CE_WARN, "rootnex: invalid register spec"
1079 		    " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
1080 		    rp->regspec_addr, rp->regspec_size);
1081 		return (DDI_FAILURE);
1082 	}
1083 
1084 	if (rp->regspec_bustype != 0) {
1085 		/*
1086 		 * I/O space - needs a handle.
1087 		 */
1088 		if (hp == NULL) {
1089 			return (DDI_FAILURE);
1090 		}
1091 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1092 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
1093 		impl_acc_hdl_init(hp);
1094 
1095 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1096 #ifdef  DDI_MAP_DEBUG
1097 			ddi_map_debug("rootnex_map_regspec: mmap() \
1098 to I/O space is not supported.\n");
1099 #endif  /* DDI_MAP_DEBUG */
1100 			return (DDI_ME_INVAL);
1101 		} else {
1102 			/*
1103 			 * 1275-compliant vs. compatibility i/o mapping
1104 			 */
1105 			*vaddrp =
1106 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
1107 				((caddr_t)(uintptr_t)rp->regspec_bustype) :
1108 				((caddr_t)(uintptr_t)rp->regspec_addr);
1109 		}
1110 
1111 #ifdef	DDI_MAP_DEBUG
1112 		ddi_map_debug(
1113 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1114 		    rp->regspec_size, *vaddrp);
1115 #endif	/* DDI_MAP_DEBUG */
1116 		return (DDI_SUCCESS);
1117 	}
1118 
1119 	/*
1120 	 * Memory space
1121 	 */
1122 
1123 	if (hp != NULL) {
1124 		/*
1125 		 * hat layer ignores
1126 		 * hp->ah_acc.devacc_attr_endian_flags.
1127 		 */
1128 		switch (hp->ah_acc.devacc_attr_dataorder) {
1129 		case DDI_STRICTORDER_ACC:
1130 			hat_acc_flags = HAT_STRICTORDER;
1131 			break;
1132 		case DDI_UNORDERED_OK_ACC:
1133 			hat_acc_flags = HAT_UNORDERED_OK;
1134 			break;
1135 		case DDI_MERGING_OK_ACC:
1136 			hat_acc_flags = HAT_MERGING_OK;
1137 			break;
1138 		case DDI_LOADCACHING_OK_ACC:
1139 			hat_acc_flags = HAT_LOADCACHING_OK;
1140 			break;
1141 		case DDI_STORECACHING_OK_ACC:
1142 			hat_acc_flags = HAT_STORECACHING_OK;
1143 			break;
1144 		}
1145 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1146 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1147 		impl_acc_hdl_init(hp);
1148 		hp->ah_hat_flags = hat_acc_flags;
1149 	} else {
1150 		hat_acc_flags = HAT_STRICTORDER;
1151 	}
1152 
1153 	base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */
1154 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */
1155 
1156 	if (rp->regspec_size == 0) {
1157 #ifdef  DDI_MAP_DEBUG
1158 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1159 #endif  /* DDI_MAP_DEBUG */
1160 		return (DDI_ME_INVAL);
1161 	}
1162 
1163 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1164 		*vaddrp = (caddr_t)mmu_btop(base);
1165 	} else {
1166 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1167 
1168 #ifdef	DDI_MAP_DEBUG
1169 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages \
1170 physical %x ",
1171 		    npages, base);
1172 #endif	/* DDI_MAP_DEBUG */
1173 
1174 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1175 		if (cvaddr == NULL)
1176 			return (DDI_ME_NORESOURCES);
1177 
1178 		/*
1179 		 * Now map in the pages we've allocated...
1180 		 */
1181 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base),
1182 		    mp->map_prot | hat_acc_flags, HAT_LOAD_LOCK);
1183 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1184 	}
1185 
1186 #ifdef	DDI_MAP_DEBUG
1187 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1188 #endif	/* DDI_MAP_DEBUG */
1189 	return (DDI_SUCCESS);
1190 }
1191 
1192 
1193 /*
1194  * rootnex_unmap_regspec()
1195  *
1196  */
1197 static int
1198 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1199 {
1200 	caddr_t addr = (caddr_t)*vaddrp;
1201 	uint_t npages, pgoffset;
1202 	struct regspec *rp;
1203 
1204 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1205 		return (0);
1206 
1207 	rp = mp->map_obj.rp;
1208 
1209 	if (rp->regspec_size == 0) {
1210 #ifdef  DDI_MAP_DEBUG
1211 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1212 #endif  /* DDI_MAP_DEBUG */
1213 		return (DDI_ME_INVAL);
1214 	}
1215 
1216 	/*
1217 	 * I/O or memory mapping:
1218 	 *
1219 	 *	<bustype=0, addr=x, len=x>: memory
1220 	 *	<bustype=1, addr=x, len=x>: i/o
1221 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1222 	 */
1223 	if (rp->regspec_bustype != 0) {
1224 		/*
1225 		 * This is I/O space, which requires no particular
1226 		 * processing on unmap since it isn't mapped in the
1227 		 * first place.
1228 		 */
1229 		return (DDI_SUCCESS);
1230 	}
1231 
1232 	/*
1233 	 * Memory space
1234 	 */
1235 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1236 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1237 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1238 	device_arena_free(addr - pgoffset, ptob(npages));
1239 
1240 	/*
1241 	 * Destroy the pointer - the mapping has logically gone
1242 	 */
1243 	*vaddrp = NULL;
1244 
1245 	return (DDI_SUCCESS);
1246 }
1247 
1248 
1249 /*
1250  * rootnex_map_handle()
1251  *
1252  */
1253 static int
1254 rootnex_map_handle(ddi_map_req_t *mp)
1255 {
1256 	ddi_acc_hdl_t *hp;
1257 	ulong_t base;
1258 	uint_t pgoffset;
1259 	struct regspec *rp;
1260 
1261 	rp = mp->map_obj.rp;
1262 
1263 #ifdef	DDI_MAP_DEBUG
1264 	ddi_map_debug(
1265 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1266 	    rp->regspec_bustype, rp->regspec_addr,
1267 	    rp->regspec_size, mp->map_handlep);
1268 #endif	/* DDI_MAP_DEBUG */
1269 
1270 	/*
1271 	 * I/O or memory mapping:
1272 	 *
1273 	 *	<bustype=0, addr=x, len=x>: memory
1274 	 *	<bustype=1, addr=x, len=x>: i/o
1275 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1276 	 */
1277 	if (rp->regspec_bustype != 0) {
1278 		/*
1279 		 * This refers to I/O space, and we don't support "mapping"
1280 		 * I/O space to a user.
1281 		 */
1282 		return (DDI_FAILURE);
1283 	}
1284 
1285 	/*
1286 	 * Set up the hat_flags for the mapping.
1287 	 */
1288 	hp = mp->map_handlep;
1289 
1290 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1291 	case DDI_NEVERSWAP_ACC:
1292 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1293 		break;
1294 	case DDI_STRUCTURE_LE_ACC:
1295 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1296 		break;
1297 	case DDI_STRUCTURE_BE_ACC:
1298 		return (DDI_FAILURE);
1299 	default:
1300 		return (DDI_REGS_ACC_CONFLICT);
1301 	}
1302 
1303 	switch (hp->ah_acc.devacc_attr_dataorder) {
1304 	case DDI_STRICTORDER_ACC:
1305 		break;
1306 	case DDI_UNORDERED_OK_ACC:
1307 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1308 		break;
1309 	case DDI_MERGING_OK_ACC:
1310 		hp->ah_hat_flags |= HAT_MERGING_OK;
1311 		break;
1312 	case DDI_LOADCACHING_OK_ACC:
1313 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1314 		break;
1315 	case DDI_STORECACHING_OK_ACC:
1316 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1317 		break;
1318 	default:
1319 		return (DDI_FAILURE);
1320 	}
1321 
1322 	base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */
1323 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */
1324 
1325 	if (rp->regspec_size == 0)
1326 		return (DDI_ME_INVAL);
1327 
1328 	hp->ah_pfn = mmu_btop(base);
1329 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1330 
1331 	return (DDI_SUCCESS);
1332 }
1333 
1334 
1335 
1336 /*
1337  * ************************
1338  *  interrupt related code
1339  * ************************
1340  */
1341 
1342 /*
1343  * rootnex_intr_ops()
1344  *	bus_intr_op() function for interrupt support
1345  */
1346 /* ARGSUSED */
1347 static int
1348 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1349     ddi_intr_handle_impl_t *hdlp, void *result)
1350 {
1351 	struct intrspec			*ispec;
1352 	struct ddi_parent_private_data	*pdp;
1353 
1354 	DDI_INTR_NEXDBG((CE_CONT,
1355 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1356 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1357 
1358 	/* Process the interrupt operation */
1359 	switch (intr_op) {
1360 	case DDI_INTROP_GETCAP:
1361 		/* First check with pcplusmp */
1362 		if (psm_intr_ops == NULL)
1363 			return (DDI_FAILURE);
1364 
1365 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1366 			*(int *)result = 0;
1367 			return (DDI_FAILURE);
1368 		}
1369 		break;
1370 	case DDI_INTROP_SETCAP:
1371 		if (psm_intr_ops == NULL)
1372 			return (DDI_FAILURE);
1373 
1374 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1375 			return (DDI_FAILURE);
1376 		break;
1377 	case DDI_INTROP_ALLOC:
1378 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1379 			return (DDI_FAILURE);
1380 		hdlp->ih_pri = ispec->intrspec_pri;
1381 		*(int *)result = hdlp->ih_scratch1;
1382 		break;
1383 	case DDI_INTROP_FREE:
1384 		pdp = ddi_get_parent_data(rdip);
1385 		/*
1386 		 * Special case for 'pcic' driver' only.
1387 		 * If an intrspec was created for it, clean it up here
1388 		 * See detailed comments on this in the function
1389 		 * rootnex_get_ispec().
1390 		 */
1391 		if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1392 			kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1393 			    pdp->par_nintr);
1394 			/*
1395 			 * Set it to zero; so that
1396 			 * DDI framework doesn't free it again
1397 			 */
1398 			pdp->par_intr = NULL;
1399 			pdp->par_nintr = 0;
1400 		}
1401 		break;
1402 	case DDI_INTROP_GETPRI:
1403 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1404 			return (DDI_FAILURE);
1405 		*(int *)result = ispec->intrspec_pri;
1406 		break;
1407 	case DDI_INTROP_SETPRI:
1408 		/* Validate the interrupt priority passed to us */
1409 		if (*(int *)result > LOCK_LEVEL)
1410 			return (DDI_FAILURE);
1411 
1412 		/* Ensure that PSM is all initialized and ispec is ok */
1413 		if ((psm_intr_ops == NULL) ||
1414 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1415 			return (DDI_FAILURE);
1416 
1417 		/* Change the priority */
1418 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1419 		    PSM_FAILURE)
1420 			return (DDI_FAILURE);
1421 
1422 		/* update the ispec with the new priority */
1423 		ispec->intrspec_pri =  *(int *)result;
1424 		break;
1425 	case DDI_INTROP_ADDISR:
1426 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1427 			return (DDI_FAILURE);
1428 		ispec->intrspec_func = hdlp->ih_cb_func;
1429 		break;
1430 	case DDI_INTROP_REMISR:
1431 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1432 			return (DDI_FAILURE);
1433 		ispec->intrspec_func = (uint_t (*)()) 0;
1434 		break;
1435 	case DDI_INTROP_ENABLE:
1436 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1437 			return (DDI_FAILURE);
1438 
1439 		/* Call psmi to translate irq with the dip */
1440 		if (psm_intr_ops == NULL)
1441 			return (DDI_FAILURE);
1442 
1443 		hdlp->ih_private = (void *)ispec;
1444 		(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1445 		    (int *)&hdlp->ih_vector);
1446 
1447 		/* Add the interrupt handler */
1448 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1449 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1450 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, rdip))
1451 			return (DDI_FAILURE);
1452 		break;
1453 	case DDI_INTROP_DISABLE:
1454 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1455 			return (DDI_FAILURE);
1456 
1457 		/* Call psm_ops() to translate irq with the dip */
1458 		if (psm_intr_ops == NULL)
1459 			return (DDI_FAILURE);
1460 
1461 		hdlp->ih_private = (void *)ispec;
1462 		(void) (*psm_intr_ops)(rdip, hdlp,
1463 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1464 
1465 		/* Remove the interrupt handler */
1466 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1467 		    hdlp->ih_cb_func, hdlp->ih_vector);
1468 		break;
1469 	case DDI_INTROP_SETMASK:
1470 		if (psm_intr_ops == NULL)
1471 			return (DDI_FAILURE);
1472 
1473 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1474 			return (DDI_FAILURE);
1475 		break;
1476 	case DDI_INTROP_CLRMASK:
1477 		if (psm_intr_ops == NULL)
1478 			return (DDI_FAILURE);
1479 
1480 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1481 			return (DDI_FAILURE);
1482 		break;
1483 	case DDI_INTROP_GETPENDING:
1484 		if (psm_intr_ops == NULL)
1485 			return (DDI_FAILURE);
1486 
1487 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1488 		    result)) {
1489 			*(int *)result = 0;
1490 			return (DDI_FAILURE);
1491 		}
1492 		break;
1493 	case DDI_INTROP_NINTRS:
1494 		if ((pdp = ddi_get_parent_data(rdip)) == NULL)
1495 			return (DDI_FAILURE);
1496 		*(int *)result = pdp->par_nintr;
1497 		if (pdp->par_nintr == 0) {
1498 			/*
1499 			 * Special case for 'pcic' driver' only. This driver
1500 			 * driver is a child of 'isa' and 'rootnex' drivers.
1501 			 *
1502 			 * See detailed comments on this in the function
1503 			 * rootnex_get_ispec().
1504 			 *
1505 			 * Children of 'pcic' send 'NINITR' request all the
1506 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1507 			 * field may not initialized. So, we fake it here
1508 			 * to return 1 (a la what PCMCIA nexus does).
1509 			 */
1510 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1511 				*(int *)result = 1;
1512 		}
1513 		break;
1514 	case DDI_INTROP_SUPPORTED_TYPES:
1515 		*(int *)result = 0;
1516 		*(int *)result |= DDI_INTR_TYPE_FIXED;	/* Always ... */
1517 		break;
1518 	case DDI_INTROP_NAVAIL:
1519 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1520 			return (DDI_FAILURE);
1521 
1522 		if (psm_intr_ops == NULL) {
1523 			*(int *)result = 1;
1524 			break;
1525 		}
1526 
1527 		/* Priority in the handle not initialized yet */
1528 		hdlp->ih_pri = ispec->intrspec_pri;
1529 		(void) (*psm_intr_ops)(rdip, hdlp,
1530 		    PSM_INTR_OP_NAVAIL_VECTORS, result);
1531 		break;
1532 	default:
1533 		return (DDI_FAILURE);
1534 	}
1535 
1536 	return (DDI_SUCCESS);
1537 }
1538 
1539 
1540 /*
1541  * rootnex_get_ispec()
1542  *	convert an interrupt number to an interrupt specification.
1543  *	The interrupt number determines which interrupt spec will be
1544  *	returned if more than one exists.
1545  *
1546  *	Look into the parent private data area of the 'rdip' to find out
1547  *	the interrupt specification.  First check to make sure there is
1548  *	one that matchs "inumber" and then return a pointer to it.
1549  *
1550  *	Return NULL if one could not be found.
1551  *
1552  *	NOTE: This is needed for rootnex_intr_ops()
1553  */
1554 static struct intrspec *
1555 rootnex_get_ispec(dev_info_t *rdip, int inum)
1556 {
1557 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1558 
1559 	/*
1560 	 * Special case handling for drivers that provide their own
1561 	 * intrspec structures instead of relying on the DDI framework.
1562 	 *
1563 	 * A broken hardware driver in ON could potentially provide its
1564 	 * own intrspec structure, instead of relying on the hardware.
1565 	 * If these drivers are children of 'rootnex' then we need to
1566 	 * continue to provide backward compatibility to them here.
1567 	 *
1568 	 * Following check is a special case for 'pcic' driver which
1569 	 * was found to have broken hardwre andby provides its own intrspec.
1570 	 *
1571 	 * Verbatim comments from this driver are shown here:
1572 	 * "Don't use the ddi_add_intr since we don't have a
1573 	 * default intrspec in all cases."
1574 	 *
1575 	 * Since an 'ispec' may not be always created for it,
1576 	 * check for that and create one if so.
1577 	 *
1578 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1579 	 */
1580 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1581 		pdp->par_nintr = 1;
1582 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1583 		    pdp->par_nintr, KM_SLEEP);
1584 	}
1585 
1586 	/* Validate the interrupt number */
1587 	if (inum >= pdp->par_nintr)
1588 		return (NULL);
1589 
1590 	/* Get the interrupt structure pointer and return that */
1591 	return ((struct intrspec *)&pdp->par_intr[inum]);
1592 }
1593 
1594 
1595 /*
1596  * rootnex_xlate_intrs()
1597  *     For the x86 rootnexus, we're prepared to claim that the interrupt string
1598  *     is in the form of a list of <ipl,vec> specifications.
1599  */
1600 static int
1601 rootnex_xlate_intrs(dev_info_t *dip, dev_info_t *rdip, int *in,
1602     struct ddi_parent_private_data *pdptr)
1603 {
1604 	size_t size;
1605 	int n;
1606 	struct intrspec *new;
1607 	caddr_t got_prop;
1608 	int *inpri;
1609 	int got_len;
1610 
1611 	static char bad_intr_fmt[] =
1612 	    "rootnex: bad interrupt spec from %s%d - ipl %d, irq %d\n";
1613 
1614 #ifdef	lint
1615 	dip = dip;
1616 #endif
1617 	/*
1618 	 * determine if the driver is expecting the new style "interrupts"
1619 	 * property which just contains the IRQ, or the old style which
1620 	 * contains pairs of <IPL,IRQ>.  if it is the new style, we always
1621 	 * assign IPL 5 unless an "interrupt-priorities" property exists.
1622 	 * in that case, the "interrupt-priorities" property contains the
1623 	 * IPL values that match, one for one, the IRQ values in the
1624 	 * "interrupts" property.
1625 	 */
1626 	inpri = NULL;
1627 	if ((ddi_getprop(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS,
1628 	    "ignore-hardware-nodes", -1) != -1) ||
1629 	    ignore_hardware_nodes) {
1630 		/* the old style "interrupts" property... */
1631 
1632 		/*
1633 		 * The list consists of <ipl,vec> elements
1634 		 */
1635 		if ((n = (*in++ >> 1)) < 1)
1636 			return (DDI_FAILURE);
1637 
1638 		pdptr->par_nintr = n;
1639 		size = n * sizeof (struct intrspec);
1640 		new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
1641 
1642 		while (n--) {
1643 			int level = *in++;
1644 			int vec = *in++;
1645 
1646 			if (level < 1 || level > MAXIPL ||
1647 			    vec < VEC_MIN || vec > VEC_MAX) {
1648 				cmn_err(CE_CONT, bad_intr_fmt,
1649 				    DEVI(rdip)->devi_name,
1650 				    DEVI(rdip)->devi_instance, level, vec);
1651 				goto broken;
1652 			}
1653 			new->intrspec_pri = level;
1654 			if (vec != 2)
1655 				new->intrspec_vec = vec;
1656 			else
1657 				/*
1658 				 * irq 2 on the PC bus is tied to irq 9
1659 				 * on ISA, EISA and MicroChannel
1660 				 */
1661 				new->intrspec_vec = 9;
1662 			new++;
1663 		}
1664 
1665 		return (DDI_SUCCESS);
1666 	} else {
1667 		/* the new style "interrupts" property... */
1668 
1669 		/*
1670 		 * The list consists of <vec> elements
1671 		 */
1672 		if ((n = (*in++)) < 1)
1673 			return (DDI_FAILURE);
1674 
1675 		pdptr->par_nintr = n;
1676 		size = n * sizeof (struct intrspec);
1677 		new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP);
1678 
1679 		/* XXX check for "interrupt-priorities" property... */
1680 		if (ddi_getlongprop(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS,
1681 		    "interrupt-priorities", (caddr_t)&got_prop, &got_len)
1682 		    == DDI_PROP_SUCCESS) {
1683 			if (n != (got_len / sizeof (int))) {
1684 				cmn_err(CE_CONT,
1685 				    "rootnex: bad interrupt-priorities length"
1686 				    " from %s%d: expected %d, got %d\n",
1687 				    DEVI(rdip)->devi_name,
1688 				    DEVI(rdip)->devi_instance, n,
1689 				    (int)(got_len / sizeof (int)));
1690 				goto broken;
1691 			}
1692 			inpri = (int *)got_prop;
1693 		}
1694 
1695 		while (n--) {
1696 			int level;
1697 			int vec = *in++;
1698 
1699 			if (inpri == NULL)
1700 				level = 5;
1701 			else
1702 				level = *inpri++;
1703 
1704 			if (level < 1 || level > MAXIPL ||
1705 			    vec < VEC_MIN || vec > VEC_MAX) {
1706 				cmn_err(CE_CONT, bad_intr_fmt,
1707 				    DEVI(rdip)->devi_name,
1708 				    DEVI(rdip)->devi_instance, level, vec);
1709 				goto broken;
1710 			}
1711 			new->intrspec_pri = level;
1712 			if (vec != 2)
1713 				new->intrspec_vec = vec;
1714 			else
1715 				/*
1716 				 * irq 2 on the PC bus is tied to irq 9
1717 				 * on ISA, EISA and MicroChannel
1718 				 */
1719 				new->intrspec_vec = 9;
1720 			new++;
1721 		}
1722 
1723 		if (inpri != NULL)
1724 			kmem_free(got_prop, got_len);
1725 		return (DDI_SUCCESS);
1726 	}
1727 
1728 broken:
1729 	kmem_free(pdptr->par_intr, size);
1730 	pdptr->par_intr = NULL;
1731 	pdptr->par_nintr = 0;
1732 	if (inpri != NULL)
1733 		kmem_free(got_prop, got_len);
1734 	return (DDI_FAILURE);
1735 }
1736 
1737 
1738 /*
1739  * ******************
1740  *  dma related code
1741  * ******************
1742  */
1743 
1744 /*
1745  * rootnex_dma_allochdl()
1746  *    called from ddi_dma_alloc_handle().
1747  */
1748 /*ARGSUSED*/
1749 static int
1750 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1751     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1752 {
1753 	uint64_t maxsegmentsize_ll;
1754 	uint_t maxsegmentsize;
1755 	ddi_dma_impl_t *hp;
1756 	rootnex_dma_t *dma;
1757 	uint64_t count_max;
1758 	uint64_t seg;
1759 	int kmflag;
1760 	int e;
1761 
1762 
1763 	/* convert our sleep flags */
1764 	if (waitfp == DDI_DMA_SLEEP) {
1765 		kmflag = KM_SLEEP;
1766 	} else {
1767 		kmflag = KM_NOSLEEP;
1768 	}
1769 
1770 	/*
1771 	 * We try to do only one memory allocation here. We'll do a little
1772 	 * pointer manipulation later. If the bind ends up taking more than
1773 	 * our prealloc's space, we'll have to allocate more memory in the
1774 	 * bind operation. Not great, but much better than before and the
1775 	 * best we can do with the current bind interfaces.
1776 	 */
1777 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1778 	if (hp == NULL) {
1779 		if (waitfp != DDI_DMA_DONTWAIT) {
1780 			ddi_set_callback(waitfp, arg,
1781 			    &rootnex_state->r_dvma_call_list_id);
1782 		}
1783 		return (DDI_DMA_NORESOURCES);
1784 	}
1785 
1786 	/* Do our pointer manipulation now, align the structures */
1787 	hp->dmai_private = (void *)(((uintptr_t)hp +
1788 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1789 	dma = (rootnex_dma_t *)hp->dmai_private;
1790 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1791 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1792 
1793 	/* setup the handle */
1794 	rootnex_clean_dmahdl(hp);
1795 	dma->dp_dip = rdip;
1796 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1797 	dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1798 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1799 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1800 	hp->dmai_rdip = rdip;
1801 	hp->dmai_attr = *attr;
1802 
1803 	/* we don't need to worry about the SPL since we do a tryenter */
1804 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1805 
1806 	/*
1807 	 * Figure out our maximum segment size. If the segment size is greater
1808 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1809 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1810 	 * dma_attr_count_max are size-1 type values.
1811 	 *
1812 	 * Maximum segment size is the largest physically contiguous chunk of
1813 	 * memory that we can return from a bind (i.e. the maximum size of a
1814 	 * single cookie).
1815 	 */
1816 
1817 	/* handle the rollover cases */
1818 	seg = attr->dma_attr_seg + 1;
1819 	if (seg < attr->dma_attr_seg) {
1820 		seg = attr->dma_attr_seg;
1821 	}
1822 	count_max = attr->dma_attr_count_max + 1;
1823 	if (count_max < attr->dma_attr_count_max) {
1824 		count_max = attr->dma_attr_count_max;
1825 	}
1826 
1827 	/*
1828 	 * granularity may or may not be a power of two. If it isn't, we can't
1829 	 * use a simple mask.
1830 	 */
1831 	if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) {
1832 		dma->dp_granularity_power_2 = B_FALSE;
1833 	} else {
1834 		dma->dp_granularity_power_2 = B_TRUE;
1835 	}
1836 
1837 	/*
1838 	 * maxxfer should be a whole multiple of granularity. If we're going to
1839 	 * break up a window because we're greater than maxxfer, we might as
1840 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1841 	 * worry about triming the window later on for this case.
1842 	 */
1843 	if (attr->dma_attr_granular > 1) {
1844 		if (dma->dp_granularity_power_2) {
1845 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1846 			    (attr->dma_attr_maxxfer &
1847 			    (attr->dma_attr_granular - 1));
1848 		} else {
1849 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1850 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1851 		}
1852 	} else {
1853 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1854 	}
1855 
1856 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1857 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1858 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1859 		maxsegmentsize = 0xFFFFFFFF;
1860 	} else {
1861 		maxsegmentsize = maxsegmentsize_ll;
1862 	}
1863 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1864 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1865 
1866 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1867 	if (rootnex_alloc_check_parms) {
1868 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1869 		if (e != DDI_SUCCESS) {
1870 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1871 			(void) rootnex_dma_freehdl(dip, rdip,
1872 			    (ddi_dma_handle_t)hp);
1873 			return (e);
1874 		}
1875 	}
1876 
1877 	*handlep = (ddi_dma_handle_t)hp;
1878 
1879 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1880 	DTRACE_PROBE1(rootnex__alloc__handle, uint64_t,
1881 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1882 
1883 	return (DDI_SUCCESS);
1884 }
1885 
1886 
1887 /*
1888  * rootnex_dma_freehdl()
1889  *    called from ddi_dma_free_handle().
1890  */
1891 /*ARGSUSED*/
1892 static int
1893 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1894 {
1895 	ddi_dma_impl_t *hp;
1896 	rootnex_dma_t *dma;
1897 
1898 
1899 	hp = (ddi_dma_impl_t *)handle;
1900 	dma = (rootnex_dma_t *)hp->dmai_private;
1901 
1902 	/* unbind should have been called first */
1903 	ASSERT(!dma->dp_inuse);
1904 
1905 	mutex_destroy(&dma->dp_mutex);
1906 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1907 
1908 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1909 	DTRACE_PROBE1(rootnex__free__handle, uint64_t,
1910 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1911 
1912 	if (rootnex_state->r_dvma_call_list_id)
1913 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1914 
1915 	return (DDI_SUCCESS);
1916 }
1917 
1918 
1919 /*
1920  * rootnex_dma_bindhdl()
1921  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
1922  */
1923 /*ARGSUSED*/
1924 static int
1925 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
1926     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1927 {
1928 	rootnex_sglinfo_t *sinfo;
1929 	ddi_dma_attr_t *attr;
1930 	ddi_dma_impl_t *hp;
1931 	rootnex_dma_t *dma;
1932 	int kmflag;
1933 	int e;
1934 
1935 
1936 	hp = (ddi_dma_impl_t *)handle;
1937 	dma = (rootnex_dma_t *)hp->dmai_private;
1938 	sinfo = &dma->dp_sglinfo;
1939 	attr = &hp->dmai_attr;
1940 
1941 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1942 
1943 	/*
1944 	 * This is useful for debugging a driver. Not as useful in a production
1945 	 * system. The only time this will fail is if you have a driver bug.
1946 	 */
1947 	if (rootnex_bind_check_inuse) {
1948 		/*
1949 		 * No one else should ever have this lock unless someone else
1950 		 * is trying to use this handle. So contention on the lock
1951 		 * is the same as inuse being set.
1952 		 */
1953 		e = mutex_tryenter(&dma->dp_mutex);
1954 		if (e == 0) {
1955 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1956 			return (DDI_DMA_INUSE);
1957 		}
1958 		if (dma->dp_inuse) {
1959 			mutex_exit(&dma->dp_mutex);
1960 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1961 			return (DDI_DMA_INUSE);
1962 		}
1963 		dma->dp_inuse = B_TRUE;
1964 		mutex_exit(&dma->dp_mutex);
1965 	}
1966 
1967 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1968 	if (rootnex_bind_check_parms) {
1969 		e = rootnex_valid_bind_parms(dmareq, attr);
1970 		if (e != DDI_SUCCESS) {
1971 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1972 			rootnex_clean_dmahdl(hp);
1973 			return (e);
1974 		}
1975 	}
1976 
1977 	/* save away the original bind info */
1978 	dma->dp_dma = dmareq->dmar_object;
1979 
1980 	/*
1981 	 * Figure out a rough estimate of what maximum number of pages this
1982 	 * buffer could use (a high estimate of course).
1983 	 */
1984 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
1985 
1986 	/*
1987 	 * We'll use the pre-allocated cookies for any bind that will *always*
1988 	 * fit (more important to be consistent, we don't want to create
1989 	 * additional degenerate cases).
1990 	 */
1991 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
1992 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
1993 		dma->dp_need_to_free_cookie = B_FALSE;
1994 		DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
1995 		    uint_t, sinfo->si_max_pages);
1996 
1997 	/*
1998 	 * For anything larger than that, we'll go ahead and allocate the
1999 	 * maximum number of pages we expect to see. Hopefuly, we won't be
2000 	 * seeing this path in the fast path for high performance devices very
2001 	 * frequently.
2002 	 *
2003 	 * a ddi bind interface that allowed the driver to provide storage to
2004 	 * the bind interface would speed this case up.
2005 	 */
2006 	} else {
2007 		/* convert the sleep flags */
2008 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2009 			kmflag =  KM_SLEEP;
2010 		} else {
2011 			kmflag =  KM_NOSLEEP;
2012 		}
2013 
2014 		/*
2015 		 * Save away how much memory we allocated. If we're doing a
2016 		 * nosleep, the alloc could fail...
2017 		 */
2018 		dma->dp_cookie_size = sinfo->si_max_pages *
2019 		    sizeof (ddi_dma_cookie_t);
2020 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
2021 		if (dma->dp_cookies == NULL) {
2022 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2023 			rootnex_clean_dmahdl(hp);
2024 			return (DDI_DMA_NORESOURCES);
2025 		}
2026 		dma->dp_need_to_free_cookie = B_TRUE;
2027 		DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t,
2028 		    sinfo->si_max_pages);
2029 	}
2030 	hp->dmai_cookie = dma->dp_cookies;
2031 
2032 	/*
2033 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
2034 	 * looking at the contraints in the dma structure. It will then put some
2035 	 * additional state about the sgl in the dma struct (i.e. is the sgl
2036 	 * clean, or do we need to do some munging; how many pages need to be
2037 	 * copied, etc.)
2038 	 */
2039 	rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies,
2040 	    &dma->dp_sglinfo);
2041 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
2042 
2043 	/* if we don't need a copy buffer, we don't need to sync */
2044 	if (sinfo->si_copybuf_req == 0) {
2045 		hp->dmai_rflags |= DMP_NOSYNC;
2046 	}
2047 
2048 	/*
2049 	 * if we don't need the copybuf and we don't need to do a partial,  we
2050 	 * hit the fast path. All the high performance devices should be trying
2051 	 * to hit this path. To hit this path, a device should be able to reach
2052 	 * all of memory, shouldn't try to bind more than it can transfer, and
2053 	 * the buffer shouldn't require more cookies than the driver/device can
2054 	 * handle [sgllen]).
2055 	 */
2056 	if ((sinfo->si_copybuf_req == 0) &&
2057 	    (sinfo->si_sgl_size <= attr->dma_attr_sgllen) &&
2058 	    (dma->dp_dma.dmao_size < dma->dp_maxxfer)) {
2059 		/*
2060 		 * copy out the first cookie and ccountp, set the cookie
2061 		 * pointer to the second cookie. The first cookie is passed
2062 		 * back on the stack. Additional cookies are accessed via
2063 		 * ddi_dma_nextcookie()
2064 		 */
2065 		*cookiep = dma->dp_cookies[0];
2066 		*ccountp = sinfo->si_sgl_size;
2067 		hp->dmai_cookie++;
2068 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2069 		hp->dmai_nwin = 1;
2070 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2071 		DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t,
2072 		    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2073 		    dma->dp_dma.dmao_size);
2074 		return (DDI_DMA_MAPPED);
2075 	}
2076 
2077 	/*
2078 	 * go to the slow path, we may need to alloc more memory, create
2079 	 * multiple windows, and munge up a sgl to make the device happy.
2080 	 */
2081 	e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag);
2082 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
2083 		if (dma->dp_need_to_free_cookie) {
2084 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2085 		}
2086 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
2087 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
2088 		return (e);
2089 	}
2090 
2091 	/* if the first window uses the copy buffer, sync it for the device */
2092 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
2093 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
2094 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
2095 		    DDI_DMA_SYNC_FORDEV);
2096 	}
2097 
2098 	/*
2099 	 * copy out the first cookie and ccountp, set the cookie pointer to the
2100 	 * second cookie. Make sure the partial flag is set/cleared correctly.
2101 	 * If we have a partial map (i.e. multiple windows), the number of
2102 	 * cookies we return is the number of cookies in the first window.
2103 	 */
2104 	if (e == DDI_DMA_MAPPED) {
2105 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
2106 		*ccountp = sinfo->si_sgl_size;
2107 	} else {
2108 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
2109 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
2110 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
2111 	}
2112 	*cookiep = dma->dp_cookies[0];
2113 	hp->dmai_cookie++;
2114 
2115 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2116 	DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
2117 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
2118 	    dma->dp_dma.dmao_size);
2119 	return (e);
2120 }
2121 
2122 
2123 /*
2124  * rootnex_dma_unbindhdl()
2125  *    called from ddi_dma_unbind_handle()
2126  */
2127 /*ARGSUSED*/
2128 static int
2129 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
2130     ddi_dma_handle_t handle)
2131 {
2132 	ddi_dma_impl_t *hp;
2133 	rootnex_dma_t *dma;
2134 	int e;
2135 
2136 
2137 	hp = (ddi_dma_impl_t *)handle;
2138 	dma = (rootnex_dma_t *)hp->dmai_private;
2139 
2140 	/* make sure the buffer wasn't free'd before calling unbind */
2141 	if (rootnex_unbind_verify_buffer) {
2142 		e = rootnex_verify_buffer(dma);
2143 		if (e != DDI_SUCCESS) {
2144 			ASSERT(0);
2145 			return (DDI_FAILURE);
2146 		}
2147 	}
2148 
2149 	/* sync the current window before unbinding the buffer */
2150 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
2151 	    (hp->dmai_rflags & DDI_DMA_READ)) {
2152 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
2153 		    DDI_DMA_SYNC_FORCPU);
2154 	}
2155 
2156 	/*
2157 	 * cleanup and copy buffer or window state. if we didn't use the copy
2158 	 * buffer or windows, there won't be much to do :-)
2159 	 */
2160 	rootnex_teardown_copybuf(dma);
2161 	rootnex_teardown_windows(dma);
2162 
2163 	/*
2164 	 * If we had to allocate space to for the worse case sgl (it didn't
2165 	 * fit into our pre-allocate buffer), free that up now
2166 	 */
2167 	if (dma->dp_need_to_free_cookie) {
2168 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
2169 	}
2170 
2171 	/*
2172 	 * clean up the handle so it's ready for the next bind (i.e. if the
2173 	 * handle is reused).
2174 	 */
2175 	rootnex_clean_dmahdl(hp);
2176 
2177 	if (rootnex_state->r_dvma_call_list_id)
2178 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
2179 
2180 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2181 	DTRACE_PROBE1(rootnex__unbind, uint64_t,
2182 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
2183 
2184 	return (DDI_SUCCESS);
2185 }
2186 
2187 
2188 /*
2189  * rootnex_verify_buffer()
2190  *   verify buffer wasn't free'd
2191  */
2192 static int
2193 rootnex_verify_buffer(rootnex_dma_t *dma)
2194 {
2195 	peekpoke_ctlops_t peek;
2196 	page_t **pplist;
2197 	caddr_t vaddr;
2198 	uint_t pcnt;
2199 	uint_t poff;
2200 	page_t *pp;
2201 	uint8_t b;
2202 	int i;
2203 	int e;
2204 
2205 
2206 	/* Figure out how many pages this buffer occupies */
2207 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
2208 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
2209 	} else {
2210 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
2211 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2212 	}
2213 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
2214 
2215 	switch (dma->dp_dma.dmao_type) {
2216 	case DMA_OTYP_PAGES:
2217 		/*
2218 		 * for a linked list of pp's walk through them to make sure
2219 		 * they're locked and not free.
2220 		 */
2221 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
2222 		for (i = 0; i < pcnt; i++) {
2223 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
2224 				return (DDI_FAILURE);
2225 			}
2226 			pp = pp->p_next;
2227 		}
2228 		break;
2229 
2230 	case DMA_OTYP_VADDR:
2231 	case DMA_OTYP_BUFVADDR:
2232 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
2233 		/*
2234 		 * for an array of pp's walk through them to make sure they're
2235 		 * not free. It's possible that they may not be locked.
2236 		 */
2237 		if (pplist) {
2238 			for (i = 0; i < pcnt; i++) {
2239 				if (PP_ISFREE(pplist[i])) {
2240 					return (DDI_FAILURE);
2241 				}
2242 			}
2243 
2244 		/* For a virtual address, try to peek at each page */
2245 		} else {
2246 			if (dma->dp_sglinfo.si_asp == &kas) {
2247 				bzero(&peek, sizeof (peekpoke_ctlops_t));
2248 				peek.host_addr = (uintptr_t)&b;
2249 				peek.size = sizeof (uint8_t);
2250 				peek.dev_addr = (uintptr_t)vaddr;
2251 				for (i = 0; i < pcnt; i++) {
2252 					e = rootnex_ctlops_peek(&peek, &b);
2253 					if (e != DDI_SUCCESS) {
2254 						return (DDI_FAILURE);
2255 					}
2256 					peek.dev_addr += MMU_PAGESIZE;
2257 				}
2258 			}
2259 		}
2260 		break;
2261 
2262 	default:
2263 		ASSERT(0);
2264 		break;
2265 	}
2266 
2267 	return (DDI_SUCCESS);
2268 }
2269 
2270 
2271 /*
2272  * rootnex_clean_dmahdl()
2273  *    Clean the dma handle. This should be called on a handle alloc and an
2274  *    unbind handle. Set the handle state to the default settings.
2275  */
2276 static void
2277 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2278 {
2279 	rootnex_dma_t *dma;
2280 
2281 
2282 	dma = (rootnex_dma_t *)hp->dmai_private;
2283 
2284 	hp->dmai_nwin = 0;
2285 	dma->dp_current_cookie = 0;
2286 	dma->dp_copybuf_size = 0;
2287 	dma->dp_window = NULL;
2288 	dma->dp_cbaddr = NULL;
2289 	dma->dp_inuse = B_FALSE;
2290 	dma->dp_need_to_free_cookie = B_FALSE;
2291 	dma->dp_need_to_free_window = B_FALSE;
2292 	dma->dp_partial_required = B_FALSE;
2293 	dma->dp_trim_required = B_FALSE;
2294 	dma->dp_sglinfo.si_copybuf_req = 0;
2295 #if !defined(__amd64)
2296 	dma->dp_cb_remaping = B_FALSE;
2297 	dma->dp_kva = NULL;
2298 #endif
2299 
2300 	/* FMA related initialization */
2301 	hp->dmai_fault = 0;
2302 	hp->dmai_fault_check = NULL;
2303 	hp->dmai_fault_notify = NULL;
2304 	hp->dmai_error.err_ena = 0;
2305 	hp->dmai_error.err_status = DDI_FM_OK;
2306 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2307 	hp->dmai_error.err_ontrap = NULL;
2308 	hp->dmai_error.err_fep = NULL;
2309 }
2310 
2311 
2312 /*
2313  * rootnex_valid_alloc_parms()
2314  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2315  */
2316 static int
2317 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2318 {
2319 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2320 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2321 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2322 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2323 		return (DDI_DMA_BADATTR);
2324 	}
2325 
2326 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2327 		return (DDI_DMA_BADATTR);
2328 	}
2329 
2330 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2331 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2332 	    attr->dma_attr_sgllen <= 0) {
2333 		return (DDI_DMA_BADATTR);
2334 	}
2335 
2336 	/* We should be able to DMA into every byte offset in a page */
2337 	if (maxsegmentsize < MMU_PAGESIZE) {
2338 		return (DDI_DMA_BADATTR);
2339 	}
2340 
2341 	return (DDI_SUCCESS);
2342 }
2343 
2344 
2345 /*
2346  * rootnex_valid_bind_parms()
2347  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2348  */
2349 /* ARGSUSED */
2350 static int
2351 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2352 {
2353 #if !defined(__amd64)
2354 	/*
2355 	 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2356 	 * we can track the offset for the obsoleted interfaces.
2357 	 */
2358 	if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2359 		return (DDI_DMA_TOOBIG);
2360 	}
2361 #endif
2362 
2363 	return (DDI_SUCCESS);
2364 }
2365 
2366 
2367 /*
2368  * rootnex_get_sgl()
2369  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2370  *    with a call to the vm layer when vm2.0 comes around...
2371  */
2372 static void
2373 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2374     rootnex_sglinfo_t *sglinfo)
2375 {
2376 	ddi_dma_atyp_t buftype;
2377 	uint64_t last_page;
2378 	uint64_t offset;
2379 	uint64_t addrhi;
2380 	uint64_t addrlo;
2381 	uint64_t maxseg;
2382 	page_t **pplist;
2383 	uint64_t paddr;
2384 	uint32_t psize;
2385 	uint32_t size;
2386 	caddr_t vaddr;
2387 	uint_t pcnt;
2388 	page_t *pp;
2389 	uint_t cnt;
2390 
2391 
2392 	/* shortcuts */
2393 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2394 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2395 	maxseg = sglinfo->si_max_cookie_size;
2396 	buftype = dmar_object->dmao_type;
2397 	addrhi = sglinfo->si_max_addr;
2398 	addrlo = sglinfo->si_min_addr;
2399 	size = dmar_object->dmao_size;
2400 
2401 	pcnt = 0;
2402 	cnt = 0;
2403 
2404 	/*
2405 	 * if we were passed down a linked list of pages, i.e. pointer to
2406 	 * page_t, use this to get our physical address and buf offset.
2407 	 */
2408 	if (buftype == DMA_OTYP_PAGES) {
2409 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2410 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2411 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2412 		    MMU_PAGEOFFSET;
2413 		paddr = ptob64(pp->p_pagenum) + offset;
2414 		psize = MIN(size, (MMU_PAGESIZE - offset));
2415 		pp = pp->p_next;
2416 		sglinfo->si_asp = NULL;
2417 
2418 	/*
2419 	 * We weren't passed down a linked list of pages, but if we were passed
2420 	 * down an array of pages, use this to get our physical address and buf
2421 	 * offset.
2422 	 */
2423 	} else if (pplist != NULL) {
2424 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2425 		    (buftype == DMA_OTYP_BUFVADDR));
2426 
2427 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2428 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2429 		if (sglinfo->si_asp == NULL) {
2430 			sglinfo->si_asp = &kas;
2431 		}
2432 
2433 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2434 		paddr = ptob64(pplist[pcnt]->p_pagenum);
2435 		paddr += offset;
2436 		psize = MIN(size, (MMU_PAGESIZE - offset));
2437 		pcnt++;
2438 
2439 	/*
2440 	 * All we have is a virtual address, we'll need to call into the VM
2441 	 * to get the physical address.
2442 	 */
2443 	} else {
2444 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2445 		    (buftype == DMA_OTYP_BUFVADDR));
2446 
2447 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2448 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2449 		if (sglinfo->si_asp == NULL) {
2450 			sglinfo->si_asp = &kas;
2451 		}
2452 
2453 		paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2454 		paddr += offset;
2455 		psize = MIN(size, (MMU_PAGESIZE - offset));
2456 		vaddr += psize;
2457 	}
2458 
2459 	/*
2460 	 * Setup the first cookie with the physical address of the page and the
2461 	 * size of the page (which takes into account the initial offset into
2462 	 * the page.
2463 	 */
2464 	sgl[cnt].dmac_laddress = paddr;
2465 	sgl[cnt].dmac_size = psize;
2466 	sgl[cnt].dmac_type = 0;
2467 
2468 	/*
2469 	 * Save away the buffer offset into the page. We'll need this later in
2470 	 * the copy buffer code to help figure out the page index within the
2471 	 * buffer and the offset into the current page.
2472 	 */
2473 	sglinfo->si_buf_offset = offset;
2474 
2475 	/*
2476 	 * If the DMA engine can't reach the physical address, increase how
2477 	 * much copy buffer we need. We always increase by pagesize so we don't
2478 	 * have to worry about converting offsets. Set a flag in the cookies
2479 	 * dmac_type to indicate that it uses the copy buffer. If this isn't the
2480 	 * last cookie, go to the next cookie (since we separate each page which
2481 	 * uses the copy buffer in case the copy buffer is not physically
2482 	 * contiguous.
2483 	 */
2484 	if ((paddr < addrlo) || ((paddr + psize) > addrhi)) {
2485 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
2486 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2487 		if ((cnt + 1) < sglinfo->si_max_pages) {
2488 			cnt++;
2489 			sgl[cnt].dmac_laddress = 0;
2490 			sgl[cnt].dmac_size = 0;
2491 			sgl[cnt].dmac_type = 0;
2492 		}
2493 	}
2494 
2495 	/*
2496 	 * save this page's physical address so we can figure out if the next
2497 	 * page is physically contiguous. Keep decrementing size until we are
2498 	 * done with the buffer.
2499 	 */
2500 	last_page = paddr & MMU_PAGEMASK;
2501 	size -= psize;
2502 
2503 	while (size > 0) {
2504 		/* Get the size for this page (i.e. partial or full page) */
2505 		psize = MIN(size, MMU_PAGESIZE);
2506 
2507 		if (buftype == DMA_OTYP_PAGES) {
2508 			/* get the paddr from the page_t */
2509 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2510 			paddr = ptob64(pp->p_pagenum);
2511 			pp = pp->p_next;
2512 		} else if (pplist != NULL) {
2513 			/* index into the array of page_t's to get the paddr */
2514 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2515 			paddr = ptob64(pplist[pcnt]->p_pagenum);
2516 			pcnt++;
2517 		} else {
2518 			/* call into the VM to get the paddr */
2519 			paddr =  ptob64(hat_getpfnum(sglinfo->si_asp->a_hat,
2520 			    vaddr));
2521 			vaddr += psize;
2522 		}
2523 
2524 		/* check to see if this page needs the copy buffer */
2525 		if ((paddr < addrlo) || ((paddr + psize) > addrhi)) {
2526 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
2527 
2528 			/*
2529 			 * if there is something in the current cookie, go to
2530 			 * the next one. We only want one page in a cookie which
2531 			 * uses the copybuf since the copybuf doesn't have to
2532 			 * be physically contiguous.
2533 			 */
2534 			if (sgl[cnt].dmac_size != 0) {
2535 				cnt++;
2536 			}
2537 			sgl[cnt].dmac_laddress = paddr;
2538 			sgl[cnt].dmac_size = psize;
2539 #if defined(__amd64)
2540 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2541 #else
2542 			/*
2543 			 * save the buf offset for 32-bit kernel. used in the
2544 			 * obsoleted interfaces.
2545 			 */
2546 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
2547 			    (dmar_object->dmao_size - size);
2548 #endif
2549 			/* if this isn't the last cookie, go to the next one */
2550 			if ((cnt + 1) < sglinfo->si_max_pages) {
2551 				cnt++;
2552 				sgl[cnt].dmac_laddress = 0;
2553 				sgl[cnt].dmac_size = 0;
2554 				sgl[cnt].dmac_type = 0;
2555 			}
2556 
2557 		/*
2558 		 * this page didn't need the copy buffer, if it's not physically
2559 		 * contiguous, or it would put us over a segment boundary, or it
2560 		 * puts us over the max cookie size, or the current sgl doesn't
2561 		 * have anything in it.
2562 		 */
2563 		} else if (((last_page + MMU_PAGESIZE) != paddr) ||
2564 		    !(paddr & sglinfo->si_segmask) ||
2565 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
2566 		    (sgl[cnt].dmac_size == 0)) {
2567 			/*
2568 			 * if we're not already in a new cookie, go to the next
2569 			 * cookie.
2570 			 */
2571 			if (sgl[cnt].dmac_size != 0) {
2572 				cnt++;
2573 			}
2574 
2575 			/* save the cookie information */
2576 			sgl[cnt].dmac_laddress = paddr;
2577 			sgl[cnt].dmac_size = psize;
2578 #if defined(__amd64)
2579 			sgl[cnt].dmac_type = 0;
2580 #else
2581 			/*
2582 			 * save the buf offset for 32-bit kernel. used in the
2583 			 * obsoleted interfaces.
2584 			 */
2585 			sgl[cnt].dmac_type = dmar_object->dmao_size - size;
2586 #endif
2587 
2588 		/*
2589 		 * this page didn't need the copy buffer, it is physically
2590 		 * contiguous with the last page, and it's <= the max cookie
2591 		 * size.
2592 		 */
2593 		} else {
2594 			sgl[cnt].dmac_size += psize;
2595 
2596 			/*
2597 			 * if this exactly ==  the maximum cookie size, and
2598 			 * it isn't the last cookie, go to the next cookie.
2599 			 */
2600 			if (((sgl[cnt].dmac_size + psize) == maxseg) &&
2601 			    ((cnt + 1) < sglinfo->si_max_pages)) {
2602 				cnt++;
2603 				sgl[cnt].dmac_laddress = 0;
2604 				sgl[cnt].dmac_size = 0;
2605 				sgl[cnt].dmac_type = 0;
2606 			}
2607 		}
2608 
2609 		/*
2610 		 * save this page's physical address so we can figure out if the
2611 		 * next page is physically contiguous. Keep decrementing size
2612 		 * until we are done with the buffer.
2613 		 */
2614 		last_page = paddr;
2615 		size -= psize;
2616 	}
2617 
2618 	/* we're done, save away how many cookies the sgl has */
2619 	if (sgl[cnt].dmac_size == 0) {
2620 		ASSERT(cnt < sglinfo->si_max_pages);
2621 		sglinfo->si_sgl_size = cnt;
2622 	} else {
2623 		sglinfo->si_sgl_size = cnt + 1;
2624 	}
2625 }
2626 
2627 
2628 /*
2629  * rootnex_bind_slowpath()
2630  *    Call in the bind path if the calling driver can't use the sgl without
2631  *    modifying it. We either need to use the copy buffer and/or we will end up
2632  *    with a partial bind.
2633  */
2634 static int
2635 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2636     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag)
2637 {
2638 	rootnex_sglinfo_t *sinfo;
2639 	rootnex_window_t *window;
2640 	ddi_dma_cookie_t *cookie;
2641 	size_t copybuf_used;
2642 	size_t dmac_size;
2643 	boolean_t partial;
2644 	off_t cur_offset;
2645 	page_t *cur_pp;
2646 	major_t mnum;
2647 	int e;
2648 	int i;
2649 
2650 
2651 	sinfo = &dma->dp_sglinfo;
2652 	copybuf_used = 0;
2653 	partial = B_FALSE;
2654 
2655 	/*
2656 	 * If we're using the copybuf, set the copybuf state in dma struct.
2657 	 * Needs to be first since it sets the copy buffer size.
2658 	 */
2659 	if (sinfo->si_copybuf_req != 0) {
2660 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
2661 		if (e != DDI_SUCCESS) {
2662 			return (e);
2663 		}
2664 	} else {
2665 		dma->dp_copybuf_size = 0;
2666 	}
2667 
2668 	/*
2669 	 * Figure out if we need to do a partial mapping. If so, figure out
2670 	 * if we need to trim the buffers when we munge the sgl.
2671 	 */
2672 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
2673 	    (dma->dp_dma.dmao_size > dma->dp_maxxfer) ||
2674 	    (attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
2675 		dma->dp_partial_required = B_TRUE;
2676 		if (attr->dma_attr_granular != 1) {
2677 			dma->dp_trim_required = B_TRUE;
2678 		}
2679 	} else {
2680 		dma->dp_partial_required = B_FALSE;
2681 		dma->dp_trim_required = B_FALSE;
2682 	}
2683 
2684 	/* If we need to do a partial bind, make sure the driver supports it */
2685 	if (dma->dp_partial_required &&
2686 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
2687 
2688 		mnum = ddi_driver_major(dma->dp_dip);
2689 		/*
2690 		 * patchable which allows us to print one warning per major
2691 		 * number.
2692 		 */
2693 		if ((rootnex_bind_warn) &&
2694 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
2695 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
2696 			cmn_err(CE_WARN, "!%s: coding error detected, the "
2697 			    "driver is using ddi_dma_attr(9S) incorrectly. "
2698 			    "There is a small risk of data corruption in "
2699 			    "particular with large I/Os. The driver should be "
2700 			    "replaced with a corrected version for proper "
2701 			    "system operation. To disable this warning, add "
2702 			    "'set rootnex:rootnex_bind_warn=0' to "
2703 			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
2704 		}
2705 		return (DDI_DMA_TOOBIG);
2706 	}
2707 
2708 	/*
2709 	 * we might need multiple windows, setup state to handle them. In this
2710 	 * code path, we will have at least one window.
2711 	 */
2712 	e = rootnex_setup_windows(hp, dma, attr, kmflag);
2713 	if (e != DDI_SUCCESS) {
2714 		rootnex_teardown_copybuf(dma);
2715 		return (e);
2716 	}
2717 
2718 	window = &dma->dp_window[0];
2719 	cookie = &dma->dp_cookies[0];
2720 	cur_offset = 0;
2721 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
2722 	if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) {
2723 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
2724 	}
2725 
2726 	/* loop though all the cookies we got back from get_sgl() */
2727 	for (i = 0; i < sinfo->si_sgl_size; i++) {
2728 		/*
2729 		 * If we're using the copy buffer, check this cookie and setup
2730 		 * its associated copy buffer state. If this cookie uses the
2731 		 * copy buffer, make sure we sync this window during dma_sync.
2732 		 */
2733 		if (dma->dp_copybuf_size > 0) {
2734 			rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie,
2735 			    cur_offset, &copybuf_used, &cur_pp);
2736 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2737 				window->wd_dosync = B_TRUE;
2738 			}
2739 		}
2740 
2741 		/*
2742 		 * save away the cookie size, since it could be modified in
2743 		 * the windowing code.
2744 		 */
2745 		dmac_size = cookie->dmac_size;
2746 
2747 		/* if we went over max copybuf size */
2748 		if (dma->dp_copybuf_size &&
2749 		    (copybuf_used > dma->dp_copybuf_size)) {
2750 			partial = B_TRUE;
2751 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
2752 			    cookie, cur_offset, &copybuf_used);
2753 			if (e != DDI_SUCCESS) {
2754 				rootnex_teardown_copybuf(dma);
2755 				rootnex_teardown_windows(dma);
2756 				return (e);
2757 			}
2758 
2759 			/*
2760 			 * if the coookie uses the copy buffer, make sure the
2761 			 * new window we just moved to is set to sync.
2762 			 */
2763 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2764 				window->wd_dosync = B_TRUE;
2765 			}
2766 			DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *,
2767 			    dma->dp_dip);
2768 
2769 		/* if the cookie cnt == max sgllen, move to the next window */
2770 		} else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) {
2771 			partial = B_TRUE;
2772 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
2773 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
2774 			    cookie, attr, cur_offset);
2775 			if (e != DDI_SUCCESS) {
2776 				rootnex_teardown_copybuf(dma);
2777 				rootnex_teardown_windows(dma);
2778 				return (e);
2779 			}
2780 
2781 			/*
2782 			 * if the coookie uses the copy buffer, make sure the
2783 			 * new window we just moved to is set to sync.
2784 			 */
2785 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2786 				window->wd_dosync = B_TRUE;
2787 			}
2788 			DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *,
2789 			    dma->dp_dip);
2790 
2791 		/* else if we will be over maxxfer */
2792 		} else if ((window->wd_size + dmac_size) >
2793 		    dma->dp_maxxfer) {
2794 			partial = B_TRUE;
2795 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
2796 			    cookie);
2797 			if (e != DDI_SUCCESS) {
2798 				rootnex_teardown_copybuf(dma);
2799 				rootnex_teardown_windows(dma);
2800 				return (e);
2801 			}
2802 
2803 			/*
2804 			 * if the coookie uses the copy buffer, make sure the
2805 			 * new window we just moved to is set to sync.
2806 			 */
2807 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2808 				window->wd_dosync = B_TRUE;
2809 			}
2810 			DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *,
2811 			    dma->dp_dip);
2812 
2813 		/* else this cookie fits in the current window */
2814 		} else {
2815 			window->wd_cookie_cnt++;
2816 			window->wd_size += dmac_size;
2817 		}
2818 
2819 		/* track our offset into the buffer, go to the next cookie */
2820 		ASSERT(dmac_size <= dma->dp_dma.dmao_size);
2821 		ASSERT(cookie->dmac_size <= dmac_size);
2822 		cur_offset += dmac_size;
2823 		cookie++;
2824 	}
2825 
2826 	/* if we ended up with a zero sized window in the end, clean it up */
2827 	if (window->wd_size == 0) {
2828 		hp->dmai_nwin--;
2829 		window--;
2830 	}
2831 
2832 	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
2833 
2834 	if (!partial) {
2835 		return (DDI_DMA_MAPPED);
2836 	}
2837 
2838 	ASSERT(dma->dp_partial_required);
2839 	return (DDI_DMA_PARTIAL_MAP);
2840 }
2841 
2842 
2843 /*
2844  * rootnex_setup_copybuf()
2845  *    Called in bind slowpath. Figures out if we're going to use the copy
2846  *    buffer, and if we do, sets up the basic state to handle it.
2847  */
2848 static int
2849 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2850     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
2851 {
2852 	rootnex_sglinfo_t *sinfo;
2853 	ddi_dma_attr_t lattr;
2854 	size_t max_copybuf;
2855 	int cansleep;
2856 	int e;
2857 #if !defined(__amd64)
2858 	int vmflag;
2859 #endif
2860 
2861 
2862 	sinfo = &dma->dp_sglinfo;
2863 
2864 	/*
2865 	 * read this first so it's consistent through the routine so we can
2866 	 * patch it on the fly.
2867 	 */
2868 	max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK;
2869 
2870 	/* We need to call into the rootnex on ddi_dma_sync() */
2871 	hp->dmai_rflags &= ~DMP_NOSYNC;
2872 
2873 	/* make sure the copybuf size <= the max size */
2874 	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
2875 	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
2876 
2877 #if !defined(__amd64)
2878 	/*
2879 	 * if we don't have kva space to copy to/from, allocate the KVA space
2880 	 * now. We only do this for the 32-bit kernel. We use seg kpm space for
2881 	 * the 64-bit kernel.
2882 	 */
2883 	if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
2884 	    (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
2885 
2886 		/* convert the sleep flags */
2887 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2888 			vmflag = VM_SLEEP;
2889 		} else {
2890 			vmflag = VM_NOSLEEP;
2891 		}
2892 
2893 		/* allocate Kernel VA space that we can bcopy to/from */
2894 		dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
2895 		    vmflag);
2896 		if (dma->dp_kva == NULL) {
2897 			return (DDI_DMA_NORESOURCES);
2898 		}
2899 	}
2900 #endif
2901 
2902 	/* convert the sleep flags */
2903 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2904 		cansleep = 1;
2905 	} else {
2906 		cansleep = 0;
2907 	}
2908 
2909 	/*
2910 	 * Allocated the actual copy buffer. This needs to fit within the DMA
2911 	 * engines limits, so we can't use kmem_alloc...
2912 	 */
2913 	lattr = *attr;
2914 	lattr.dma_attr_align = MMU_PAGESIZE;
2915 	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
2916 	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
2917 	if (e != DDI_SUCCESS) {
2918 #if !defined(__amd64)
2919 		if (dma->dp_kva != NULL) {
2920 			vmem_free(heap_arena, dma->dp_kva,
2921 			    dma->dp_copybuf_size);
2922 		}
2923 #endif
2924 		return (DDI_DMA_NORESOURCES);
2925 	}
2926 
2927 	DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
2928 	    size_t, dma->dp_copybuf_size);
2929 
2930 	return (DDI_SUCCESS);
2931 }
2932 
2933 
2934 /*
2935  * rootnex_setup_windows()
2936  *    Called in bind slowpath to setup the window state. We always have windows
2937  *    in the slowpath. Even if the window count = 1.
2938  */
2939 static int
2940 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
2941     ddi_dma_attr_t *attr, int kmflag)
2942 {
2943 	rootnex_window_t *windowp;
2944 	rootnex_sglinfo_t *sinfo;
2945 	size_t copy_state_size;
2946 	size_t win_state_size;
2947 	size_t state_available;
2948 	size_t space_needed;
2949 	uint_t copybuf_win;
2950 	uint_t maxxfer_win;
2951 	size_t space_used;
2952 	uint_t sglwin;
2953 
2954 
2955 	sinfo = &dma->dp_sglinfo;
2956 
2957 	dma->dp_current_win = 0;
2958 	hp->dmai_nwin = 0;
2959 
2960 	/* If we don't need to do a partial, we only have one window */
2961 	if (!dma->dp_partial_required) {
2962 		dma->dp_max_win = 1;
2963 
2964 	/*
2965 	 * we need multiple windows, need to figure out the worse case number
2966 	 * of windows.
2967 	 */
2968 	} else {
2969 		/*
2970 		 * if we need windows because we need more copy buffer that
2971 		 * we allow, the worse case number of windows we could need
2972 		 * here would be (copybuf space required / copybuf space that
2973 		 * we have) plus one for remainder, and plus 2 to handle the
2974 		 * extra pages on the trim for the first and last pages of the
2975 		 * buffer (a page is the minimum window size so under the right
2976 		 * attr settings, you could have a window for each page).
2977 		 * The last page will only be hit here if the size is not a
2978 		 * multiple of the granularity (which theoretically shouldn't
2979 		 * be the case but never has been enforced, so we could have
2980 		 * broken things without it).
2981 		 */
2982 		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
2983 			ASSERT(dma->dp_copybuf_size > 0);
2984 			copybuf_win = (sinfo->si_copybuf_req /
2985 			    dma->dp_copybuf_size) + 1 + 2;
2986 		} else {
2987 			copybuf_win = 0;
2988 		}
2989 
2990 		/*
2991 		 * if we need windows because we have more cookies than the H/W
2992 		 * can handle, the number of windows we would need here would
2993 		 * be (cookie count / cookies count H/W supports) plus one for
2994 		 * remainder, and plus 2 to handle the extra pages on the trim
2995 		 * (see above comment about trim)
2996 		 */
2997 		if (attr->dma_attr_sgllen < sinfo->si_sgl_size) {
2998 			sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen)
2999 			    + 1) + 2;
3000 		} else {
3001 			sglwin = 0;
3002 		}
3003 
3004 		/*
3005 		 * if we need windows because we're binding more memory than the
3006 		 * H/W can transfer at once, the number of windows we would need
3007 		 * here would be (xfer count / max xfer H/W supports) plus one
3008 		 * for remainder, and plus 2 to handle the extra pages on the
3009 		 * trim (see above comment about trim)
3010 		 */
3011 		if (dma->dp_dma.dmao_size > dma->dp_maxxfer) {
3012 			maxxfer_win = (dma->dp_dma.dmao_size /
3013 			    dma->dp_maxxfer) + 1 + 2;
3014 		} else {
3015 			maxxfer_win = 0;
3016 		}
3017 		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
3018 		ASSERT(dma->dp_max_win > 0);
3019 	}
3020 	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
3021 
3022 	/*
3023 	 * Get space for window and potential copy buffer state. Before we
3024 	 * go and allocate memory, see if we can get away with using what's
3025 	 * left in the pre-allocted state or the dynamically allocated sgl.
3026 	 */
3027 	space_used = (uintptr_t)(sinfo->si_sgl_size *
3028 	    sizeof (ddi_dma_cookie_t));
3029 
3030 	/* if we dynamically allocated space for the cookies */
3031 	if (dma->dp_need_to_free_cookie) {
3032 		/* if we have more space in the pre-allocted buffer, use it */
3033 		ASSERT(space_used <= dma->dp_cookie_size);
3034 		if ((dma->dp_cookie_size - space_used) <=
3035 		    rootnex_state->r_prealloc_size) {
3036 			state_available = rootnex_state->r_prealloc_size;
3037 			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
3038 
3039 		/*
3040 		 * else, we have more free space in the dynamically allocated
3041 		 * buffer, i.e. the buffer wasn't worse case fragmented so we
3042 		 * didn't need a lot of cookies.
3043 		 */
3044 		} else {
3045 			state_available = dma->dp_cookie_size - space_used;
3046 			windowp = (rootnex_window_t *)
3047 			    &dma->dp_cookies[sinfo->si_sgl_size];
3048 		}
3049 
3050 	/* we used the pre-alloced buffer */
3051 	} else {
3052 		ASSERT(space_used <= rootnex_state->r_prealloc_size);
3053 		state_available = rootnex_state->r_prealloc_size - space_used;
3054 		windowp = (rootnex_window_t *)
3055 		    &dma->dp_cookies[sinfo->si_sgl_size];
3056 	}
3057 
3058 	/*
3059 	 * figure out how much state we need to track the copy buffer. Add an
3060 	 * addition 8 bytes for pointer alignemnt later.
3061 	 */
3062 	if (dma->dp_copybuf_size > 0) {
3063 		copy_state_size = sinfo->si_max_pages *
3064 		    sizeof (rootnex_pgmap_t);
3065 	} else {
3066 		copy_state_size = 0;
3067 	}
3068 	/* add an additional 8 bytes for pointer alignment */
3069 	space_needed = win_state_size + copy_state_size + 0x8;
3070 
3071 	/* if we have enough space already, use it */
3072 	if (state_available >= space_needed) {
3073 		dma->dp_window = windowp;
3074 		dma->dp_need_to_free_window = B_FALSE;
3075 
3076 	/* not enough space, need to allocate more. */
3077 	} else {
3078 		dma->dp_window = kmem_alloc(space_needed, kmflag);
3079 		if (dma->dp_window == NULL) {
3080 			return (DDI_DMA_NORESOURCES);
3081 		}
3082 		dma->dp_need_to_free_window = B_TRUE;
3083 		dma->dp_window_size = space_needed;
3084 		DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *,
3085 		    dma->dp_dip, size_t, space_needed);
3086 	}
3087 
3088 	/*
3089 	 * we allocate copy buffer state and window state at the same time.
3090 	 * setup our copy buffer state pointers. Make sure it's aligned.
3091 	 */
3092 	if (dma->dp_copybuf_size > 0) {
3093 		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
3094 		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
3095 
3096 #if !defined(__amd64)
3097 		/*
3098 		 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
3099 		 * false/NULL. Should be quicker to bzero vs loop and set.
3100 		 */
3101 		bzero(dma->dp_pgmap, copy_state_size);
3102 #endif
3103 	} else {
3104 		dma->dp_pgmap = NULL;
3105 	}
3106 
3107 	return (DDI_SUCCESS);
3108 }
3109 
3110 
3111 /*
3112  * rootnex_teardown_copybuf()
3113  *    cleans up after rootnex_setup_copybuf()
3114  */
3115 static void
3116 rootnex_teardown_copybuf(rootnex_dma_t *dma)
3117 {
3118 #if !defined(__amd64)
3119 	int i;
3120 
3121 	/*
3122 	 * if we allocated kernel heap VMEM space, go through all the pages and
3123 	 * map out any of the ones that we're mapped into the kernel heap VMEM
3124 	 * arena. Then free the VMEM space.
3125 	 */
3126 	if (dma->dp_kva != NULL) {
3127 		for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
3128 			if (dma->dp_pgmap[i].pm_mapped) {
3129 				hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
3130 				    MMU_PAGESIZE, HAT_UNLOAD);
3131 				dma->dp_pgmap[i].pm_mapped = B_FALSE;
3132 			}
3133 		}
3134 
3135 		vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
3136 	}
3137 
3138 #endif
3139 
3140 	/* if we allocated a copy buffer, free it */
3141 	if (dma->dp_cbaddr != NULL) {
3142 		i_ddi_mem_free(dma->dp_cbaddr, 0);
3143 	}
3144 }
3145 
3146 
3147 /*
3148  * rootnex_teardown_windows()
3149  *    cleans up after rootnex_setup_windows()
3150  */
3151 static void
3152 rootnex_teardown_windows(rootnex_dma_t *dma)
3153 {
3154 	/*
3155 	 * if we had to allocate window state on the last bind (because we
3156 	 * didn't have enough pre-allocated space in the handle), free it.
3157 	 */
3158 	if (dma->dp_need_to_free_window) {
3159 		kmem_free(dma->dp_window, dma->dp_window_size);
3160 	}
3161 }
3162 
3163 
3164 /*
3165  * rootnex_init_win()
3166  *    Called in bind slow path during creation of a new window. Initializes
3167  *    window state to default values.
3168  */
3169 /*ARGSUSED*/
3170 static void
3171 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3172     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
3173 {
3174 	hp->dmai_nwin++;
3175 	window->wd_dosync = B_FALSE;
3176 	window->wd_offset = cur_offset;
3177 	window->wd_size = 0;
3178 	window->wd_first_cookie = cookie;
3179 	window->wd_cookie_cnt = 0;
3180 	window->wd_trim.tr_trim_first = B_FALSE;
3181 	window->wd_trim.tr_trim_last = B_FALSE;
3182 	window->wd_trim.tr_first_copybuf_win = B_FALSE;
3183 	window->wd_trim.tr_last_copybuf_win = B_FALSE;
3184 #if !defined(__amd64)
3185 	window->wd_remap_copybuf = dma->dp_cb_remaping;
3186 #endif
3187 }
3188 
3189 
3190 /*
3191  * rootnex_setup_cookie()
3192  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
3193  *    the sgl uses the copy buffer, we need to go through each cookie, figure
3194  *    out if it uses the copy buffer, and if it does, save away everything we'll
3195  *    need during sync.
3196  */
3197 static void
3198 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
3199     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
3200     page_t **cur_pp)
3201 {
3202 	boolean_t copybuf_sz_power_2;
3203 	rootnex_sglinfo_t *sinfo;
3204 	uint_t pidx;
3205 	uint_t pcnt;
3206 	off_t poff;
3207 #if defined(__amd64)
3208 	pfn_t pfn;
3209 #else
3210 	page_t **pplist;
3211 #endif
3212 
3213 	sinfo = &dma->dp_sglinfo;
3214 
3215 	/*
3216 	 * Calculate the page index relative to the start of the buffer. The
3217 	 * index to the current page for our buffer is the offset into the
3218 	 * first page of the buffer plus our current offset into the buffer
3219 	 * itself, shifted of course...
3220 	 */
3221 	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
3222 	ASSERT(pidx < sinfo->si_max_pages);
3223 
3224 	/* if this cookie uses the copy buffer */
3225 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3226 		/*
3227 		 * NOTE: we know that since this cookie uses the copy buffer, it
3228 		 * is <= MMU_PAGESIZE.
3229 		 */
3230 
3231 		/*
3232 		 * get the offset into the page. For the 64-bit kernel, get the
3233 		 * pfn which we'll use with seg kpm.
3234 		 */
3235 		poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET;
3236 #if defined(__amd64)
3237 		pfn = cookie->_dmu._dmac_ll >> MMU_PAGESHIFT;
3238 #endif
3239 
3240 		/* figure out if the copybuf size is a power of 2 */
3241 		if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) {
3242 			copybuf_sz_power_2 = B_FALSE;
3243 		} else {
3244 			copybuf_sz_power_2 = B_TRUE;
3245 		}
3246 
3247 		/* This page uses the copy buffer */
3248 		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
3249 
3250 		/*
3251 		 * save the copy buffer KVA that we'll use with this page.
3252 		 * if we still fit within the copybuf, it's a simple add.
3253 		 * otherwise, we need to wrap over using & or % accordingly.
3254 		 */
3255 		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3256 			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3257 			    *copybuf_used;
3258 		} else {
3259 			if (copybuf_sz_power_2) {
3260 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3261 				    (uintptr_t)dma->dp_cbaddr +
3262 				    (*copybuf_used &
3263 				    (dma->dp_copybuf_size - 1)));
3264 			} else {
3265 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3266 				    (uintptr_t)dma->dp_cbaddr +
3267 				    (*copybuf_used % dma->dp_copybuf_size));
3268 			}
3269 		}
3270 
3271 		/*
3272 		 * over write the cookie physical address with the address of
3273 		 * the physical address of the copy buffer page that we will
3274 		 * use.
3275 		 */
3276 		cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat,
3277 		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3278 
3279 		/* if we have a kernel VA, it's easy, just save that address */
3280 		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3281 		    (sinfo->si_asp == &kas)) {
3282 			/*
3283 			 * save away the page aligned virtual address of the
3284 			 * driver buffer. Offsets are handled in the sync code.
3285 			 */
3286 			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3287 			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3288 			    & MMU_PAGEMASK);
3289 #if !defined(__amd64)
3290 			/*
3291 			 * we didn't need to, and will never need to map this
3292 			 * page.
3293 			 */
3294 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3295 #endif
3296 
3297 		/* we don't have a kernel VA. We need one for the bcopy. */
3298 		} else {
3299 #if defined(__amd64)
3300 			/*
3301 			 * for the 64-bit kernel, it's easy. We use seg kpm to
3302 			 * get a Kernel VA for the corresponding pfn.
3303 			 */
3304 			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3305 #else
3306 			/*
3307 			 * for the 32-bit kernel, this is a pain. First we'll
3308 			 * save away the page_t or user VA for this page. This
3309 			 * is needed in rootnex_dma_win() when we switch to a
3310 			 * new window which requires us to re-map the copy
3311 			 * buffer.
3312 			 */
3313 			pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3314 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3315 				dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3316 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3317 			} else if (pplist != NULL) {
3318 				dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3319 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3320 			} else {
3321 				dma->dp_pgmap[pidx].pm_pp = NULL;
3322 				dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3323 				    (((uintptr_t)
3324 				    dmar_object->dmao_obj.virt_obj.v_addr +
3325 				    cur_offset) & MMU_PAGEMASK);
3326 			}
3327 
3328 			/*
3329 			 * save away the page aligned virtual address which was
3330 			 * allocated from the kernel heap arena (taking into
3331 			 * account if we need more copy buffer than we alloced
3332 			 * and use multiple windows to handle this, i.e. &,%).
3333 			 * NOTE: there isn't and physical memory backing up this
3334 			 * virtual address space currently.
3335 			 */
3336 			if ((*copybuf_used + MMU_PAGESIZE) <=
3337 			    dma->dp_copybuf_size) {
3338 				dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3339 				    (((uintptr_t)dma->dp_kva + *copybuf_used) &
3340 				    MMU_PAGEMASK);
3341 			} else {
3342 				if (copybuf_sz_power_2) {
3343 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3344 					    (((uintptr_t)dma->dp_kva +
3345 					    (*copybuf_used &
3346 					    (dma->dp_copybuf_size - 1))) &
3347 					    MMU_PAGEMASK);
3348 				} else {
3349 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3350 					    (((uintptr_t)dma->dp_kva +
3351 					    (*copybuf_used %
3352 					    dma->dp_copybuf_size)) &
3353 					    MMU_PAGEMASK);
3354 				}
3355 			}
3356 
3357 			/*
3358 			 * if we haven't used up the available copy buffer yet,
3359 			 * map the kva to the physical page.
3360 			 */
3361 			if (!dma->dp_cb_remaping && ((*copybuf_used +
3362 			    MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3363 				dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3364 				if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3365 					i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3366 					    dma->dp_pgmap[pidx].pm_kaddr);
3367 				} else {
3368 					i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
3369 					    sinfo->si_asp,
3370 					    dma->dp_pgmap[pidx].pm_kaddr);
3371 				}
3372 
3373 			/*
3374 			 * we've used up the available copy buffer, this page
3375 			 * will have to be mapped during rootnex_dma_win() when
3376 			 * we switch to a new window which requires a re-map
3377 			 * the copy buffer. (32-bit kernel only)
3378 			 */
3379 			} else {
3380 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3381 			}
3382 #endif
3383 			/* go to the next page_t */
3384 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3385 				*cur_pp = (*cur_pp)->p_next;
3386 			}
3387 		}
3388 
3389 		/* add to the copy buffer count */
3390 		*copybuf_used += MMU_PAGESIZE;
3391 
3392 	/*
3393 	 * This cookie doesn't use the copy buffer. Walk through the pages this
3394 	 * cookie occupies to reflect this.
3395 	 */
3396 	} else {
3397 		/*
3398 		 * figure out how many pages the cookie occupies. We need to
3399 		 * use the original page offset of the buffer and the cookies
3400 		 * offset in the buffer to do this.
3401 		 */
3402 		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
3403 		pcnt = mmu_btopr(cookie->dmac_size + poff);
3404 
3405 		while (pcnt > 0) {
3406 #if !defined(__amd64)
3407 			/*
3408 			 * the 32-bit kernel doesn't have seg kpm, so we need
3409 			 * to map in the driver buffer (if it didn't come down
3410 			 * with a kernel VA) on the fly. Since this page doesn't
3411 			 * use the copy buffer, it's not, or will it ever, have
3412 			 * to be mapped in.
3413 			 */
3414 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3415 #endif
3416 			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
3417 
3418 			/*
3419 			 * we need to update pidx and cur_pp or we'll loose
3420 			 * track of where we are.
3421 			 */
3422 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3423 				*cur_pp = (*cur_pp)->p_next;
3424 			}
3425 			pidx++;
3426 			pcnt--;
3427 		}
3428 	}
3429 }
3430 
3431 
3432 /*
3433  * rootnex_sgllen_window_boundary()
3434  *    Called in the bind slow path when the next cookie causes us to exceed (in
3435  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
3436  *    length supported by the DMA H/W.
3437  */
3438 static int
3439 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3440     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
3441     off_t cur_offset)
3442 {
3443 	off_t new_offset;
3444 	size_t trim_sz;
3445 	off_t coffset;
3446 
3447 
3448 	/*
3449 	 * if we know we'll never have to trim, it's pretty easy. Just move to
3450 	 * the next window and init it. We're done.
3451 	 */
3452 	if (!dma->dp_trim_required) {
3453 		(*windowp)++;
3454 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3455 		(*windowp)->wd_cookie_cnt++;
3456 		(*windowp)->wd_size = cookie->dmac_size;
3457 		return (DDI_SUCCESS);
3458 	}
3459 
3460 	/* figure out how much we need to trim from the window */
3461 	ASSERT(attr->dma_attr_granular != 0);
3462 	if (dma->dp_granularity_power_2) {
3463 		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
3464 	} else {
3465 		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
3466 	}
3467 
3468 	/* The window's a whole multiple of granularity. We're done */
3469 	if (trim_sz == 0) {
3470 		(*windowp)++;
3471 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3472 		(*windowp)->wd_cookie_cnt++;
3473 		(*windowp)->wd_size = cookie->dmac_size;
3474 		return (DDI_SUCCESS);
3475 	}
3476 
3477 	/*
3478 	 * The window's not a whole multiple of granularity, since we know this
3479 	 * is due to the sgllen, we need to go back to the last cookie and trim
3480 	 * that one, add the left over part of the old cookie into the new
3481 	 * window, and then add in the new cookie into the new window.
3482 	 */
3483 
3484 	/*
3485 	 * make sure the driver isn't making us do something bad... Trimming and
3486 	 * sgllen == 1 don't go together.
3487 	 */
3488 	if (attr->dma_attr_sgllen == 1) {
3489 		return (DDI_DMA_NOMAPPING);
3490 	}
3491 
3492 	/*
3493 	 * first, setup the current window to account for the trim. Need to go
3494 	 * back to the last cookie for this.
3495 	 */
3496 	cookie--;
3497 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3498 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3499 	(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3500 	ASSERT(cookie->dmac_size > trim_sz);
3501 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3502 	(*windowp)->wd_size -= trim_sz;
3503 
3504 	/* save the buffer offsets for the next window */
3505 	coffset = cookie->dmac_size - trim_sz;
3506 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3507 
3508 	/*
3509 	 * set this now in case this is the first window. all other cases are
3510 	 * set in dma_win()
3511 	 */
3512 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3513 
3514 	/*
3515 	 * initialize the next window using what's left over in the previous
3516 	 * cookie.
3517 	 */
3518 	(*windowp)++;
3519 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3520 	(*windowp)->wd_cookie_cnt++;
3521 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3522 	(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset;
3523 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3524 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3525 		(*windowp)->wd_dosync = B_TRUE;
3526 	}
3527 
3528 	/*
3529 	 * now go back to the current cookie and add it to the new window. set
3530 	 * the new window size to the what was left over from the previous
3531 	 * cookie and what's in the current cookie.
3532 	 */
3533 	cookie++;
3534 	(*windowp)->wd_cookie_cnt++;
3535 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3536 
3537 	/*
3538 	 * trim plus the next cookie could put us over maxxfer (a cookie can be
3539 	 * a max size of maxxfer). Handle that case.
3540 	 */
3541 	if ((*windowp)->wd_size > dma->dp_maxxfer) {
3542 		/*
3543 		 * maxxfer is already a whole multiple of granularity, and this
3544 		 * trim will be <= the previous trim (since a cookie can't be
3545 		 * larger than maxxfer). Make things simple here.
3546 		 */
3547 		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
3548 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3549 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3550 		(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3551 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3552 		(*windowp)->wd_size -= trim_sz;
3553 		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
3554 
3555 		/* save the buffer offsets for the next window */
3556 		coffset = cookie->dmac_size - trim_sz;
3557 		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3558 
3559 		/* setup the next window */
3560 		(*windowp)++;
3561 		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3562 		(*windowp)->wd_cookie_cnt++;
3563 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3564 		(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll +
3565 		    coffset;
3566 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3567 	}
3568 
3569 	return (DDI_SUCCESS);
3570 }
3571 
3572 
3573 /*
3574  * rootnex_copybuf_window_boundary()
3575  *    Called in bind slowpath when we get to a window boundary because we used
3576  *    up all the copy buffer that we have.
3577  */
3578 static int
3579 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3580     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
3581     size_t *copybuf_used)
3582 {
3583 	rootnex_sglinfo_t *sinfo;
3584 	off_t new_offset;
3585 	size_t trim_sz;
3586 	off_t coffset;
3587 	uint_t pidx;
3588 	off_t poff;
3589 
3590 
3591 	sinfo = &dma->dp_sglinfo;
3592 
3593 	/*
3594 	 * the copy buffer should be a whole multiple of page size. We know that
3595 	 * this cookie is <= MMU_PAGESIZE.
3596 	 */
3597 	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
3598 
3599 	/*
3600 	 * from now on, all new windows in this bind need to be re-mapped during
3601 	 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
3602 	 * space...
3603 	 */
3604 #if !defined(__amd64)
3605 	dma->dp_cb_remaping = B_TRUE;
3606 #endif
3607 
3608 	/* reset copybuf used */
3609 	*copybuf_used = 0;
3610 
3611 	/*
3612 	 * if we don't have to trim (since granularity is set to 1), go to the
3613 	 * next window and add the current cookie to it. We know the current
3614 	 * cookie uses the copy buffer since we're in this code path.
3615 	 */
3616 	if (!dma->dp_trim_required) {
3617 		(*windowp)++;
3618 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3619 
3620 		/* Add this cookie to the new window */
3621 		(*windowp)->wd_cookie_cnt++;
3622 		(*windowp)->wd_size += cookie->dmac_size;
3623 		*copybuf_used += MMU_PAGESIZE;
3624 		return (DDI_SUCCESS);
3625 	}
3626 
3627 	/*
3628 	 * *** may need to trim, figure it out.
3629 	 */
3630 
3631 	/* figure out how much we need to trim from the window */
3632 	if (dma->dp_granularity_power_2) {
3633 		trim_sz = (*windowp)->wd_size &
3634 		    (hp->dmai_attr.dma_attr_granular - 1);
3635 	} else {
3636 		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
3637 	}
3638 
3639 	/*
3640 	 * if the window's a whole multiple of granularity, go to the next
3641 	 * window, init it, then add in the current cookie. We know the current
3642 	 * cookie uses the copy buffer since we're in this code path.
3643 	 */
3644 	if (trim_sz == 0) {
3645 		(*windowp)++;
3646 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3647 
3648 		/* Add this cookie to the new window */
3649 		(*windowp)->wd_cookie_cnt++;
3650 		(*windowp)->wd_size += cookie->dmac_size;
3651 		*copybuf_used += MMU_PAGESIZE;
3652 		return (DDI_SUCCESS);
3653 	}
3654 
3655 	/*
3656 	 * *** We figured it out, we definitly need to trim
3657 	 */
3658 
3659 	/*
3660 	 * make sure the driver isn't making us do something bad...
3661 	 * Trimming and sgllen == 1 don't go together.
3662 	 */
3663 	if (hp->dmai_attr.dma_attr_sgllen == 1) {
3664 		return (DDI_DMA_NOMAPPING);
3665 	}
3666 
3667 	/*
3668 	 * first, setup the current window to account for the trim. Need to go
3669 	 * back to the last cookie for this. Some of the last cookie will be in
3670 	 * the current window, and some of the last cookie will be in the new
3671 	 * window. All of the current cookie will be in the new window.
3672 	 */
3673 	cookie--;
3674 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3675 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3676 	(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3677 	ASSERT(cookie->dmac_size > trim_sz);
3678 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3679 	(*windowp)->wd_size -= trim_sz;
3680 
3681 	/*
3682 	 * we're trimming the last cookie (not the current cookie). So that
3683 	 * last cookie may have or may not have been using the copy buffer (
3684 	 * we know the cookie passed in uses the copy buffer since we're in
3685 	 * this code path).
3686 	 *
3687 	 * If the last cookie doesn't use the copy buffer, nothing special to
3688 	 * do. However, if it does uses the copy buffer, it will be both the
3689 	 * last page in the current window and the first page in the next
3690 	 * window. Since we are reusing the copy buffer (and KVA space on the
3691 	 * 32-bit kernel), this page will use the end of the copy buffer in the
3692 	 * current window, and the start of the copy buffer in the next window.
3693 	 * Track that info... The cookie physical address was already set to
3694 	 * the copy buffer physical address in setup_cookie..
3695 	 */
3696 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3697 		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
3698 		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
3699 		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
3700 		(*windowp)->wd_trim.tr_last_pidx = pidx;
3701 		(*windowp)->wd_trim.tr_last_cbaddr =
3702 		    dma->dp_pgmap[pidx].pm_cbaddr;
3703 #if !defined(__amd64)
3704 		(*windowp)->wd_trim.tr_last_kaddr =
3705 		    dma->dp_pgmap[pidx].pm_kaddr;
3706 #endif
3707 	}
3708 
3709 	/* save the buffer offsets for the next window */
3710 	coffset = cookie->dmac_size - trim_sz;
3711 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3712 
3713 	/*
3714 	 * set this now in case this is the first window. all other cases are
3715 	 * set in dma_win()
3716 	 */
3717 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3718 
3719 	/*
3720 	 * initialize the next window using what's left over in the previous
3721 	 * cookie.
3722 	 */
3723 	(*windowp)++;
3724 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3725 	(*windowp)->wd_cookie_cnt++;
3726 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3727 	(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset;
3728 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3729 
3730 	/*
3731 	 * again, we're tracking if the last cookie uses the copy buffer.
3732 	 * read the comment above for more info on why we need to track
3733 	 * additional state.
3734 	 *
3735 	 * For the first cookie in the new window, we need reset the physical
3736 	 * address to DMA into to the start of the copy buffer plus any
3737 	 * initial page offset which may be present.
3738 	 */
3739 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3740 		(*windowp)->wd_dosync = B_TRUE;
3741 		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
3742 		(*windowp)->wd_trim.tr_first_pidx = pidx;
3743 		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
3744 		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
3745 		(*windowp)->wd_trim.tr_first_paddr = ptob64(hat_getpfnum(
3746 		    kas.a_hat, dma->dp_cbaddr)) + poff;
3747 #if !defined(__amd64)
3748 		(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
3749 #endif
3750 		/* account for the cookie copybuf usage in the new window */
3751 		*copybuf_used += MMU_PAGESIZE;
3752 
3753 		/*
3754 		 * every piece of code has to have a hack, and here is this
3755 		 * ones :-)
3756 		 *
3757 		 * There is a complex interaction between setup_cookie and the
3758 		 * copybuf window boundary. The complexity had to be in either
3759 		 * the maxxfer window, or the copybuf window, and I chose the
3760 		 * copybuf code.
3761 		 *
3762 		 * So in this code path, we have taken the last cookie,
3763 		 * virtually broken it in half due to the trim, and it happens
3764 		 * to use the copybuf which further complicates life. At the
3765 		 * same time, we have already setup the current cookie, which
3766 		 * is now wrong. More background info: the current cookie uses
3767 		 * the copybuf, so it is only a page long max. So we need to
3768 		 * fix the current cookies copy buffer address, physical
3769 		 * address, and kva for the 32-bit kernel. We due this by
3770 		 * bumping them by page size (of course, we can't due this on
3771 		 * the physical address since the copy buffer may not be
3772 		 * physically contiguous).
3773 		 */
3774 		cookie++;
3775 		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
3776 		poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET;
3777 		cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat,
3778 		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
3779 #if !defined(__amd64)
3780 		ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
3781 		dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
3782 #endif
3783 	} else {
3784 		/* go back to the current cookie */
3785 		cookie++;
3786 	}
3787 
3788 	/*
3789 	 * add the current cookie to the new window. set the new window size to
3790 	 * the what was left over from the previous cookie and what's in the
3791 	 * current cookie.
3792 	 */
3793 	(*windowp)->wd_cookie_cnt++;
3794 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3795 	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
3796 
3797 	/*
3798 	 * we know that the cookie passed in always uses the copy buffer. We
3799 	 * wouldn't be here if it didn't.
3800 	 */
3801 	*copybuf_used += MMU_PAGESIZE;
3802 
3803 	return (DDI_SUCCESS);
3804 }
3805 
3806 
3807 /*
3808  * rootnex_maxxfer_window_boundary()
3809  *    Called in bind slowpath when we get to a window boundary because we will
3810  *    go over maxxfer.
3811  */
3812 static int
3813 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3814     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
3815 {
3816 	size_t dmac_size;
3817 	off_t new_offset;
3818 	size_t trim_sz;
3819 	off_t coffset;
3820 
3821 
3822 	/*
3823 	 * calculate how much we have to trim off of the current cookie to equal
3824 	 * maxxfer. We don't have to account for granularity here since our
3825 	 * maxxfer already takes that into account.
3826 	 */
3827 	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
3828 	ASSERT(trim_sz <= cookie->dmac_size);
3829 	ASSERT(trim_sz <= dma->dp_maxxfer);
3830 
3831 	/* save cookie size since we need it later and we might change it */
3832 	dmac_size = cookie->dmac_size;
3833 
3834 	/*
3835 	 * if we're not trimming the entire cookie, setup the current window to
3836 	 * account for the trim.
3837 	 */
3838 	if (trim_sz < cookie->dmac_size) {
3839 		(*windowp)->wd_cookie_cnt++;
3840 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3841 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3842 		(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3843 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3844 		(*windowp)->wd_size = dma->dp_maxxfer;
3845 
3846 		/*
3847 		 * set the adjusted cookie size now in case this is the first
3848 		 * window. All other windows are taken care of in get win
3849 		 */
3850 		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3851 	}
3852 
3853 	/*
3854 	 * coffset is the current offset within the cookie, new_offset is the
3855 	 * current offset with the entire buffer.
3856 	 */
3857 	coffset = dmac_size - trim_sz;
3858 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3859 
3860 	/* initialize the next window */
3861 	(*windowp)++;
3862 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3863 	(*windowp)->wd_cookie_cnt++;
3864 	(*windowp)->wd_size = trim_sz;
3865 	if (trim_sz < dmac_size) {
3866 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3867 		(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll +
3868 		    coffset;
3869 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3870 	}
3871 
3872 	return (DDI_SUCCESS);
3873 }
3874 
3875 
3876 /*
3877  * rootnex_dma_sync()
3878  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
3879  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
3880  *    is set, ddi_dma_sync() returns immediately passing back success.
3881  */
3882 /*ARGSUSED*/
3883 static int
3884 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
3885     off_t off, size_t len, uint_t cache_flags)
3886 {
3887 	rootnex_sglinfo_t *sinfo;
3888 	rootnex_pgmap_t *cbpage;
3889 	rootnex_window_t *win;
3890 	ddi_dma_impl_t *hp;
3891 	rootnex_dma_t *dma;
3892 	caddr_t fromaddr;
3893 	caddr_t toaddr;
3894 	uint_t psize;
3895 	off_t offset;
3896 	uint_t pidx;
3897 	size_t size;
3898 	off_t poff;
3899 	int e;
3900 
3901 
3902 	hp = (ddi_dma_impl_t *)handle;
3903 	dma = (rootnex_dma_t *)hp->dmai_private;
3904 	sinfo = &dma->dp_sglinfo;
3905 
3906 	/*
3907 	 * if we don't have any windows, we don't need to sync. A copybuf
3908 	 * will cause us to have at least one window.
3909 	 */
3910 	if (dma->dp_window == NULL) {
3911 		return (DDI_SUCCESS);
3912 	}
3913 
3914 	/* This window may not need to be sync'd */
3915 	win = &dma->dp_window[dma->dp_current_win];
3916 	if (!win->wd_dosync) {
3917 		return (DDI_SUCCESS);
3918 	}
3919 
3920 	/* handle off and len special cases */
3921 	if ((off == 0) || (rootnex_sync_ignore_params)) {
3922 		offset = win->wd_offset;
3923 	} else {
3924 		offset = off;
3925 	}
3926 	if ((len == 0) || (rootnex_sync_ignore_params)) {
3927 		size = win->wd_size;
3928 	} else {
3929 		size = len;
3930 	}
3931 
3932 	/* check the sync args to make sure they make a little sense */
3933 	if (rootnex_sync_check_parms) {
3934 		e = rootnex_valid_sync_parms(hp, win, offset, size,
3935 		    cache_flags);
3936 		if (e != DDI_SUCCESS) {
3937 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
3938 			return (DDI_FAILURE);
3939 		}
3940 	}
3941 
3942 	/*
3943 	 * special case the first page to handle the offset into the page. The
3944 	 * offset to the current page for our buffer is the offset into the
3945 	 * first page of the buffer plus our current offset into the buffer
3946 	 * itself, masked of course.
3947 	 */
3948 	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
3949 	psize = MIN((MMU_PAGESIZE - poff), size);
3950 
3951 	/* go through all the pages that we want to sync */
3952 	while (size > 0) {
3953 		/*
3954 		 * Calculate the page index relative to the start of the buffer.
3955 		 * The index to the current page for our buffer is the offset
3956 		 * into the first page of the buffer plus our current offset
3957 		 * into the buffer itself, shifted of course...
3958 		 */
3959 		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
3960 		ASSERT(pidx < sinfo->si_max_pages);
3961 
3962 		/*
3963 		 * if this page uses the copy buffer, we need to sync it,
3964 		 * otherwise, go on to the next page.
3965 		 */
3966 		cbpage = &dma->dp_pgmap[pidx];
3967 		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
3968 		    (cbpage->pm_uses_copybuf == B_FALSE));
3969 		if (cbpage->pm_uses_copybuf) {
3970 			/* cbaddr and kaddr should be page aligned */
3971 			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
3972 			    MMU_PAGEOFFSET) == 0);
3973 			ASSERT(((uintptr_t)cbpage->pm_kaddr &
3974 			    MMU_PAGEOFFSET) == 0);
3975 
3976 			/*
3977 			 * if we're copying for the device, we are going to
3978 			 * copy from the drivers buffer and to the rootnex
3979 			 * allocated copy buffer.
3980 			 */
3981 			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
3982 				fromaddr = cbpage->pm_kaddr + poff;
3983 				toaddr = cbpage->pm_cbaddr + poff;
3984 				DTRACE_PROBE2(rootnex__sync__dev,
3985 				    dev_info_t *, dma->dp_dip, size_t, psize);
3986 
3987 			/*
3988 			 * if we're copying for the cpu/kernel, we are going to
3989 			 * copy from the rootnex allocated copy buffer to the
3990 			 * drivers buffer.
3991 			 */
3992 			} else {
3993 				fromaddr = cbpage->pm_cbaddr + poff;
3994 				toaddr = cbpage->pm_kaddr + poff;
3995 				DTRACE_PROBE2(rootnex__sync__cpu,
3996 				    dev_info_t *, dma->dp_dip, size_t, psize);
3997 			}
3998 
3999 			bcopy(fromaddr, toaddr, psize);
4000 		}
4001 
4002 		/*
4003 		 * decrement size until we're done, update our offset into the
4004 		 * buffer, and get the next page size.
4005 		 */
4006 		size -= psize;
4007 		offset += psize;
4008 		psize = MIN(MMU_PAGESIZE, size);
4009 
4010 		/* page offset is zero for the rest of this loop */
4011 		poff = 0;
4012 	}
4013 
4014 	return (DDI_SUCCESS);
4015 }
4016 
4017 
4018 /*
4019  * rootnex_valid_sync_parms()
4020  *    checks the parameters passed to sync to verify they are correct.
4021  */
4022 static int
4023 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
4024     off_t offset, size_t size, uint_t cache_flags)
4025 {
4026 	off_t woffset;
4027 
4028 
4029 	/*
4030 	 * the first part of the test to make sure the offset passed in is
4031 	 * within the window.
4032 	 */
4033 	if (offset < win->wd_offset) {
4034 		return (DDI_FAILURE);
4035 	}
4036 
4037 	/*
4038 	 * second and last part of the test to make sure the offset and length
4039 	 * passed in is within the window.
4040 	 */
4041 	woffset = offset - win->wd_offset;
4042 	if ((woffset + size) > win->wd_size) {
4043 		return (DDI_FAILURE);
4044 	}
4045 
4046 	/*
4047 	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
4048 	 * be set too.
4049 	 */
4050 	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
4051 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
4052 		return (DDI_SUCCESS);
4053 	}
4054 
4055 	/*
4056 	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
4057 	 * should be set. Also DDI_DMA_READ should be set in the flags.
4058 	 */
4059 	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
4060 	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
4061 	    (hp->dmai_rflags & DDI_DMA_READ)) {
4062 		return (DDI_SUCCESS);
4063 	}
4064 
4065 	return (DDI_FAILURE);
4066 }
4067 
4068 
4069 /*
4070  * rootnex_dma_win()
4071  *    called from ddi_dma_getwin()
4072  */
4073 /*ARGSUSED*/
4074 static int
4075 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4076     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
4077     uint_t *ccountp)
4078 {
4079 	rootnex_window_t *window;
4080 	rootnex_trim_t *trim;
4081 	ddi_dma_impl_t *hp;
4082 	rootnex_dma_t *dma;
4083 #if !defined(__amd64)
4084 	rootnex_sglinfo_t *sinfo;
4085 	rootnex_pgmap_t *pmap;
4086 	uint_t pidx;
4087 	uint_t pcnt;
4088 	off_t poff;
4089 	int i;
4090 #endif
4091 
4092 
4093 	hp = (ddi_dma_impl_t *)handle;
4094 	dma = (rootnex_dma_t *)hp->dmai_private;
4095 #if !defined(__amd64)
4096 	sinfo = &dma->dp_sglinfo;
4097 #endif
4098 
4099 	/* If we try and get a window which doesn't exist, return failure */
4100 	if (win >= hp->dmai_nwin) {
4101 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4102 		return (DDI_FAILURE);
4103 	}
4104 
4105 	/*
4106 	 * if we don't have any windows, and they're asking for the first
4107 	 * window, setup the cookie pointer to the first cookie in the bind.
4108 	 * setup our return values, then increment the cookie since we return
4109 	 * the first cookie on the stack.
4110 	 */
4111 	if (dma->dp_window == NULL) {
4112 		if (win != 0) {
4113 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
4114 			return (DDI_FAILURE);
4115 		}
4116 		hp->dmai_cookie = dma->dp_cookies;
4117 		*offp = 0;
4118 		*lenp = dma->dp_dma.dmao_size;
4119 		*ccountp = dma->dp_sglinfo.si_sgl_size;
4120 		*cookiep = hp->dmai_cookie[0];
4121 		hp->dmai_cookie++;
4122 		return (DDI_SUCCESS);
4123 	}
4124 
4125 	/* sync the old window before moving on to the new one */
4126 	window = &dma->dp_window[dma->dp_current_win];
4127 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
4128 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
4129 		    DDI_DMA_SYNC_FORCPU);
4130 	}
4131 
4132 #if !defined(__amd64)
4133 	/*
4134 	 * before we move to the next window, if we need to re-map, unmap all
4135 	 * the pages in this window.
4136 	 */
4137 	if (dma->dp_cb_remaping) {
4138 		/*
4139 		 * If we switch to this window again, we'll need to map in
4140 		 * on the fly next time.
4141 		 */
4142 		window->wd_remap_copybuf = B_TRUE;
4143 
4144 		/*
4145 		 * calculate the page index into the buffer where this window
4146 		 * starts, and the number of pages this window takes up.
4147 		 */
4148 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4149 		    MMU_PAGESHIFT;
4150 		poff = (sinfo->si_buf_offset + window->wd_offset) &
4151 		    MMU_PAGEOFFSET;
4152 		pcnt = mmu_btopr(window->wd_size + poff);
4153 		ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
4154 
4155 		/* unmap pages which are currently mapped in this window */
4156 		for (i = 0; i < pcnt; i++) {
4157 			if (dma->dp_pgmap[pidx].pm_mapped) {
4158 				hat_unload(kas.a_hat,
4159 				    dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
4160 				    HAT_UNLOAD);
4161 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
4162 			}
4163 			pidx++;
4164 		}
4165 	}
4166 #endif
4167 
4168 	/*
4169 	 * Move to the new window.
4170 	 * NOTE: current_win must be set for sync to work right
4171 	 */
4172 	dma->dp_current_win = win;
4173 	window = &dma->dp_window[win];
4174 
4175 	/* if needed, adjust the first and/or last cookies for trim */
4176 	trim = &window->wd_trim;
4177 	if (trim->tr_trim_first) {
4178 		window->wd_first_cookie->_dmu._dmac_ll = trim->tr_first_paddr;
4179 		window->wd_first_cookie->dmac_size = trim->tr_first_size;
4180 #if !defined(__amd64)
4181 		window->wd_first_cookie->dmac_type =
4182 		    (window->wd_first_cookie->dmac_type &
4183 		    ROOTNEX_USES_COPYBUF) + window->wd_offset;
4184 #endif
4185 		if (trim->tr_first_copybuf_win) {
4186 			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
4187 			    trim->tr_first_cbaddr;
4188 #if !defined(__amd64)
4189 			dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
4190 			    trim->tr_first_kaddr;
4191 #endif
4192 		}
4193 	}
4194 	if (trim->tr_trim_last) {
4195 		trim->tr_last_cookie->_dmu._dmac_ll = trim->tr_last_paddr;
4196 		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
4197 		if (trim->tr_last_copybuf_win) {
4198 			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
4199 			    trim->tr_last_cbaddr;
4200 #if !defined(__amd64)
4201 			dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
4202 			    trim->tr_last_kaddr;
4203 #endif
4204 		}
4205 	}
4206 
4207 	/*
4208 	 * setup the cookie pointer to the first cookie in the window. setup
4209 	 * our return values, then increment the cookie since we return the
4210 	 * first cookie on the stack.
4211 	 */
4212 	hp->dmai_cookie = window->wd_first_cookie;
4213 	*offp = window->wd_offset;
4214 	*lenp = window->wd_size;
4215 	*ccountp = window->wd_cookie_cnt;
4216 	*cookiep = hp->dmai_cookie[0];
4217 	hp->dmai_cookie++;
4218 
4219 #if !defined(__amd64)
4220 	/* re-map copybuf if required for this window */
4221 	if (dma->dp_cb_remaping) {
4222 		/*
4223 		 * calculate the page index into the buffer where this
4224 		 * window starts.
4225 		 */
4226 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
4227 		    MMU_PAGESHIFT;
4228 		ASSERT(pidx < sinfo->si_max_pages);
4229 
4230 		/*
4231 		 * the first page can get unmapped if it's shared with the
4232 		 * previous window. Even if the rest of this window is already
4233 		 * mapped in, we need to still check this one.
4234 		 */
4235 		pmap = &dma->dp_pgmap[pidx];
4236 		if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
4237 			if (pmap->pm_pp != NULL) {
4238 				pmap->pm_mapped = B_TRUE;
4239 				i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
4240 			} else if (pmap->pm_vaddr != NULL) {
4241 				pmap->pm_mapped = B_TRUE;
4242 				i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
4243 				    pmap->pm_kaddr);
4244 			}
4245 		}
4246 		pidx++;
4247 
4248 		/* map in the rest of the pages if required */
4249 		if (window->wd_remap_copybuf) {
4250 			window->wd_remap_copybuf = B_FALSE;
4251 
4252 			/* figure out many pages this window takes up */
4253 			poff = (sinfo->si_buf_offset + window->wd_offset) &
4254 			    MMU_PAGEOFFSET;
4255 			pcnt = mmu_btopr(window->wd_size + poff);
4256 			ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4257 
4258 			/* map pages which require it */
4259 			for (i = 1; i < pcnt; i++) {
4260 				pmap = &dma->dp_pgmap[pidx];
4261 				if (pmap->pm_uses_copybuf) {
4262 					ASSERT(pmap->pm_mapped == B_FALSE);
4263 					if (pmap->pm_pp != NULL) {
4264 						pmap->pm_mapped = B_TRUE;
4265 						i86_pp_map(pmap->pm_pp,
4266 						    pmap->pm_kaddr);
4267 					} else if (pmap->pm_vaddr != NULL) {
4268 						pmap->pm_mapped = B_TRUE;
4269 						i86_va_map(pmap->pm_vaddr,
4270 						    sinfo->si_asp,
4271 						    pmap->pm_kaddr);
4272 					}
4273 				}
4274 				pidx++;
4275 			}
4276 		}
4277 	}
4278 #endif
4279 
4280 	/* if the new window uses the copy buffer, sync it for the device */
4281 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4282 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
4283 		    DDI_DMA_SYNC_FORDEV);
4284 	}
4285 
4286 	return (DDI_SUCCESS);
4287 }
4288 
4289 
4290 
4291 /*
4292  * ************************
4293  *  obsoleted dma routines
4294  * ************************
4295  */
4296 
4297 /*
4298  * rootnex_dma_map()
4299  *    called from ddi_dma_setup()
4300  */
4301 /* ARGSUSED */
4302 static int
4303 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq,
4304     ddi_dma_handle_t *handlep)
4305 {
4306 #if defined(__amd64)
4307 	/*
4308 	 * this interface is not supported in 64-bit x86 kernel. See comment in
4309 	 * rootnex_dma_mctl()
4310 	 */
4311 	ASSERT(0);
4312 	return (DDI_DMA_NORESOURCES);
4313 
4314 #else /* 32-bit x86 kernel */
4315 	ddi_dma_handle_t *lhandlep;
4316 	ddi_dma_handle_t lhandle;
4317 	ddi_dma_cookie_t cookie;
4318 	ddi_dma_attr_t dma_attr;
4319 	ddi_dma_lim_t *dma_lim;
4320 	uint_t ccnt;
4321 	int e;
4322 
4323 
4324 	/*
4325 	 * if the driver is just testing to see if it's possible to do the bind,
4326 	 * we'll use local state. Otherwise, use the handle pointer passed in.
4327 	 */
4328 	if (handlep == NULL) {
4329 		lhandlep = &lhandle;
4330 	} else {
4331 		lhandlep = handlep;
4332 	}
4333 
4334 	/* convert the limit structure to a dma_attr one */
4335 	dma_lim = dmareq->dmar_limits;
4336 	dma_attr.dma_attr_version = DMA_ATTR_V0;
4337 	dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo;
4338 	dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi;
4339 	dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer;
4340 	dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max;
4341 	dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max;
4342 	dma_attr.dma_attr_granular = dma_lim->dlim_granular;
4343 	dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen;
4344 	dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize;
4345 	dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes;
4346 	dma_attr.dma_attr_align = MMU_PAGESIZE;
4347 	dma_attr.dma_attr_flags = 0;
4348 
4349 	e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp,
4350 	    dmareq->dmar_arg, lhandlep);
4351 	if (e != DDI_SUCCESS) {
4352 		return (e);
4353 	}
4354 
4355 	e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt);
4356 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
4357 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4358 		return (e);
4359 	}
4360 
4361 	/*
4362 	 * if the driver is just testing to see if it's possible to do the bind,
4363 	 * free up the local state and return the result.
4364 	 */
4365 	if (handlep == NULL) {
4366 		(void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep);
4367 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4368 		if (e == DDI_DMA_MAPPED) {
4369 			return (DDI_DMA_MAPOK);
4370 		} else {
4371 			return (DDI_DMA_NOMAPPING);
4372 		}
4373 	}
4374 
4375 	return (e);
4376 #endif /* defined(__amd64) */
4377 }
4378 
4379 
4380 /*
4381  * rootnex_dma_mctl()
4382  *
4383  */
4384 /* ARGSUSED */
4385 static int
4386 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4387     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
4388     uint_t cache_flags)
4389 {
4390 #if defined(__amd64)
4391 	/*
4392 	 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a
4393 	 * common implementation in genunix, so they no longer have x86
4394 	 * specific functionality which called into dma_ctl.
4395 	 *
4396 	 * The rest of the obsoleted interfaces were never supported in the
4397 	 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface
4398 	 * was not ported to the x86 64-bit kernel do to serious x86 rootnex
4399 	 * implementation issues.
4400 	 *
4401 	 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and
4402 	 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we
4403 	 * reflect that now too...
4404 	 *
4405 	 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are
4406 	 * not going to put this functionality into the 64-bit x86 kernel now.
4407 	 * It wasn't ported to the 64-bit kernel for s10, no reason to change
4408 	 * that in a future release.
4409 	 */
4410 	ASSERT(0);
4411 	return (DDI_FAILURE);
4412 
4413 #else /* 32-bit x86 kernel */
4414 	ddi_dma_cookie_t lcookie;
4415 	ddi_dma_cookie_t *cookie;
4416 	rootnex_window_t *window;
4417 	ddi_dma_impl_t *hp;
4418 	rootnex_dma_t *dma;
4419 	uint_t nwin;
4420 	uint_t ccnt;
4421 	size_t len;
4422 	off_t off;
4423 	int e;
4424 
4425 
4426 	/*
4427 	 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little
4428 	 * hacky since were optimizing for the current interfaces and so we can
4429 	 * cleanup the mess in genunix. Hopefully we will remove the this
4430 	 * obsoleted routines someday soon.
4431 	 */
4432 
4433 	switch (request) {
4434 
4435 	case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */
4436 		hp = (ddi_dma_impl_t *)handle;
4437 		cookie = (ddi_dma_cookie_t *)objpp;
4438 
4439 		/*
4440 		 * convert segment to cookie. We don't distinguish between the
4441 		 * two :-)
4442 		 */
4443 		*cookie = *hp->dmai_cookie;
4444 		*lenp = cookie->dmac_size;
4445 		*offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF;
4446 		return (DDI_SUCCESS);
4447 
4448 	case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */
4449 		hp = (ddi_dma_impl_t *)handle;
4450 		dma = (rootnex_dma_t *)hp->dmai_private;
4451 
4452 		if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) {
4453 			return (DDI_DMA_STALE);
4454 		}
4455 
4456 		/* handle the case where we don't have any windows */
4457 		if (dma->dp_window == NULL) {
4458 			/*
4459 			 * if seg == NULL, and we don't have any windows,
4460 			 * return the first cookie in the sgl.
4461 			 */
4462 			if (*lenp == NULL) {
4463 				dma->dp_current_cookie = 0;
4464 				hp->dmai_cookie = dma->dp_cookies;
4465 				*objpp = (caddr_t)handle;
4466 				return (DDI_SUCCESS);
4467 
4468 			/* if we have more cookies, go to the next cookie */
4469 			} else {
4470 				if ((dma->dp_current_cookie + 1) >=
4471 				    dma->dp_sglinfo.si_sgl_size) {
4472 					return (DDI_DMA_DONE);
4473 				}
4474 				dma->dp_current_cookie++;
4475 				hp->dmai_cookie++;
4476 				return (DDI_SUCCESS);
4477 			}
4478 		}
4479 
4480 		/* We have one or more windows */
4481 		window = &dma->dp_window[dma->dp_current_win];
4482 
4483 		/*
4484 		 * if seg == NULL, return the first cookie in the current
4485 		 * window
4486 		 */
4487 		if (*lenp == NULL) {
4488 			dma->dp_current_cookie = 0;
4489 			hp->dmai_cookie = dma->dp_cookies;
4490 
4491 		/*
4492 		 * go to the next cookie in the window then see if we done with
4493 		 * this window.
4494 		 */
4495 		} else {
4496 			if ((dma->dp_current_cookie + 1) >=
4497 			    window->wd_cookie_cnt) {
4498 				return (DDI_DMA_DONE);
4499 			}
4500 			dma->dp_current_cookie++;
4501 			hp->dmai_cookie++;
4502 		}
4503 		*objpp = (caddr_t)handle;
4504 		return (DDI_SUCCESS);
4505 
4506 	case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */
4507 		hp = (ddi_dma_impl_t *)handle;
4508 		dma = (rootnex_dma_t *)hp->dmai_private;
4509 
4510 		if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) {
4511 			return (DDI_DMA_STALE);
4512 		}
4513 
4514 		/* if win == NULL, return the first window in the bind */
4515 		if (*offp == NULL) {
4516 			nwin = 0;
4517 
4518 		/*
4519 		 * else, go to the next window then see if we're done with all
4520 		 * the windows.
4521 		 */
4522 		} else {
4523 			nwin = dma->dp_current_win + 1;
4524 			if (nwin >= hp->dmai_nwin) {
4525 				return (DDI_DMA_DONE);
4526 			}
4527 		}
4528 
4529 		/* switch to the next window */
4530 		e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len,
4531 		    &lcookie, &ccnt);
4532 		ASSERT(e == DDI_SUCCESS);
4533 		if (e != DDI_SUCCESS) {
4534 			return (DDI_DMA_STALE);
4535 		}
4536 
4537 		/* reset the cookie back to the first cookie in the window */
4538 		if (dma->dp_window != NULL) {
4539 			window = &dma->dp_window[dma->dp_current_win];
4540 			hp->dmai_cookie = window->wd_first_cookie;
4541 		} else {
4542 			hp->dmai_cookie = dma->dp_cookies;
4543 		}
4544 
4545 		*objpp = (caddr_t)handle;
4546 		return (DDI_SUCCESS);
4547 
4548 	case DDI_DMA_FREE: /* ddi_dma_free() */
4549 		(void) rootnex_dma_unbindhdl(dip, rdip, handle);
4550 		(void) rootnex_dma_freehdl(dip, rdip, handle);
4551 		if (rootnex_state->r_dvma_call_list_id) {
4552 			ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
4553 		}
4554 		return (DDI_SUCCESS);
4555 
4556 	case DDI_DMA_IOPB_ALLOC:	/* get contiguous DMA-able memory */
4557 	case DDI_DMA_SMEM_ALLOC:	/* get contiguous DMA-able memory */
4558 		/* should never get here, handled in genunix */
4559 		ASSERT(0);
4560 		return (DDI_FAILURE);
4561 
4562 	case DDI_DMA_KVADDR:
4563 	case DDI_DMA_GETERR:
4564 	case DDI_DMA_COFF:
4565 		return (DDI_FAILURE);
4566 	}
4567 
4568 	return (DDI_FAILURE);
4569 #endif /* defined(__amd64) */
4570 }
4571