xref: /titanic_44/usr/src/uts/i86pc/io/rootnex.c (revision 18c2aff776a775d34a4c9893a4c72e0434d68e36)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 /*
29  * x86 root nexus driver
30  */
31 
32 #include <sys/sysmacros.h>
33 #include <sys/conf.h>
34 #include <sys/autoconf.h>
35 #include <sys/sysmacros.h>
36 #include <sys/debug.h>
37 #include <sys/psw.h>
38 #include <sys/ddidmareq.h>
39 #include <sys/promif.h>
40 #include <sys/devops.h>
41 #include <sys/kmem.h>
42 #include <sys/cmn_err.h>
43 #include <vm/seg.h>
44 #include <vm/seg_kmem.h>
45 #include <vm/seg_dev.h>
46 #include <sys/vmem.h>
47 #include <sys/mman.h>
48 #include <vm/hat.h>
49 #include <vm/as.h>
50 #include <vm/page.h>
51 #include <sys/avintr.h>
52 #include <sys/errno.h>
53 #include <sys/modctl.h>
54 #include <sys/ddi_impldefs.h>
55 #include <sys/sunddi.h>
56 #include <sys/sunndi.h>
57 #include <sys/mach_intr.h>
58 #include <sys/psm.h>
59 #include <sys/ontrap.h>
60 #include <sys/atomic.h>
61 #include <sys/sdt.h>
62 #include <sys/rootnex.h>
63 #include <vm/hat_i86.h>
64 #include <sys/ddifm.h>
65 
66 /*
67  * enable/disable extra checking of function parameters. Useful for debugging
68  * drivers.
69  */
70 #ifdef	DEBUG
71 int rootnex_alloc_check_parms = 1;
72 int rootnex_bind_check_parms = 1;
73 int rootnex_bind_check_inuse = 1;
74 int rootnex_unbind_verify_buffer = 0;
75 int rootnex_sync_check_parms = 1;
76 #else
77 int rootnex_alloc_check_parms = 0;
78 int rootnex_bind_check_parms = 0;
79 int rootnex_bind_check_inuse = 0;
80 int rootnex_unbind_verify_buffer = 0;
81 int rootnex_sync_check_parms = 0;
82 #endif
83 
84 /* Master Abort and Target Abort panic flag */
85 int rootnex_fm_ma_ta_panic_flag = 0;
86 
87 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */
88 int rootnex_bind_fail = 1;
89 int rootnex_bind_warn = 1;
90 uint8_t *rootnex_warn_list;
91 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */
92 #define	ROOTNEX_BIND_WARNING	(0x1 << 0)
93 
94 /*
95  * revert back to old broken behavior of always sync'ing entire copy buffer.
96  * This is useful if be have a buggy driver which doesn't correctly pass in
97  * the offset and size into ddi_dma_sync().
98  */
99 int rootnex_sync_ignore_params = 0;
100 
101 /*
102  * maximum size that we will allow for a copy buffer. Can be patched on the
103  * fly
104  */
105 size_t rootnex_max_copybuf_size = 0x100000;
106 
107 /*
108  * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1
109  * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a
110  * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit
111  * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65
112  * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages
113  * (< 8K). We will still need to allocate the copy buffer during bind though
114  * (if we need one). These can only be modified in /etc/system before rootnex
115  * attach.
116  */
117 #if defined(__amd64)
118 int rootnex_prealloc_cookies = 65;
119 int rootnex_prealloc_windows = 4;
120 int rootnex_prealloc_copybuf = 2;
121 #else
122 int rootnex_prealloc_cookies = 33;
123 int rootnex_prealloc_windows = 4;
124 int rootnex_prealloc_copybuf = 2;
125 #endif
126 
127 /* driver global state */
128 static rootnex_state_t *rootnex_state;
129 
130 /* shortcut to rootnex counters */
131 static uint64_t *rootnex_cnt;
132 
133 /*
134  * XXX - does x86 even need these or are they left over from the SPARC days?
135  */
136 /* statically defined integer/boolean properties for the root node */
137 static rootnex_intprop_t rootnex_intprp[] = {
138 	{ "PAGESIZE",			PAGESIZE },
139 	{ "MMU_PAGESIZE",		MMU_PAGESIZE },
140 	{ "MMU_PAGEOFFSET",		MMU_PAGEOFFSET },
141 	{ DDI_RELATIVE_ADDRESSING,	1 },
142 };
143 #define	NROOT_INTPROPS	(sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t))
144 
145 
146 static struct cb_ops rootnex_cb_ops = {
147 	nodev,		/* open */
148 	nodev,		/* close */
149 	nodev,		/* strategy */
150 	nodev,		/* print */
151 	nodev,		/* dump */
152 	nodev,		/* read */
153 	nodev,		/* write */
154 	nodev,		/* ioctl */
155 	nodev,		/* devmap */
156 	nodev,		/* mmap */
157 	nodev,		/* segmap */
158 	nochpoll,	/* chpoll */
159 	ddi_prop_op,	/* cb_prop_op */
160 	NULL,		/* struct streamtab */
161 	D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */
162 	CB_REV,		/* Rev */
163 	nodev,		/* cb_aread */
164 	nodev		/* cb_awrite */
165 };
166 
167 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp,
168     off_t offset, off_t len, caddr_t *vaddrp);
169 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip,
170     struct hat *hat, struct seg *seg, caddr_t addr,
171     struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock);
172 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip,
173     struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep);
174 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip,
175     ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg,
176     ddi_dma_handle_t *handlep);
177 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip,
178     ddi_dma_handle_t handle);
179 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip,
180     ddi_dma_handle_t handle, struct ddi_dma_req *dmareq,
181     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
182 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
183     ddi_dma_handle_t handle);
184 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip,
185     ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags);
186 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip,
187     ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp,
188     ddi_dma_cookie_t *cookiep, uint_t *ccountp);
189 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip,
190     ddi_dma_handle_t handle, enum ddi_dma_ctlops request,
191     off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags);
192 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip,
193     ddi_ctl_enum_t ctlop, void *arg, void *result);
194 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
195     ddi_iblock_cookie_t *ibc);
196 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip,
197     ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result);
198 
199 
200 static struct bus_ops rootnex_bus_ops = {
201 	BUSO_REV,
202 	rootnex_map,
203 	NULL,
204 	NULL,
205 	NULL,
206 	rootnex_map_fault,
207 	rootnex_dma_map,
208 	rootnex_dma_allochdl,
209 	rootnex_dma_freehdl,
210 	rootnex_dma_bindhdl,
211 	rootnex_dma_unbindhdl,
212 	rootnex_dma_sync,
213 	rootnex_dma_win,
214 	rootnex_dma_mctl,
215 	rootnex_ctlops,
216 	ddi_bus_prop_op,
217 	i_ddi_rootnex_get_eventcookie,
218 	i_ddi_rootnex_add_eventcall,
219 	i_ddi_rootnex_remove_eventcall,
220 	i_ddi_rootnex_post_event,
221 	0,			/* bus_intr_ctl */
222 	0,			/* bus_config */
223 	0,			/* bus_unconfig */
224 	rootnex_fm_init,	/* bus_fm_init */
225 	NULL,			/* bus_fm_fini */
226 	NULL,			/* bus_fm_access_enter */
227 	NULL,			/* bus_fm_access_exit */
228 	NULL,			/* bus_powr */
229 	rootnex_intr_ops	/* bus_intr_op */
230 };
231 
232 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
233 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
234 
235 static struct dev_ops rootnex_ops = {
236 	DEVO_REV,
237 	0,
238 	ddi_no_info,
239 	nulldev,
240 	nulldev,
241 	rootnex_attach,
242 	rootnex_detach,
243 	nulldev,
244 	&rootnex_cb_ops,
245 	&rootnex_bus_ops
246 };
247 
248 static struct modldrv rootnex_modldrv = {
249 	&mod_driverops,
250 	"i86pc root nexus %I%",
251 	&rootnex_ops
252 };
253 
254 static struct modlinkage rootnex_modlinkage = {
255 	MODREV_1,
256 	(void *)&rootnex_modldrv,
257 	NULL
258 };
259 
260 
261 /*
262  *  extern hacks
263  */
264 extern struct seg_ops segdev_ops;
265 extern int ignore_hardware_nodes;	/* force flag from ddi_impl.c */
266 #ifdef	DDI_MAP_DEBUG
267 extern int ddi_map_debug_flag;
268 #define	ddi_map_debug	if (ddi_map_debug_flag) prom_printf
269 #endif
270 #define	ptob64(x)	(((uint64_t)(x)) << MMU_PAGESHIFT)
271 extern void i86_pp_map(page_t *pp, caddr_t kaddr);
272 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr);
273 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *,
274     psm_intr_op_t, int *);
275 extern int impl_ddi_sunbus_initchild(dev_info_t *dip);
276 extern void impl_ddi_sunbus_removechild(dev_info_t *dip);
277 /*
278  * Use device arena to use for device control register mappings.
279  * Various kernel memory walkers (debugger, dtrace) need to know
280  * to avoid this address range to prevent undesired device activity.
281  */
282 extern void *device_arena_alloc(size_t size, int vm_flag);
283 extern void device_arena_free(void * vaddr, size_t size);
284 
285 
286 /*
287  *  Internal functions
288  */
289 static int rootnex_dma_init();
290 static void rootnex_add_props(dev_info_t *);
291 static int rootnex_ctl_reportdev(dev_info_t *dip);
292 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum);
293 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
294 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp);
295 static int rootnex_map_handle(ddi_map_req_t *mp);
296 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp);
297 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize);
298 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq,
299     ddi_dma_attr_t *attr);
300 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
301     rootnex_sglinfo_t *sglinfo);
302 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
303     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag);
304 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
305     rootnex_dma_t *dma, ddi_dma_attr_t *attr);
306 static void rootnex_teardown_copybuf(rootnex_dma_t *dma);
307 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
308     ddi_dma_attr_t *attr, int kmflag);
309 static void rootnex_teardown_windows(rootnex_dma_t *dma);
310 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
311     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset);
312 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object,
313     rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset,
314     size_t *copybuf_used, page_t **cur_pp);
315 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp,
316     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie,
317     ddi_dma_attr_t *attr, off_t cur_offset);
318 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp,
319     rootnex_dma_t *dma, rootnex_window_t **windowp,
320     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used);
321 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp,
322     rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie);
323 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
324     off_t offset, size_t size, uint_t cache_flags);
325 static int rootnex_verify_buffer(rootnex_dma_t *dma);
326 static int rootnex_dma_check(dev_info_t *dip, const void *handle,
327     const void *comp_addr, const void *not_used);
328 
329 /*
330  * _init()
331  *
332  */
333 int
334 _init(void)
335 {
336 
337 	rootnex_state = NULL;
338 	return (mod_install(&rootnex_modlinkage));
339 }
340 
341 
342 /*
343  * _info()
344  *
345  */
346 int
347 _info(struct modinfo *modinfop)
348 {
349 	return (mod_info(&rootnex_modlinkage, modinfop));
350 }
351 
352 
353 /*
354  * _fini()
355  *
356  */
357 int
358 _fini(void)
359 {
360 	return (EBUSY);
361 }
362 
363 
364 /*
365  * rootnex_attach()
366  *
367  */
368 static int
369 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
370 {
371 	int fmcap;
372 	int e;
373 
374 
375 	switch (cmd) {
376 	case DDI_ATTACH:
377 		break;
378 	case DDI_RESUME:
379 		return (DDI_SUCCESS);
380 	default:
381 		return (DDI_FAILURE);
382 	}
383 
384 	/*
385 	 * We should only have one instance of rootnex. Save it away since we
386 	 * don't have an easy way to get it back later.
387 	 */
388 	ASSERT(rootnex_state == NULL);
389 	rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP);
390 
391 	rootnex_state->r_dip = dip;
392 	rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15);
393 	rootnex_state->r_reserved_msg_printed = B_FALSE;
394 	rootnex_cnt = &rootnex_state->r_counters[0];
395 
396 	/*
397 	 * Set minimum fm capability level for i86pc platforms and then
398 	 * initialize error handling. Since we're the rootnex, we don't
399 	 * care what's returned in the fmcap field.
400 	 */
401 	ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE |
402 	    DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE;
403 	fmcap = ddi_system_fmcap;
404 	ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc);
405 
406 	/* initialize DMA related state */
407 	e = rootnex_dma_init();
408 	if (e != DDI_SUCCESS) {
409 		kmem_free(rootnex_state, sizeof (rootnex_state_t));
410 		return (DDI_FAILURE);
411 	}
412 
413 	/* Add static root node properties */
414 	rootnex_add_props(dip);
415 
416 	/* since we can't call ddi_report_dev() */
417 	cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip));
418 
419 	/* Initialize rootnex event handle */
420 	i_ddi_rootnex_init_events(dip);
421 
422 	return (DDI_SUCCESS);
423 }
424 
425 
426 /*
427  * rootnex_detach()
428  *
429  */
430 /*ARGSUSED*/
431 static int
432 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
433 {
434 	switch (cmd) {
435 	case DDI_SUSPEND:
436 		break;
437 	default:
438 		return (DDI_FAILURE);
439 	}
440 
441 	return (DDI_SUCCESS);
442 }
443 
444 
445 /*
446  * rootnex_dma_init()
447  *
448  */
449 /*ARGSUSED*/
450 static int
451 rootnex_dma_init()
452 {
453 	size_t bufsize;
454 
455 
456 	/*
457 	 * size of our cookie/window/copybuf state needed in dma bind that we
458 	 * pre-alloc in dma_alloc_handle
459 	 */
460 	rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies;
461 	rootnex_state->r_prealloc_size =
462 	    (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) +
463 	    (rootnex_prealloc_windows * sizeof (rootnex_window_t)) +
464 	    (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t));
465 
466 	/*
467 	 * setup DDI DMA handle kmem cache, align each handle on 64 bytes,
468 	 * allocate 16 extra bytes for struct pointer alignment
469 	 * (p->dmai_private & dma->dp_prealloc_buffer)
470 	 */
471 	bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) +
472 	    rootnex_state->r_prealloc_size + 0x10;
473 	rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl",
474 	    bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0);
475 	if (rootnex_state->r_dmahdl_cache == NULL) {
476 		return (DDI_FAILURE);
477 	}
478 
479 	/*
480 	 * allocate array to track which major numbers we have printed warnings
481 	 * for.
482 	 */
483 	rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list),
484 	    KM_SLEEP);
485 
486 	return (DDI_SUCCESS);
487 }
488 
489 
490 /*
491  * rootnex_add_props()
492  *
493  */
494 static void
495 rootnex_add_props(dev_info_t *dip)
496 {
497 	rootnex_intprop_t *rpp;
498 	int i;
499 
500 	/* Add static integer/boolean properties to the root node */
501 	rpp = rootnex_intprp;
502 	for (i = 0; i < NROOT_INTPROPS; i++) {
503 		(void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip,
504 		    rpp[i].prop_name, rpp[i].prop_value);
505 	}
506 }
507 
508 
509 
510 /*
511  * *************************
512  *  ctlops related routines
513  * *************************
514  */
515 
516 /*
517  * rootnex_ctlops()
518  *
519  */
520 /*ARGSUSED*/
521 static int
522 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
523     void *arg, void *result)
524 {
525 	int n, *ptr;
526 	struct ddi_parent_private_data *pdp;
527 
528 	switch (ctlop) {
529 	case DDI_CTLOPS_DMAPMAPC:
530 		/*
531 		 * Return 'partial' to indicate that dma mapping
532 		 * has to be done in the main MMU.
533 		 */
534 		return (DDI_DMA_PARTIAL);
535 
536 	case DDI_CTLOPS_BTOP:
537 		/*
538 		 * Convert byte count input to physical page units.
539 		 * (byte counts that are not a page-size multiple
540 		 * are rounded down)
541 		 */
542 		*(ulong_t *)result = btop(*(ulong_t *)arg);
543 		return (DDI_SUCCESS);
544 
545 	case DDI_CTLOPS_PTOB:
546 		/*
547 		 * Convert size in physical pages to bytes
548 		 */
549 		*(ulong_t *)result = ptob(*(ulong_t *)arg);
550 		return (DDI_SUCCESS);
551 
552 	case DDI_CTLOPS_BTOPR:
553 		/*
554 		 * Convert byte count input to physical page units
555 		 * (byte counts that are not a page-size multiple
556 		 * are rounded up)
557 		 */
558 		*(ulong_t *)result = btopr(*(ulong_t *)arg);
559 		return (DDI_SUCCESS);
560 
561 	case DDI_CTLOPS_INITCHILD:
562 		return (impl_ddi_sunbus_initchild(arg));
563 
564 	case DDI_CTLOPS_UNINITCHILD:
565 		impl_ddi_sunbus_removechild(arg);
566 		return (DDI_SUCCESS);
567 
568 	case DDI_CTLOPS_REPORTDEV:
569 		return (rootnex_ctl_reportdev(rdip));
570 
571 	case DDI_CTLOPS_IOMIN:
572 		/*
573 		 * Nothing to do here but reflect back..
574 		 */
575 		return (DDI_SUCCESS);
576 
577 	case DDI_CTLOPS_REGSIZE:
578 	case DDI_CTLOPS_NREGS:
579 		break;
580 
581 	case DDI_CTLOPS_SIDDEV:
582 		if (ndi_dev_is_prom_node(rdip))
583 			return (DDI_SUCCESS);
584 		if (ndi_dev_is_persistent_node(rdip))
585 			return (DDI_SUCCESS);
586 		return (DDI_FAILURE);
587 
588 	case DDI_CTLOPS_POWER:
589 		return ((*pm_platform_power)((power_req_t *)arg));
590 
591 	case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */
592 	case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */
593 	case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */
594 	case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */
595 	case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */
596 	case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */
597 		if (!rootnex_state->r_reserved_msg_printed) {
598 			rootnex_state->r_reserved_msg_printed = B_TRUE;
599 			cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for "
600 			    "1 or more reserved/obsolete operations.");
601 		}
602 		return (DDI_FAILURE);
603 
604 	default:
605 		return (DDI_FAILURE);
606 	}
607 	/*
608 	 * The rest are for "hardware" properties
609 	 */
610 	if ((pdp = ddi_get_parent_data(rdip)) == NULL)
611 		return (DDI_FAILURE);
612 
613 	if (ctlop == DDI_CTLOPS_NREGS) {
614 		ptr = (int *)result;
615 		*ptr = pdp->par_nreg;
616 	} else {
617 		off_t *size = (off_t *)result;
618 
619 		ptr = (int *)arg;
620 		n = *ptr;
621 		if (n >= pdp->par_nreg) {
622 			return (DDI_FAILURE);
623 		}
624 		*size = (off_t)pdp->par_reg[n].regspec_size;
625 	}
626 	return (DDI_SUCCESS);
627 }
628 
629 
630 /*
631  * rootnex_ctl_reportdev()
632  *
633  */
634 static int
635 rootnex_ctl_reportdev(dev_info_t *dev)
636 {
637 	int i, n, len, f_len = 0;
638 	char *buf;
639 
640 	buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP);
641 	f_len += snprintf(buf, REPORTDEV_BUFSIZE,
642 	    "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev));
643 	len = strlen(buf);
644 
645 	for (i = 0; i < sparc_pd_getnreg(dev); i++) {
646 
647 		struct regspec *rp = sparc_pd_getreg(dev, i);
648 
649 		if (i == 0)
650 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
651 			    ": ");
652 		else
653 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
654 			    " and ");
655 		len = strlen(buf);
656 
657 		switch (rp->regspec_bustype) {
658 
659 		case BTEISA:
660 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
661 			    "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr);
662 			break;
663 
664 		case BTISA:
665 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
666 			    "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr);
667 			break;
668 
669 		default:
670 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
671 			    "space %x offset %x",
672 			    rp->regspec_bustype, rp->regspec_addr);
673 			break;
674 		}
675 		len = strlen(buf);
676 	}
677 	for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) {
678 		int pri;
679 
680 		if (i != 0) {
681 			f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
682 			    ",");
683 			len = strlen(buf);
684 		}
685 		pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri);
686 		f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len,
687 		    " sparc ipl %d", pri);
688 		len = strlen(buf);
689 	}
690 #ifdef DEBUG
691 	if (f_len + 1 >= REPORTDEV_BUFSIZE) {
692 		cmn_err(CE_NOTE, "next message is truncated: "
693 		    "printed length 1024, real length %d", f_len);
694 	}
695 #endif /* DEBUG */
696 	cmn_err(CE_CONT, "?%s\n", buf);
697 	kmem_free(buf, REPORTDEV_BUFSIZE);
698 	return (DDI_SUCCESS);
699 }
700 
701 
702 /*
703  * ******************
704  *  map related code
705  * ******************
706  */
707 
708 /*
709  * rootnex_map()
710  *
711  */
712 static int
713 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset,
714     off_t len, caddr_t *vaddrp)
715 {
716 	struct regspec *rp, tmp_reg;
717 	ddi_map_req_t mr = *mp;		/* Get private copy of request */
718 	int error;
719 
720 	mp = &mr;
721 
722 	switch (mp->map_op)  {
723 	case DDI_MO_MAP_LOCKED:
724 	case DDI_MO_UNMAP:
725 	case DDI_MO_MAP_HANDLE:
726 		break;
727 	default:
728 #ifdef	DDI_MAP_DEBUG
729 		cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.",
730 		    mp->map_op);
731 #endif	/* DDI_MAP_DEBUG */
732 		return (DDI_ME_UNIMPLEMENTED);
733 	}
734 
735 	if (mp->map_flags & DDI_MF_USER_MAPPING)  {
736 #ifdef	DDI_MAP_DEBUG
737 		cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user.");
738 #endif	/* DDI_MAP_DEBUG */
739 		return (DDI_ME_UNIMPLEMENTED);
740 	}
741 
742 	/*
743 	 * First, if given an rnumber, convert it to a regspec...
744 	 * (Presumably, this is on behalf of a child of the root node?)
745 	 */
746 
747 	if (mp->map_type == DDI_MT_RNUMBER)  {
748 
749 		int rnumber = mp->map_obj.rnumber;
750 #ifdef	DDI_MAP_DEBUG
751 		static char *out_of_range =
752 		    "rootnex_map: Out of range rnumber <%d>, device <%s>";
753 #endif	/* DDI_MAP_DEBUG */
754 
755 		rp = i_ddi_rnumber_to_regspec(rdip, rnumber);
756 		if (rp == NULL)  {
757 #ifdef	DDI_MAP_DEBUG
758 			cmn_err(CE_WARN, out_of_range, rnumber,
759 			    ddi_get_name(rdip));
760 #endif	/* DDI_MAP_DEBUG */
761 			return (DDI_ME_RNUMBER_RANGE);
762 		}
763 
764 		/*
765 		 * Convert the given ddi_map_req_t from rnumber to regspec...
766 		 */
767 
768 		mp->map_type = DDI_MT_REGSPEC;
769 		mp->map_obj.rp = rp;
770 	}
771 
772 	/*
773 	 * Adjust offset and length correspnding to called values...
774 	 * XXX: A non-zero length means override the one in the regspec
775 	 * XXX: (regardless of what's in the parent's range?)
776 	 */
777 
778 	tmp_reg = *(mp->map_obj.rp);		/* Preserve underlying data */
779 	rp = mp->map_obj.rp = &tmp_reg;		/* Use tmp_reg in request */
780 
781 #ifdef	DDI_MAP_DEBUG
782 	cmn_err(CE_CONT,
783 		"rootnex: <%s,%s> <0x%x, 0x%x, 0x%d>"
784 		" offset %d len %d handle 0x%x\n",
785 		ddi_get_name(dip), ddi_get_name(rdip),
786 		rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
787 		offset, len, mp->map_handlep);
788 #endif	/* DDI_MAP_DEBUG */
789 
790 	/*
791 	 * I/O or memory mapping:
792 	 *
793 	 *	<bustype=0, addr=x, len=x>: memory
794 	 *	<bustype=1, addr=x, len=x>: i/o
795 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
796 	 */
797 
798 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
799 		cmn_err(CE_WARN, "<%s,%s> invalid register spec"
800 		    " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip),
801 		    ddi_get_name(rdip), rp->regspec_bustype,
802 		    rp->regspec_addr, rp->regspec_size);
803 		return (DDI_ME_INVAL);
804 	}
805 
806 	if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) {
807 		/*
808 		 * compatibility i/o mapping
809 		 */
810 		rp->regspec_bustype += (uint_t)offset;
811 	} else {
812 		/*
813 		 * Normal memory or i/o mapping
814 		 */
815 		rp->regspec_addr += (uint_t)offset;
816 	}
817 
818 	if (len != 0)
819 		rp->regspec_size = (uint_t)len;
820 
821 #ifdef	DDI_MAP_DEBUG
822 	cmn_err(CE_CONT,
823 		"             <%s,%s> <0x%x, 0x%x, 0x%d>"
824 		" offset %d len %d handle 0x%x\n",
825 		ddi_get_name(dip), ddi_get_name(rdip),
826 		rp->regspec_bustype, rp->regspec_addr, rp->regspec_size,
827 		offset, len, mp->map_handlep);
828 #endif	/* DDI_MAP_DEBUG */
829 
830 	/*
831 	 * Apply any parent ranges at this level, if applicable.
832 	 * (This is where nexus specific regspec translation takes place.
833 	 * Use of this function is implicit agreement that translation is
834 	 * provided via ddi_apply_range.)
835 	 */
836 
837 #ifdef	DDI_MAP_DEBUG
838 	ddi_map_debug("applying range of parent <%s> to child <%s>...\n",
839 	    ddi_get_name(dip), ddi_get_name(rdip));
840 #endif	/* DDI_MAP_DEBUG */
841 
842 	if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0)
843 		return (error);
844 
845 	switch (mp->map_op)  {
846 	case DDI_MO_MAP_LOCKED:
847 
848 		/*
849 		 * Set up the locked down kernel mapping to the regspec...
850 		 */
851 
852 		return (rootnex_map_regspec(mp, vaddrp));
853 
854 	case DDI_MO_UNMAP:
855 
856 		/*
857 		 * Release mapping...
858 		 */
859 
860 		return (rootnex_unmap_regspec(mp, vaddrp));
861 
862 	case DDI_MO_MAP_HANDLE:
863 
864 		return (rootnex_map_handle(mp));
865 
866 	default:
867 		return (DDI_ME_UNIMPLEMENTED);
868 	}
869 }
870 
871 
872 /*
873  * rootnex_map_fault()
874  *
875  *	fault in mappings for requestors
876  */
877 /*ARGSUSED*/
878 static int
879 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat,
880     struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot,
881     uint_t lock)
882 {
883 
884 #ifdef	DDI_MAP_DEBUG
885 	ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn);
886 	ddi_map_debug(" Seg <%s>\n",
887 	    seg->s_ops == &segdev_ops ? "segdev" :
888 	    seg == &kvseg ? "segkmem" : "NONE!");
889 #endif	/* DDI_MAP_DEBUG */
890 
891 	/*
892 	 * This is all terribly broken, but it is a start
893 	 *
894 	 * XXX	Note that this test means that segdev_ops
895 	 *	must be exported from seg_dev.c.
896 	 * XXX	What about devices with their own segment drivers?
897 	 */
898 	if (seg->s_ops == &segdev_ops) {
899 		struct segdev_data *sdp =
900 			(struct segdev_data *)seg->s_data;
901 
902 		if (hat == NULL) {
903 			/*
904 			 * This is one plausible interpretation of
905 			 * a null hat i.e. use the first hat on the
906 			 * address space hat list which by convention is
907 			 * the hat of the system MMU.  At alternative
908 			 * would be to panic .. this might well be better ..
909 			 */
910 			ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock));
911 			hat = seg->s_as->a_hat;
912 			cmn_err(CE_NOTE, "rootnex_map_fault: nil hat");
913 		}
914 		hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr,
915 		    (lock ? HAT_LOAD_LOCK : HAT_LOAD));
916 	} else if (seg == &kvseg && dp == NULL) {
917 		hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot,
918 		    HAT_LOAD_LOCK);
919 	} else
920 		return (DDI_FAILURE);
921 	return (DDI_SUCCESS);
922 }
923 
924 
925 /*
926  * rootnex_map_regspec()
927  *     we don't support mapping of I/O cards above 4Gb
928  */
929 static int
930 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
931 {
932 	ulong_t base;
933 	void *cvaddr;
934 	uint_t npages, pgoffset;
935 	struct regspec *rp;
936 	ddi_acc_hdl_t *hp;
937 	ddi_acc_impl_t *ap;
938 	uint_t	hat_acc_flags;
939 
940 	rp = mp->map_obj.rp;
941 	hp = mp->map_handlep;
942 
943 #ifdef	DDI_MAP_DEBUG
944 	ddi_map_debug(
945 	    "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n",
946 	    rp->regspec_bustype, rp->regspec_addr,
947 	    rp->regspec_size, mp->map_handlep);
948 #endif	/* DDI_MAP_DEBUG */
949 
950 	/*
951 	 * I/O or memory mapping
952 	 *
953 	 *	<bustype=0, addr=x, len=x>: memory
954 	 *	<bustype=1, addr=x, len=x>: i/o
955 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
956 	 */
957 
958 	if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) {
959 		cmn_err(CE_WARN, "rootnex: invalid register spec"
960 		    " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype,
961 		    rp->regspec_addr, rp->regspec_size);
962 		return (DDI_FAILURE);
963 	}
964 
965 	if (rp->regspec_bustype != 0) {
966 		/*
967 		 * I/O space - needs a handle.
968 		 */
969 		if (hp == NULL) {
970 			return (DDI_FAILURE);
971 		}
972 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
973 		ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE;
974 		impl_acc_hdl_init(hp);
975 
976 		if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
977 #ifdef  DDI_MAP_DEBUG
978 			ddi_map_debug("rootnex_map_regspec: mmap() \
979 to I/O space is not supported.\n");
980 #endif  /* DDI_MAP_DEBUG */
981 			return (DDI_ME_INVAL);
982 		} else {
983 			/*
984 			 * 1275-compliant vs. compatibility i/o mapping
985 			 */
986 			*vaddrp =
987 			    (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ?
988 				((caddr_t)(uintptr_t)rp->regspec_bustype) :
989 				((caddr_t)(uintptr_t)rp->regspec_addr);
990 
991 			hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr &
992 			    (~MMU_PAGEOFFSET));
993 			hp->ah_pnum = mmu_btopr(rp->regspec_size +
994 			    (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET);
995 		}
996 
997 #ifdef	DDI_MAP_DEBUG
998 		ddi_map_debug(
999 	    "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n",
1000 		    rp->regspec_size, *vaddrp);
1001 #endif	/* DDI_MAP_DEBUG */
1002 		return (DDI_SUCCESS);
1003 	}
1004 
1005 	/*
1006 	 * Memory space
1007 	 */
1008 
1009 	if (hp != NULL) {
1010 		/*
1011 		 * hat layer ignores
1012 		 * hp->ah_acc.devacc_attr_endian_flags.
1013 		 */
1014 		switch (hp->ah_acc.devacc_attr_dataorder) {
1015 		case DDI_STRICTORDER_ACC:
1016 			hat_acc_flags = HAT_STRICTORDER;
1017 			break;
1018 		case DDI_UNORDERED_OK_ACC:
1019 			hat_acc_flags = HAT_UNORDERED_OK;
1020 			break;
1021 		case DDI_MERGING_OK_ACC:
1022 			hat_acc_flags = HAT_MERGING_OK;
1023 			break;
1024 		case DDI_LOADCACHING_OK_ACC:
1025 			hat_acc_flags = HAT_LOADCACHING_OK;
1026 			break;
1027 		case DDI_STORECACHING_OK_ACC:
1028 			hat_acc_flags = HAT_STORECACHING_OK;
1029 			break;
1030 		}
1031 		ap = (ddi_acc_impl_t *)hp->ah_platform_private;
1032 		ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR;
1033 		impl_acc_hdl_init(hp);
1034 		hp->ah_hat_flags = hat_acc_flags;
1035 	} else {
1036 		hat_acc_flags = HAT_STRICTORDER;
1037 	}
1038 
1039 	base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */
1040 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */
1041 
1042 	if (rp->regspec_size == 0) {
1043 #ifdef  DDI_MAP_DEBUG
1044 		ddi_map_debug("rootnex_map_regspec: zero regspec_size\n");
1045 #endif  /* DDI_MAP_DEBUG */
1046 		return (DDI_ME_INVAL);
1047 	}
1048 
1049 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING) {
1050 		*vaddrp = (caddr_t)mmu_btop(base);
1051 	} else {
1052 		npages = mmu_btopr(rp->regspec_size + pgoffset);
1053 
1054 #ifdef	DDI_MAP_DEBUG
1055 		ddi_map_debug("rootnex_map_regspec: Mapping %d pages \
1056 physical %x ",
1057 		    npages, base);
1058 #endif	/* DDI_MAP_DEBUG */
1059 
1060 		cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP);
1061 		if (cvaddr == NULL)
1062 			return (DDI_ME_NORESOURCES);
1063 
1064 		/*
1065 		 * Now map in the pages we've allocated...
1066 		 */
1067 		hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base),
1068 		    mp->map_prot | hat_acc_flags, HAT_LOAD_LOCK);
1069 		*vaddrp = (caddr_t)cvaddr + pgoffset;
1070 
1071 		/* save away pfn and npages for FMA */
1072 		hp = mp->map_handlep;
1073 		if (hp) {
1074 			hp->ah_pfn = mmu_btop(base);
1075 			hp->ah_pnum = npages;
1076 		}
1077 	}
1078 
1079 #ifdef	DDI_MAP_DEBUG
1080 	ddi_map_debug("at virtual 0x%x\n", *vaddrp);
1081 #endif	/* DDI_MAP_DEBUG */
1082 	return (DDI_SUCCESS);
1083 }
1084 
1085 
1086 /*
1087  * rootnex_unmap_regspec()
1088  *
1089  */
1090 static int
1091 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp)
1092 {
1093 	caddr_t addr = (caddr_t)*vaddrp;
1094 	uint_t npages, pgoffset;
1095 	struct regspec *rp;
1096 
1097 	if (mp->map_flags & DDI_MF_DEVICE_MAPPING)
1098 		return (0);
1099 
1100 	rp = mp->map_obj.rp;
1101 
1102 	if (rp->regspec_size == 0) {
1103 #ifdef  DDI_MAP_DEBUG
1104 		ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n");
1105 #endif  /* DDI_MAP_DEBUG */
1106 		return (DDI_ME_INVAL);
1107 	}
1108 
1109 	/*
1110 	 * I/O or memory mapping:
1111 	 *
1112 	 *	<bustype=0, addr=x, len=x>: memory
1113 	 *	<bustype=1, addr=x, len=x>: i/o
1114 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1115 	 */
1116 	if (rp->regspec_bustype != 0) {
1117 		/*
1118 		 * This is I/O space, which requires no particular
1119 		 * processing on unmap since it isn't mapped in the
1120 		 * first place.
1121 		 */
1122 		return (DDI_SUCCESS);
1123 	}
1124 
1125 	/*
1126 	 * Memory space
1127 	 */
1128 	pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET;
1129 	npages = mmu_btopr(rp->regspec_size + pgoffset);
1130 	hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK);
1131 	device_arena_free(addr - pgoffset, ptob(npages));
1132 
1133 	/*
1134 	 * Destroy the pointer - the mapping has logically gone
1135 	 */
1136 	*vaddrp = NULL;
1137 
1138 	return (DDI_SUCCESS);
1139 }
1140 
1141 
1142 /*
1143  * rootnex_map_handle()
1144  *
1145  */
1146 static int
1147 rootnex_map_handle(ddi_map_req_t *mp)
1148 {
1149 	ddi_acc_hdl_t *hp;
1150 	ulong_t base;
1151 	uint_t pgoffset;
1152 	struct regspec *rp;
1153 
1154 	rp = mp->map_obj.rp;
1155 
1156 #ifdef	DDI_MAP_DEBUG
1157 	ddi_map_debug(
1158 	    "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n",
1159 	    rp->regspec_bustype, rp->regspec_addr,
1160 	    rp->regspec_size, mp->map_handlep);
1161 #endif	/* DDI_MAP_DEBUG */
1162 
1163 	/*
1164 	 * I/O or memory mapping:
1165 	 *
1166 	 *	<bustype=0, addr=x, len=x>: memory
1167 	 *	<bustype=1, addr=x, len=x>: i/o
1168 	 *	<bustype>1, addr=0, len=x>: x86-compatibility i/o
1169 	 */
1170 	if (rp->regspec_bustype != 0) {
1171 		/*
1172 		 * This refers to I/O space, and we don't support "mapping"
1173 		 * I/O space to a user.
1174 		 */
1175 		return (DDI_FAILURE);
1176 	}
1177 
1178 	/*
1179 	 * Set up the hat_flags for the mapping.
1180 	 */
1181 	hp = mp->map_handlep;
1182 
1183 	switch (hp->ah_acc.devacc_attr_endian_flags) {
1184 	case DDI_NEVERSWAP_ACC:
1185 		hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER;
1186 		break;
1187 	case DDI_STRUCTURE_LE_ACC:
1188 		hp->ah_hat_flags = HAT_STRUCTURE_LE;
1189 		break;
1190 	case DDI_STRUCTURE_BE_ACC:
1191 		return (DDI_FAILURE);
1192 	default:
1193 		return (DDI_REGS_ACC_CONFLICT);
1194 	}
1195 
1196 	switch (hp->ah_acc.devacc_attr_dataorder) {
1197 	case DDI_STRICTORDER_ACC:
1198 		break;
1199 	case DDI_UNORDERED_OK_ACC:
1200 		hp->ah_hat_flags |= HAT_UNORDERED_OK;
1201 		break;
1202 	case DDI_MERGING_OK_ACC:
1203 		hp->ah_hat_flags |= HAT_MERGING_OK;
1204 		break;
1205 	case DDI_LOADCACHING_OK_ACC:
1206 		hp->ah_hat_flags |= HAT_LOADCACHING_OK;
1207 		break;
1208 	case DDI_STORECACHING_OK_ACC:
1209 		hp->ah_hat_flags |= HAT_STORECACHING_OK;
1210 		break;
1211 	default:
1212 		return (DDI_FAILURE);
1213 	}
1214 
1215 	base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */
1216 	pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */
1217 
1218 	if (rp->regspec_size == 0)
1219 		return (DDI_ME_INVAL);
1220 
1221 	hp->ah_pfn = mmu_btop(base);
1222 	hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset);
1223 
1224 	return (DDI_SUCCESS);
1225 }
1226 
1227 
1228 
1229 /*
1230  * ************************
1231  *  interrupt related code
1232  * ************************
1233  */
1234 
1235 /*
1236  * rootnex_intr_ops()
1237  *	bus_intr_op() function for interrupt support
1238  */
1239 /* ARGSUSED */
1240 static int
1241 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op,
1242     ddi_intr_handle_impl_t *hdlp, void *result)
1243 {
1244 	struct intrspec			*ispec;
1245 	struct ddi_parent_private_data	*pdp;
1246 
1247 	DDI_INTR_NEXDBG((CE_CONT,
1248 	    "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n",
1249 	    (void *)pdip, (void *)rdip, intr_op, (void *)hdlp));
1250 
1251 	/* Process the interrupt operation */
1252 	switch (intr_op) {
1253 	case DDI_INTROP_GETCAP:
1254 		/* First check with pcplusmp */
1255 		if (psm_intr_ops == NULL)
1256 			return (DDI_FAILURE);
1257 
1258 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) {
1259 			*(int *)result = 0;
1260 			return (DDI_FAILURE);
1261 		}
1262 		break;
1263 	case DDI_INTROP_SETCAP:
1264 		if (psm_intr_ops == NULL)
1265 			return (DDI_FAILURE);
1266 
1267 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result))
1268 			return (DDI_FAILURE);
1269 		break;
1270 	case DDI_INTROP_ALLOC:
1271 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1272 			return (DDI_FAILURE);
1273 		hdlp->ih_pri = ispec->intrspec_pri;
1274 		*(int *)result = hdlp->ih_scratch1;
1275 		break;
1276 	case DDI_INTROP_FREE:
1277 		pdp = ddi_get_parent_data(rdip);
1278 		/*
1279 		 * Special case for 'pcic' driver' only.
1280 		 * If an intrspec was created for it, clean it up here
1281 		 * See detailed comments on this in the function
1282 		 * rootnex_get_ispec().
1283 		 */
1284 		if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1285 			kmem_free(pdp->par_intr, sizeof (struct intrspec) *
1286 			    pdp->par_nintr);
1287 			/*
1288 			 * Set it to zero; so that
1289 			 * DDI framework doesn't free it again
1290 			 */
1291 			pdp->par_intr = NULL;
1292 			pdp->par_nintr = 0;
1293 		}
1294 		break;
1295 	case DDI_INTROP_GETPRI:
1296 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1297 			return (DDI_FAILURE);
1298 		*(int *)result = ispec->intrspec_pri;
1299 		break;
1300 	case DDI_INTROP_SETPRI:
1301 		/* Validate the interrupt priority passed to us */
1302 		if (*(int *)result > LOCK_LEVEL)
1303 			return (DDI_FAILURE);
1304 
1305 		/* Ensure that PSM is all initialized and ispec is ok */
1306 		if ((psm_intr_ops == NULL) ||
1307 		    ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL))
1308 			return (DDI_FAILURE);
1309 
1310 		/* Change the priority */
1311 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) ==
1312 		    PSM_FAILURE)
1313 			return (DDI_FAILURE);
1314 
1315 		/* update the ispec with the new priority */
1316 		ispec->intrspec_pri =  *(int *)result;
1317 		break;
1318 	case DDI_INTROP_ADDISR:
1319 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1320 			return (DDI_FAILURE);
1321 		ispec->intrspec_func = hdlp->ih_cb_func;
1322 		break;
1323 	case DDI_INTROP_REMISR:
1324 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1325 			return (DDI_FAILURE);
1326 		ispec->intrspec_func = (uint_t (*)()) 0;
1327 		break;
1328 	case DDI_INTROP_ENABLE:
1329 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1330 			return (DDI_FAILURE);
1331 
1332 		/* Call psmi to translate irq with the dip */
1333 		if (psm_intr_ops == NULL)
1334 			return (DDI_FAILURE);
1335 
1336 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1337 		(void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR,
1338 		    (int *)&hdlp->ih_vector);
1339 
1340 		/* Add the interrupt handler */
1341 		if (!add_avintr((void *)hdlp, ispec->intrspec_pri,
1342 		    hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector,
1343 		    hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip))
1344 			return (DDI_FAILURE);
1345 		break;
1346 	case DDI_INTROP_DISABLE:
1347 		if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)
1348 			return (DDI_FAILURE);
1349 
1350 		/* Call psm_ops() to translate irq with the dip */
1351 		if (psm_intr_ops == NULL)
1352 			return (DDI_FAILURE);
1353 
1354 		((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec;
1355 		(void) (*psm_intr_ops)(rdip, hdlp,
1356 		    PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector);
1357 
1358 		/* Remove the interrupt handler */
1359 		rem_avintr((void *)hdlp, ispec->intrspec_pri,
1360 		    hdlp->ih_cb_func, hdlp->ih_vector);
1361 		break;
1362 	case DDI_INTROP_SETMASK:
1363 		if (psm_intr_ops == NULL)
1364 			return (DDI_FAILURE);
1365 
1366 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL))
1367 			return (DDI_FAILURE);
1368 		break;
1369 	case DDI_INTROP_CLRMASK:
1370 		if (psm_intr_ops == NULL)
1371 			return (DDI_FAILURE);
1372 
1373 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL))
1374 			return (DDI_FAILURE);
1375 		break;
1376 	case DDI_INTROP_GETPENDING:
1377 		if (psm_intr_ops == NULL)
1378 			return (DDI_FAILURE);
1379 
1380 		if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING,
1381 		    result)) {
1382 			*(int *)result = 0;
1383 			return (DDI_FAILURE);
1384 		}
1385 		break;
1386 	case DDI_INTROP_NAVAIL:
1387 	case DDI_INTROP_NINTRS:
1388 		*(int *)result = i_ddi_get_intx_nintrs(rdip);
1389 		if (*(int *)result == 0) {
1390 			/*
1391 			 * Special case for 'pcic' driver' only. This driver
1392 			 * driver is a child of 'isa' and 'rootnex' drivers.
1393 			 *
1394 			 * See detailed comments on this in the function
1395 			 * rootnex_get_ispec().
1396 			 *
1397 			 * Children of 'pcic' send 'NINITR' request all the
1398 			 * way to rootnex driver. But, the 'pdp->par_nintr'
1399 			 * field may not initialized. So, we fake it here
1400 			 * to return 1 (a la what PCMCIA nexus does).
1401 			 */
1402 			if (strcmp(ddi_get_name(rdip), "pcic") == 0)
1403 				*(int *)result = 1;
1404 			else
1405 				return (DDI_FAILURE);
1406 		}
1407 		break;
1408 	case DDI_INTROP_SUPPORTED_TYPES:
1409 		*(int *)result = DDI_INTR_TYPE_FIXED;	/* Always ... */
1410 		break;
1411 	default:
1412 		return (DDI_FAILURE);
1413 	}
1414 
1415 	return (DDI_SUCCESS);
1416 }
1417 
1418 
1419 /*
1420  * rootnex_get_ispec()
1421  *	convert an interrupt number to an interrupt specification.
1422  *	The interrupt number determines which interrupt spec will be
1423  *	returned if more than one exists.
1424  *
1425  *	Look into the parent private data area of the 'rdip' to find out
1426  *	the interrupt specification.  First check to make sure there is
1427  *	one that matchs "inumber" and then return a pointer to it.
1428  *
1429  *	Return NULL if one could not be found.
1430  *
1431  *	NOTE: This is needed for rootnex_intr_ops()
1432  */
1433 static struct intrspec *
1434 rootnex_get_ispec(dev_info_t *rdip, int inum)
1435 {
1436 	struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip);
1437 
1438 	/*
1439 	 * Special case handling for drivers that provide their own
1440 	 * intrspec structures instead of relying on the DDI framework.
1441 	 *
1442 	 * A broken hardware driver in ON could potentially provide its
1443 	 * own intrspec structure, instead of relying on the hardware.
1444 	 * If these drivers are children of 'rootnex' then we need to
1445 	 * continue to provide backward compatibility to them here.
1446 	 *
1447 	 * Following check is a special case for 'pcic' driver which
1448 	 * was found to have broken hardwre andby provides its own intrspec.
1449 	 *
1450 	 * Verbatim comments from this driver are shown here:
1451 	 * "Don't use the ddi_add_intr since we don't have a
1452 	 * default intrspec in all cases."
1453 	 *
1454 	 * Since an 'ispec' may not be always created for it,
1455 	 * check for that and create one if so.
1456 	 *
1457 	 * NOTE: Currently 'pcic' is the only driver found to do this.
1458 	 */
1459 	if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) {
1460 		pdp->par_nintr = 1;
1461 		pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) *
1462 		    pdp->par_nintr, KM_SLEEP);
1463 	}
1464 
1465 	/* Validate the interrupt number */
1466 	if (inum >= pdp->par_nintr)
1467 		return (NULL);
1468 
1469 	/* Get the interrupt structure pointer and return that */
1470 	return ((struct intrspec *)&pdp->par_intr[inum]);
1471 }
1472 
1473 
1474 /*
1475  * ******************
1476  *  dma related code
1477  * ******************
1478  */
1479 
1480 /*
1481  * rootnex_dma_allochdl()
1482  *    called from ddi_dma_alloc_handle().
1483  */
1484 /*ARGSUSED*/
1485 static int
1486 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr,
1487     int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep)
1488 {
1489 	uint64_t maxsegmentsize_ll;
1490 	uint_t maxsegmentsize;
1491 	ddi_dma_impl_t *hp;
1492 	rootnex_dma_t *dma;
1493 	uint64_t count_max;
1494 	uint64_t seg;
1495 	int kmflag;
1496 	int e;
1497 
1498 
1499 	/* convert our sleep flags */
1500 	if (waitfp == DDI_DMA_SLEEP) {
1501 		kmflag = KM_SLEEP;
1502 	} else {
1503 		kmflag = KM_NOSLEEP;
1504 	}
1505 
1506 	/*
1507 	 * We try to do only one memory allocation here. We'll do a little
1508 	 * pointer manipulation later. If the bind ends up taking more than
1509 	 * our prealloc's space, we'll have to allocate more memory in the
1510 	 * bind operation. Not great, but much better than before and the
1511 	 * best we can do with the current bind interfaces.
1512 	 */
1513 	hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag);
1514 	if (hp == NULL) {
1515 		if (waitfp != DDI_DMA_DONTWAIT) {
1516 			ddi_set_callback(waitfp, arg,
1517 			    &rootnex_state->r_dvma_call_list_id);
1518 		}
1519 		return (DDI_DMA_NORESOURCES);
1520 	}
1521 
1522 	/* Do our pointer manipulation now, align the structures */
1523 	hp->dmai_private = (void *)(((uintptr_t)hp +
1524 	    (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7);
1525 	dma = (rootnex_dma_t *)hp->dmai_private;
1526 	dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma +
1527 	    sizeof (rootnex_dma_t) + 0x7) & ~0x7);
1528 
1529 	/* setup the handle */
1530 	rootnex_clean_dmahdl(hp);
1531 	dma->dp_dip = rdip;
1532 	dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo;
1533 	dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi;
1534 	hp->dmai_minxfer = attr->dma_attr_minxfer;
1535 	hp->dmai_burstsizes = attr->dma_attr_burstsizes;
1536 	hp->dmai_rdip = rdip;
1537 	hp->dmai_attr = *attr;
1538 
1539 	/* we don't need to worry about the SPL since we do a tryenter */
1540 	mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL);
1541 
1542 	/*
1543 	 * Figure out our maximum segment size. If the segment size is greater
1544 	 * than 4G, we will limit it to (4G - 1) since the max size of a dma
1545 	 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and
1546 	 * dma_attr_count_max are size-1 type values.
1547 	 *
1548 	 * Maximum segment size is the largest physically contiguous chunk of
1549 	 * memory that we can return from a bind (i.e. the maximum size of a
1550 	 * single cookie).
1551 	 */
1552 
1553 	/* handle the rollover cases */
1554 	seg = attr->dma_attr_seg + 1;
1555 	if (seg < attr->dma_attr_seg) {
1556 		seg = attr->dma_attr_seg;
1557 	}
1558 	count_max = attr->dma_attr_count_max + 1;
1559 	if (count_max < attr->dma_attr_count_max) {
1560 		count_max = attr->dma_attr_count_max;
1561 	}
1562 
1563 	/*
1564 	 * granularity may or may not be a power of two. If it isn't, we can't
1565 	 * use a simple mask.
1566 	 */
1567 	if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) {
1568 		dma->dp_granularity_power_2 = B_FALSE;
1569 	} else {
1570 		dma->dp_granularity_power_2 = B_TRUE;
1571 	}
1572 
1573 	/*
1574 	 * maxxfer should be a whole multiple of granularity. If we're going to
1575 	 * break up a window because we're greater than maxxfer, we might as
1576 	 * well make sure it's maxxfer is a whole multiple so we don't have to
1577 	 * worry about triming the window later on for this case.
1578 	 */
1579 	if (attr->dma_attr_granular > 1) {
1580 		if (dma->dp_granularity_power_2) {
1581 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1582 			    (attr->dma_attr_maxxfer &
1583 			    (attr->dma_attr_granular - 1));
1584 		} else {
1585 			dma->dp_maxxfer = attr->dma_attr_maxxfer -
1586 			    (attr->dma_attr_maxxfer % attr->dma_attr_granular);
1587 		}
1588 	} else {
1589 		dma->dp_maxxfer = attr->dma_attr_maxxfer;
1590 	}
1591 
1592 	maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer);
1593 	maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max);
1594 	if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) {
1595 		maxsegmentsize = 0xFFFFFFFF;
1596 	} else {
1597 		maxsegmentsize = maxsegmentsize_ll;
1598 	}
1599 	dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize;
1600 	dma->dp_sglinfo.si_segmask = attr->dma_attr_seg;
1601 
1602 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1603 	if (rootnex_alloc_check_parms) {
1604 		e = rootnex_valid_alloc_parms(attr, maxsegmentsize);
1605 		if (e != DDI_SUCCESS) {
1606 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]);
1607 			(void) rootnex_dma_freehdl(dip, rdip,
1608 			    (ddi_dma_handle_t)hp);
1609 			return (e);
1610 		}
1611 	}
1612 
1613 	*handlep = (ddi_dma_handle_t)hp;
1614 
1615 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1616 	DTRACE_PROBE1(rootnex__alloc__handle, uint64_t,
1617 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1618 
1619 	return (DDI_SUCCESS);
1620 }
1621 
1622 
1623 /*
1624  * rootnex_dma_freehdl()
1625  *    called from ddi_dma_free_handle().
1626  */
1627 /*ARGSUSED*/
1628 static int
1629 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle)
1630 {
1631 	ddi_dma_impl_t *hp;
1632 	rootnex_dma_t *dma;
1633 
1634 
1635 	hp = (ddi_dma_impl_t *)handle;
1636 	dma = (rootnex_dma_t *)hp->dmai_private;
1637 
1638 	/* unbind should have been called first */
1639 	ASSERT(!dma->dp_inuse);
1640 
1641 	mutex_destroy(&dma->dp_mutex);
1642 	kmem_cache_free(rootnex_state->r_dmahdl_cache, hp);
1643 
1644 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1645 	DTRACE_PROBE1(rootnex__free__handle, uint64_t,
1646 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]);
1647 
1648 	if (rootnex_state->r_dvma_call_list_id)
1649 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1650 
1651 	return (DDI_SUCCESS);
1652 }
1653 
1654 
1655 /*
1656  * rootnex_dma_bindhdl()
1657  *    called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle().
1658  */
1659 /*ARGSUSED*/
1660 static int
1661 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
1662     struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp)
1663 {
1664 	rootnex_sglinfo_t *sinfo;
1665 	ddi_dma_attr_t *attr;
1666 	ddi_dma_impl_t *hp;
1667 	rootnex_dma_t *dma;
1668 	int kmflag;
1669 	int e;
1670 
1671 
1672 	hp = (ddi_dma_impl_t *)handle;
1673 	dma = (rootnex_dma_t *)hp->dmai_private;
1674 	sinfo = &dma->dp_sglinfo;
1675 	attr = &hp->dmai_attr;
1676 
1677 	hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS;
1678 
1679 	/*
1680 	 * This is useful for debugging a driver. Not as useful in a production
1681 	 * system. The only time this will fail is if you have a driver bug.
1682 	 */
1683 	if (rootnex_bind_check_inuse) {
1684 		/*
1685 		 * No one else should ever have this lock unless someone else
1686 		 * is trying to use this handle. So contention on the lock
1687 		 * is the same as inuse being set.
1688 		 */
1689 		e = mutex_tryenter(&dma->dp_mutex);
1690 		if (e == 0) {
1691 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1692 			return (DDI_DMA_INUSE);
1693 		}
1694 		if (dma->dp_inuse) {
1695 			mutex_exit(&dma->dp_mutex);
1696 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1697 			return (DDI_DMA_INUSE);
1698 		}
1699 		dma->dp_inuse = B_TRUE;
1700 		mutex_exit(&dma->dp_mutex);
1701 	}
1702 
1703 	/* check the ddi_dma_attr arg to make sure it makes a little sense */
1704 	if (rootnex_bind_check_parms) {
1705 		e = rootnex_valid_bind_parms(dmareq, attr);
1706 		if (e != DDI_SUCCESS) {
1707 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1708 			rootnex_clean_dmahdl(hp);
1709 			return (e);
1710 		}
1711 	}
1712 
1713 	/* save away the original bind info */
1714 	dma->dp_dma = dmareq->dmar_object;
1715 
1716 	/*
1717 	 * Figure out a rough estimate of what maximum number of pages this
1718 	 * buffer could use (a high estimate of course).
1719 	 */
1720 	sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1;
1721 
1722 	/*
1723 	 * We'll use the pre-allocated cookies for any bind that will *always*
1724 	 * fit (more important to be consistent, we don't want to create
1725 	 * additional degenerate cases).
1726 	 */
1727 	if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) {
1728 		dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer;
1729 		dma->dp_need_to_free_cookie = B_FALSE;
1730 		DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip,
1731 		    uint_t, sinfo->si_max_pages);
1732 
1733 	/*
1734 	 * For anything larger than that, we'll go ahead and allocate the
1735 	 * maximum number of pages we expect to see. Hopefuly, we won't be
1736 	 * seeing this path in the fast path for high performance devices very
1737 	 * frequently.
1738 	 *
1739 	 * a ddi bind interface that allowed the driver to provide storage to
1740 	 * the bind interface would speed this case up.
1741 	 */
1742 	} else {
1743 		/* convert the sleep flags */
1744 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
1745 			kmflag =  KM_SLEEP;
1746 		} else {
1747 			kmflag =  KM_NOSLEEP;
1748 		}
1749 
1750 		/*
1751 		 * Save away how much memory we allocated. If we're doing a
1752 		 * nosleep, the alloc could fail...
1753 		 */
1754 		dma->dp_cookie_size = sinfo->si_max_pages *
1755 		    sizeof (ddi_dma_cookie_t);
1756 		dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag);
1757 		if (dma->dp_cookies == NULL) {
1758 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1759 			rootnex_clean_dmahdl(hp);
1760 			return (DDI_DMA_NORESOURCES);
1761 		}
1762 		dma->dp_need_to_free_cookie = B_TRUE;
1763 		DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t,
1764 		    sinfo->si_max_pages);
1765 	}
1766 	hp->dmai_cookie = dma->dp_cookies;
1767 
1768 	/*
1769 	 * Get the real sgl. rootnex_get_sgl will fill in cookie array while
1770 	 * looking at the contraints in the dma structure. It will then put some
1771 	 * additional state about the sgl in the dma struct (i.e. is the sgl
1772 	 * clean, or do we need to do some munging; how many pages need to be
1773 	 * copied, etc.)
1774 	 */
1775 	rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies,
1776 	    &dma->dp_sglinfo);
1777 	ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages);
1778 
1779 	/* if we don't need a copy buffer, we don't need to sync */
1780 	if (sinfo->si_copybuf_req == 0) {
1781 		hp->dmai_rflags |= DMP_NOSYNC;
1782 	}
1783 
1784 	/*
1785 	 * If the driver supports FMA, insert the handle in the FMA DMA handle
1786 	 * cache.
1787 	 */
1788 	if (attr->dma_attr_flags & DDI_DMA_FLAGERR) {
1789 		hp->dmai_error.err_cf = rootnex_dma_check;
1790 		(void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL);
1791 	}
1792 
1793 	/*
1794 	 * if we don't need the copybuf and we don't need to do a partial,  we
1795 	 * hit the fast path. All the high performance devices should be trying
1796 	 * to hit this path. To hit this path, a device should be able to reach
1797 	 * all of memory, shouldn't try to bind more than it can transfer, and
1798 	 * the buffer shouldn't require more cookies than the driver/device can
1799 	 * handle [sgllen]).
1800 	 */
1801 	if ((sinfo->si_copybuf_req == 0) &&
1802 	    (sinfo->si_sgl_size <= attr->dma_attr_sgllen) &&
1803 	    (dma->dp_dma.dmao_size < dma->dp_maxxfer)) {
1804 		/*
1805 		 * copy out the first cookie and ccountp, set the cookie
1806 		 * pointer to the second cookie. The first cookie is passed
1807 		 * back on the stack. Additional cookies are accessed via
1808 		 * ddi_dma_nextcookie()
1809 		 */
1810 		*cookiep = dma->dp_cookies[0];
1811 		*ccountp = sinfo->si_sgl_size;
1812 		hp->dmai_cookie++;
1813 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
1814 		hp->dmai_nwin = 1;
1815 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1816 		DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t,
1817 		    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
1818 		    dma->dp_dma.dmao_size);
1819 		return (DDI_DMA_MAPPED);
1820 	}
1821 
1822 	/*
1823 	 * go to the slow path, we may need to alloc more memory, create
1824 	 * multiple windows, and munge up a sgl to make the device happy.
1825 	 */
1826 	e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag);
1827 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
1828 		if (dma->dp_need_to_free_cookie) {
1829 			kmem_free(dma->dp_cookies, dma->dp_cookie_size);
1830 		}
1831 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]);
1832 		rootnex_clean_dmahdl(hp); /* must be after free cookie */
1833 		return (e);
1834 	}
1835 
1836 	/* if the first window uses the copy buffer, sync it for the device */
1837 	if ((dma->dp_window[dma->dp_current_win].wd_dosync) &&
1838 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
1839 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
1840 		    DDI_DMA_SYNC_FORDEV);
1841 	}
1842 
1843 	/*
1844 	 * copy out the first cookie and ccountp, set the cookie pointer to the
1845 	 * second cookie. Make sure the partial flag is set/cleared correctly.
1846 	 * If we have a partial map (i.e. multiple windows), the number of
1847 	 * cookies we return is the number of cookies in the first window.
1848 	 */
1849 	if (e == DDI_DMA_MAPPED) {
1850 		hp->dmai_rflags &= ~DDI_DMA_PARTIAL;
1851 		*ccountp = sinfo->si_sgl_size;
1852 	} else {
1853 		hp->dmai_rflags |= DDI_DMA_PARTIAL;
1854 		*ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt;
1855 		ASSERT(hp->dmai_nwin <= dma->dp_max_win);
1856 	}
1857 	*cookiep = dma->dp_cookies[0];
1858 	hp->dmai_cookie++;
1859 
1860 	ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1861 	DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t,
1862 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t,
1863 	    dma->dp_dma.dmao_size);
1864 	return (e);
1865 }
1866 
1867 
1868 /*
1869  * rootnex_dma_unbindhdl()
1870  *    called from ddi_dma_unbind_handle()
1871  */
1872 /*ARGSUSED*/
1873 static int
1874 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip,
1875     ddi_dma_handle_t handle)
1876 {
1877 	ddi_dma_impl_t *hp;
1878 	rootnex_dma_t *dma;
1879 	int e;
1880 
1881 
1882 	hp = (ddi_dma_impl_t *)handle;
1883 	dma = (rootnex_dma_t *)hp->dmai_private;
1884 
1885 	/* make sure the buffer wasn't free'd before calling unbind */
1886 	if (rootnex_unbind_verify_buffer) {
1887 		e = rootnex_verify_buffer(dma);
1888 		if (e != DDI_SUCCESS) {
1889 			ASSERT(0);
1890 			return (DDI_FAILURE);
1891 		}
1892 	}
1893 
1894 	/* sync the current window before unbinding the buffer */
1895 	if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync &&
1896 	    (hp->dmai_rflags & DDI_DMA_READ)) {
1897 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
1898 		    DDI_DMA_SYNC_FORCPU);
1899 	}
1900 
1901 	/*
1902 	 * If the driver supports FMA, remove the handle in the FMA DMA handle
1903 	 * cache.
1904 	 */
1905 	if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) {
1906 		if ((DEVI(rdip)->devi_fmhdl != NULL) &&
1907 		    (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) {
1908 			(void) ndi_fmc_remove(rdip, DMA_HANDLE, hp);
1909 		}
1910 	}
1911 
1912 	/*
1913 	 * cleanup and copy buffer or window state. if we didn't use the copy
1914 	 * buffer or windows, there won't be much to do :-)
1915 	 */
1916 	rootnex_teardown_copybuf(dma);
1917 	rootnex_teardown_windows(dma);
1918 
1919 	/*
1920 	 * If we had to allocate space to for the worse case sgl (it didn't
1921 	 * fit into our pre-allocate buffer), free that up now
1922 	 */
1923 	if (dma->dp_need_to_free_cookie) {
1924 		kmem_free(dma->dp_cookies, dma->dp_cookie_size);
1925 	}
1926 
1927 	/*
1928 	 * clean up the handle so it's ready for the next bind (i.e. if the
1929 	 * handle is reused).
1930 	 */
1931 	rootnex_clean_dmahdl(hp);
1932 
1933 	if (rootnex_state->r_dvma_call_list_id)
1934 		ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
1935 
1936 	ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1937 	DTRACE_PROBE1(rootnex__unbind, uint64_t,
1938 	    rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]);
1939 
1940 	return (DDI_SUCCESS);
1941 }
1942 
1943 
1944 /*
1945  * rootnex_verify_buffer()
1946  *   verify buffer wasn't free'd
1947  */
1948 static int
1949 rootnex_verify_buffer(rootnex_dma_t *dma)
1950 {
1951 	page_t **pplist;
1952 	caddr_t vaddr;
1953 	uint_t pcnt;
1954 	uint_t poff;
1955 	page_t *pp;
1956 	char b;
1957 	int i;
1958 
1959 	/* Figure out how many pages this buffer occupies */
1960 	if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) {
1961 		poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET;
1962 	} else {
1963 		vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr;
1964 		poff = (uintptr_t)vaddr & MMU_PAGEOFFSET;
1965 	}
1966 	pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff);
1967 
1968 	switch (dma->dp_dma.dmao_type) {
1969 	case DMA_OTYP_PAGES:
1970 		/*
1971 		 * for a linked list of pp's walk through them to make sure
1972 		 * they're locked and not free.
1973 		 */
1974 		pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp;
1975 		for (i = 0; i < pcnt; i++) {
1976 			if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) {
1977 				return (DDI_FAILURE);
1978 			}
1979 			pp = pp->p_next;
1980 		}
1981 		break;
1982 
1983 	case DMA_OTYP_VADDR:
1984 	case DMA_OTYP_BUFVADDR:
1985 		pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv;
1986 		/*
1987 		 * for an array of pp's walk through them to make sure they're
1988 		 * not free. It's possible that they may not be locked.
1989 		 */
1990 		if (pplist) {
1991 			for (i = 0; i < pcnt; i++) {
1992 				if (PP_ISFREE(pplist[i])) {
1993 					return (DDI_FAILURE);
1994 				}
1995 			}
1996 
1997 		/* For a virtual address, try to peek at each page */
1998 		} else {
1999 			if (dma->dp_sglinfo.si_asp == &kas) {
2000 				for (i = 0; i < pcnt; i++) {
2001 					if (ddi_peek8(NULL, vaddr, &b) ==
2002 					    DDI_FAILURE)
2003 						return (DDI_FAILURE);
2004 					vaddr += MMU_PAGESIZE;
2005 				}
2006 			}
2007 		}
2008 		break;
2009 
2010 	default:
2011 		ASSERT(0);
2012 		break;
2013 	}
2014 
2015 	return (DDI_SUCCESS);
2016 }
2017 
2018 
2019 /*
2020  * rootnex_clean_dmahdl()
2021  *    Clean the dma handle. This should be called on a handle alloc and an
2022  *    unbind handle. Set the handle state to the default settings.
2023  */
2024 static void
2025 rootnex_clean_dmahdl(ddi_dma_impl_t *hp)
2026 {
2027 	rootnex_dma_t *dma;
2028 
2029 
2030 	dma = (rootnex_dma_t *)hp->dmai_private;
2031 
2032 	hp->dmai_nwin = 0;
2033 	dma->dp_current_cookie = 0;
2034 	dma->dp_copybuf_size = 0;
2035 	dma->dp_window = NULL;
2036 	dma->dp_cbaddr = NULL;
2037 	dma->dp_inuse = B_FALSE;
2038 	dma->dp_need_to_free_cookie = B_FALSE;
2039 	dma->dp_need_to_free_window = B_FALSE;
2040 	dma->dp_partial_required = B_FALSE;
2041 	dma->dp_trim_required = B_FALSE;
2042 	dma->dp_sglinfo.si_copybuf_req = 0;
2043 #if !defined(__amd64)
2044 	dma->dp_cb_remaping = B_FALSE;
2045 	dma->dp_kva = NULL;
2046 #endif
2047 
2048 	/* FMA related initialization */
2049 	hp->dmai_fault = 0;
2050 	hp->dmai_fault_check = NULL;
2051 	hp->dmai_fault_notify = NULL;
2052 	hp->dmai_error.err_ena = 0;
2053 	hp->dmai_error.err_status = DDI_FM_OK;
2054 	hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
2055 	hp->dmai_error.err_ontrap = NULL;
2056 	hp->dmai_error.err_fep = NULL;
2057 	hp->dmai_error.err_cf = NULL;
2058 }
2059 
2060 
2061 /*
2062  * rootnex_valid_alloc_parms()
2063  *    Called in ddi_dma_alloc_handle path to validate its parameters.
2064  */
2065 static int
2066 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize)
2067 {
2068 	if ((attr->dma_attr_seg < MMU_PAGEOFFSET) ||
2069 	    (attr->dma_attr_count_max < MMU_PAGEOFFSET) ||
2070 	    (attr->dma_attr_granular > MMU_PAGESIZE) ||
2071 	    (attr->dma_attr_maxxfer < MMU_PAGESIZE)) {
2072 		return (DDI_DMA_BADATTR);
2073 	}
2074 
2075 	if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) {
2076 		return (DDI_DMA_BADATTR);
2077 	}
2078 
2079 	if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET ||
2080 	    MMU_PAGESIZE & (attr->dma_attr_granular - 1) ||
2081 	    attr->dma_attr_sgllen <= 0) {
2082 		return (DDI_DMA_BADATTR);
2083 	}
2084 
2085 	/* We should be able to DMA into every byte offset in a page */
2086 	if (maxsegmentsize < MMU_PAGESIZE) {
2087 		return (DDI_DMA_BADATTR);
2088 	}
2089 
2090 	return (DDI_SUCCESS);
2091 }
2092 
2093 
2094 /*
2095  * rootnex_valid_bind_parms()
2096  *    Called in ddi_dma_*_bind_handle path to validate its parameters.
2097  */
2098 /* ARGSUSED */
2099 static int
2100 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr)
2101 {
2102 #if !defined(__amd64)
2103 	/*
2104 	 * we only support up to a 2G-1 transfer size on 32-bit kernels so
2105 	 * we can track the offset for the obsoleted interfaces.
2106 	 */
2107 	if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) {
2108 		return (DDI_DMA_TOOBIG);
2109 	}
2110 #endif
2111 
2112 	return (DDI_SUCCESS);
2113 }
2114 
2115 
2116 /*
2117  * rootnex_get_sgl()
2118  *    Called in bind fastpath to get the sgl. Most of this will be replaced
2119  *    with a call to the vm layer when vm2.0 comes around...
2120  */
2121 static void
2122 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl,
2123     rootnex_sglinfo_t *sglinfo)
2124 {
2125 	ddi_dma_atyp_t buftype;
2126 	uint64_t last_page;
2127 	uint64_t offset;
2128 	uint64_t addrhi;
2129 	uint64_t addrlo;
2130 	uint64_t maxseg;
2131 	page_t **pplist;
2132 	uint64_t paddr;
2133 	uint32_t psize;
2134 	uint32_t size;
2135 	caddr_t vaddr;
2136 	uint_t pcnt;
2137 	page_t *pp;
2138 	uint_t cnt;
2139 
2140 
2141 	/* shortcuts */
2142 	pplist = dmar_object->dmao_obj.virt_obj.v_priv;
2143 	vaddr = dmar_object->dmao_obj.virt_obj.v_addr;
2144 	maxseg = sglinfo->si_max_cookie_size;
2145 	buftype = dmar_object->dmao_type;
2146 	addrhi = sglinfo->si_max_addr;
2147 	addrlo = sglinfo->si_min_addr;
2148 	size = dmar_object->dmao_size;
2149 
2150 	pcnt = 0;
2151 	cnt = 0;
2152 
2153 	/*
2154 	 * if we were passed down a linked list of pages, i.e. pointer to
2155 	 * page_t, use this to get our physical address and buf offset.
2156 	 */
2157 	if (buftype == DMA_OTYP_PAGES) {
2158 		pp = dmar_object->dmao_obj.pp_obj.pp_pp;
2159 		ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2160 		offset =  dmar_object->dmao_obj.pp_obj.pp_offset &
2161 		    MMU_PAGEOFFSET;
2162 		paddr = ptob64(pp->p_pagenum) + offset;
2163 		psize = MIN(size, (MMU_PAGESIZE - offset));
2164 		pp = pp->p_next;
2165 		sglinfo->si_asp = NULL;
2166 
2167 	/*
2168 	 * We weren't passed down a linked list of pages, but if we were passed
2169 	 * down an array of pages, use this to get our physical address and buf
2170 	 * offset.
2171 	 */
2172 	} else if (pplist != NULL) {
2173 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2174 		    (buftype == DMA_OTYP_BUFVADDR));
2175 
2176 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2177 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2178 		if (sglinfo->si_asp == NULL) {
2179 			sglinfo->si_asp = &kas;
2180 		}
2181 
2182 		ASSERT(!PP_ISFREE(pplist[pcnt]));
2183 		paddr = ptob64(pplist[pcnt]->p_pagenum);
2184 		paddr += offset;
2185 		psize = MIN(size, (MMU_PAGESIZE - offset));
2186 		pcnt++;
2187 
2188 	/*
2189 	 * All we have is a virtual address, we'll need to call into the VM
2190 	 * to get the physical address.
2191 	 */
2192 	} else {
2193 		ASSERT((buftype == DMA_OTYP_VADDR) ||
2194 		    (buftype == DMA_OTYP_BUFVADDR));
2195 
2196 		offset = (uintptr_t)vaddr & MMU_PAGEOFFSET;
2197 		sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as;
2198 		if (sglinfo->si_asp == NULL) {
2199 			sglinfo->si_asp = &kas;
2200 		}
2201 
2202 		paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr));
2203 		paddr += offset;
2204 		psize = MIN(size, (MMU_PAGESIZE - offset));
2205 		vaddr += psize;
2206 	}
2207 
2208 	/*
2209 	 * Setup the first cookie with the physical address of the page and the
2210 	 * size of the page (which takes into account the initial offset into
2211 	 * the page.
2212 	 */
2213 	sgl[cnt].dmac_laddress = paddr;
2214 	sgl[cnt].dmac_size = psize;
2215 	sgl[cnt].dmac_type = 0;
2216 
2217 	/*
2218 	 * Save away the buffer offset into the page. We'll need this later in
2219 	 * the copy buffer code to help figure out the page index within the
2220 	 * buffer and the offset into the current page.
2221 	 */
2222 	sglinfo->si_buf_offset = offset;
2223 
2224 	/*
2225 	 * If the DMA engine can't reach the physical address, increase how
2226 	 * much copy buffer we need. We always increase by pagesize so we don't
2227 	 * have to worry about converting offsets. Set a flag in the cookies
2228 	 * dmac_type to indicate that it uses the copy buffer. If this isn't the
2229 	 * last cookie, go to the next cookie (since we separate each page which
2230 	 * uses the copy buffer in case the copy buffer is not physically
2231 	 * contiguous.
2232 	 */
2233 	if ((paddr < addrlo) || ((paddr + psize) > addrhi)) {
2234 		sglinfo->si_copybuf_req += MMU_PAGESIZE;
2235 		sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2236 		if ((cnt + 1) < sglinfo->si_max_pages) {
2237 			cnt++;
2238 			sgl[cnt].dmac_laddress = 0;
2239 			sgl[cnt].dmac_size = 0;
2240 			sgl[cnt].dmac_type = 0;
2241 		}
2242 	}
2243 
2244 	/*
2245 	 * save this page's physical address so we can figure out if the next
2246 	 * page is physically contiguous. Keep decrementing size until we are
2247 	 * done with the buffer.
2248 	 */
2249 	last_page = paddr & MMU_PAGEMASK;
2250 	size -= psize;
2251 
2252 	while (size > 0) {
2253 		/* Get the size for this page (i.e. partial or full page) */
2254 		psize = MIN(size, MMU_PAGESIZE);
2255 
2256 		if (buftype == DMA_OTYP_PAGES) {
2257 			/* get the paddr from the page_t */
2258 			ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp));
2259 			paddr = ptob64(pp->p_pagenum);
2260 			pp = pp->p_next;
2261 		} else if (pplist != NULL) {
2262 			/* index into the array of page_t's to get the paddr */
2263 			ASSERT(!PP_ISFREE(pplist[pcnt]));
2264 			paddr = ptob64(pplist[pcnt]->p_pagenum);
2265 			pcnt++;
2266 		} else {
2267 			/* call into the VM to get the paddr */
2268 			paddr =  ptob64(hat_getpfnum(sglinfo->si_asp->a_hat,
2269 			    vaddr));
2270 			vaddr += psize;
2271 		}
2272 
2273 		/* check to see if this page needs the copy buffer */
2274 		if ((paddr < addrlo) || ((paddr + psize) > addrhi)) {
2275 			sglinfo->si_copybuf_req += MMU_PAGESIZE;
2276 
2277 			/*
2278 			 * if there is something in the current cookie, go to
2279 			 * the next one. We only want one page in a cookie which
2280 			 * uses the copybuf since the copybuf doesn't have to
2281 			 * be physically contiguous.
2282 			 */
2283 			if (sgl[cnt].dmac_size != 0) {
2284 				cnt++;
2285 			}
2286 			sgl[cnt].dmac_laddress = paddr;
2287 			sgl[cnt].dmac_size = psize;
2288 #if defined(__amd64)
2289 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF;
2290 #else
2291 			/*
2292 			 * save the buf offset for 32-bit kernel. used in the
2293 			 * obsoleted interfaces.
2294 			 */
2295 			sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF |
2296 			    (dmar_object->dmao_size - size);
2297 #endif
2298 			/* if this isn't the last cookie, go to the next one */
2299 			if ((cnt + 1) < sglinfo->si_max_pages) {
2300 				cnt++;
2301 				sgl[cnt].dmac_laddress = 0;
2302 				sgl[cnt].dmac_size = 0;
2303 				sgl[cnt].dmac_type = 0;
2304 			}
2305 
2306 		/*
2307 		 * this page didn't need the copy buffer, if it's not physically
2308 		 * contiguous, or it would put us over a segment boundary, or it
2309 		 * puts us over the max cookie size, or the current sgl doesn't
2310 		 * have anything in it.
2311 		 */
2312 		} else if (((last_page + MMU_PAGESIZE) != paddr) ||
2313 		    !(paddr & sglinfo->si_segmask) ||
2314 		    ((sgl[cnt].dmac_size + psize) > maxseg) ||
2315 		    (sgl[cnt].dmac_size == 0)) {
2316 			/*
2317 			 * if we're not already in a new cookie, go to the next
2318 			 * cookie.
2319 			 */
2320 			if (sgl[cnt].dmac_size != 0) {
2321 				cnt++;
2322 			}
2323 
2324 			/* save the cookie information */
2325 			sgl[cnt].dmac_laddress = paddr;
2326 			sgl[cnt].dmac_size = psize;
2327 #if defined(__amd64)
2328 			sgl[cnt].dmac_type = 0;
2329 #else
2330 			/*
2331 			 * save the buf offset for 32-bit kernel. used in the
2332 			 * obsoleted interfaces.
2333 			 */
2334 			sgl[cnt].dmac_type = dmar_object->dmao_size - size;
2335 #endif
2336 
2337 		/*
2338 		 * this page didn't need the copy buffer, it is physically
2339 		 * contiguous with the last page, and it's <= the max cookie
2340 		 * size.
2341 		 */
2342 		} else {
2343 			sgl[cnt].dmac_size += psize;
2344 
2345 			/*
2346 			 * if this exactly ==  the maximum cookie size, and
2347 			 * it isn't the last cookie, go to the next cookie.
2348 			 */
2349 			if (((sgl[cnt].dmac_size + psize) == maxseg) &&
2350 			    ((cnt + 1) < sglinfo->si_max_pages)) {
2351 				cnt++;
2352 				sgl[cnt].dmac_laddress = 0;
2353 				sgl[cnt].dmac_size = 0;
2354 				sgl[cnt].dmac_type = 0;
2355 			}
2356 		}
2357 
2358 		/*
2359 		 * save this page's physical address so we can figure out if the
2360 		 * next page is physically contiguous. Keep decrementing size
2361 		 * until we are done with the buffer.
2362 		 */
2363 		last_page = paddr;
2364 		size -= psize;
2365 	}
2366 
2367 	/* we're done, save away how many cookies the sgl has */
2368 	if (sgl[cnt].dmac_size == 0) {
2369 		ASSERT(cnt < sglinfo->si_max_pages);
2370 		sglinfo->si_sgl_size = cnt;
2371 	} else {
2372 		sglinfo->si_sgl_size = cnt + 1;
2373 	}
2374 }
2375 
2376 
2377 /*
2378  * rootnex_bind_slowpath()
2379  *    Call in the bind path if the calling driver can't use the sgl without
2380  *    modifying it. We either need to use the copy buffer and/or we will end up
2381  *    with a partial bind.
2382  */
2383 static int
2384 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2385     rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag)
2386 {
2387 	rootnex_sglinfo_t *sinfo;
2388 	rootnex_window_t *window;
2389 	ddi_dma_cookie_t *cookie;
2390 	size_t copybuf_used;
2391 	size_t dmac_size;
2392 	boolean_t partial;
2393 	off_t cur_offset;
2394 	page_t *cur_pp;
2395 	major_t mnum;
2396 	int e;
2397 	int i;
2398 
2399 
2400 	sinfo = &dma->dp_sglinfo;
2401 	copybuf_used = 0;
2402 	partial = B_FALSE;
2403 
2404 	/*
2405 	 * If we're using the copybuf, set the copybuf state in dma struct.
2406 	 * Needs to be first since it sets the copy buffer size.
2407 	 */
2408 	if (sinfo->si_copybuf_req != 0) {
2409 		e = rootnex_setup_copybuf(hp, dmareq, dma, attr);
2410 		if (e != DDI_SUCCESS) {
2411 			return (e);
2412 		}
2413 	} else {
2414 		dma->dp_copybuf_size = 0;
2415 	}
2416 
2417 	/*
2418 	 * Figure out if we need to do a partial mapping. If so, figure out
2419 	 * if we need to trim the buffers when we munge the sgl.
2420 	 */
2421 	if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) ||
2422 	    (dma->dp_dma.dmao_size > dma->dp_maxxfer) ||
2423 	    (attr->dma_attr_sgllen < sinfo->si_sgl_size)) {
2424 		dma->dp_partial_required = B_TRUE;
2425 		if (attr->dma_attr_granular != 1) {
2426 			dma->dp_trim_required = B_TRUE;
2427 		}
2428 	} else {
2429 		dma->dp_partial_required = B_FALSE;
2430 		dma->dp_trim_required = B_FALSE;
2431 	}
2432 
2433 	/* If we need to do a partial bind, make sure the driver supports it */
2434 	if (dma->dp_partial_required &&
2435 	    !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
2436 
2437 		mnum = ddi_driver_major(dma->dp_dip);
2438 		/*
2439 		 * patchable which allows us to print one warning per major
2440 		 * number.
2441 		 */
2442 		if ((rootnex_bind_warn) &&
2443 		    ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) {
2444 			rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING;
2445 			cmn_err(CE_WARN, "!%s: coding error detected, the "
2446 			    "driver is using ddi_dma_attr(9S) incorrectly. "
2447 			    "There is a small risk of data corruption in "
2448 			    "particular with large I/Os. The driver should be "
2449 			    "replaced with a corrected version for proper "
2450 			    "system operation. To disable this warning, add "
2451 			    "'set rootnex:rootnex_bind_warn=0' to "
2452 			    "/etc/system(4).", ddi_driver_name(dma->dp_dip));
2453 		}
2454 		return (DDI_DMA_TOOBIG);
2455 	}
2456 
2457 	/*
2458 	 * we might need multiple windows, setup state to handle them. In this
2459 	 * code path, we will have at least one window.
2460 	 */
2461 	e = rootnex_setup_windows(hp, dma, attr, kmflag);
2462 	if (e != DDI_SUCCESS) {
2463 		rootnex_teardown_copybuf(dma);
2464 		return (e);
2465 	}
2466 
2467 	window = &dma->dp_window[0];
2468 	cookie = &dma->dp_cookies[0];
2469 	cur_offset = 0;
2470 	rootnex_init_win(hp, dma, window, cookie, cur_offset);
2471 	if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) {
2472 		cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp;
2473 	}
2474 
2475 	/* loop though all the cookies we got back from get_sgl() */
2476 	for (i = 0; i < sinfo->si_sgl_size; i++) {
2477 		/*
2478 		 * If we're using the copy buffer, check this cookie and setup
2479 		 * its associated copy buffer state. If this cookie uses the
2480 		 * copy buffer, make sure we sync this window during dma_sync.
2481 		 */
2482 		if (dma->dp_copybuf_size > 0) {
2483 			rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie,
2484 			    cur_offset, &copybuf_used, &cur_pp);
2485 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2486 				window->wd_dosync = B_TRUE;
2487 			}
2488 		}
2489 
2490 		/*
2491 		 * save away the cookie size, since it could be modified in
2492 		 * the windowing code.
2493 		 */
2494 		dmac_size = cookie->dmac_size;
2495 
2496 		/* if we went over max copybuf size */
2497 		if (dma->dp_copybuf_size &&
2498 		    (copybuf_used > dma->dp_copybuf_size)) {
2499 			partial = B_TRUE;
2500 			e = rootnex_copybuf_window_boundary(hp, dma, &window,
2501 			    cookie, cur_offset, &copybuf_used);
2502 			if (e != DDI_SUCCESS) {
2503 				rootnex_teardown_copybuf(dma);
2504 				rootnex_teardown_windows(dma);
2505 				return (e);
2506 			}
2507 
2508 			/*
2509 			 * if the coookie uses the copy buffer, make sure the
2510 			 * new window we just moved to is set to sync.
2511 			 */
2512 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2513 				window->wd_dosync = B_TRUE;
2514 			}
2515 			DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *,
2516 			    dma->dp_dip);
2517 
2518 		/* if the cookie cnt == max sgllen, move to the next window */
2519 		} else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) {
2520 			partial = B_TRUE;
2521 			ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen);
2522 			e = rootnex_sgllen_window_boundary(hp, dma, &window,
2523 			    cookie, attr, cur_offset);
2524 			if (e != DDI_SUCCESS) {
2525 				rootnex_teardown_copybuf(dma);
2526 				rootnex_teardown_windows(dma);
2527 				return (e);
2528 			}
2529 
2530 			/*
2531 			 * if the coookie uses the copy buffer, make sure the
2532 			 * new window we just moved to is set to sync.
2533 			 */
2534 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2535 				window->wd_dosync = B_TRUE;
2536 			}
2537 			DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *,
2538 			    dma->dp_dip);
2539 
2540 		/* else if we will be over maxxfer */
2541 		} else if ((window->wd_size + dmac_size) >
2542 		    dma->dp_maxxfer) {
2543 			partial = B_TRUE;
2544 			e = rootnex_maxxfer_window_boundary(hp, dma, &window,
2545 			    cookie);
2546 			if (e != DDI_SUCCESS) {
2547 				rootnex_teardown_copybuf(dma);
2548 				rootnex_teardown_windows(dma);
2549 				return (e);
2550 			}
2551 
2552 			/*
2553 			 * if the coookie uses the copy buffer, make sure the
2554 			 * new window we just moved to is set to sync.
2555 			 */
2556 			if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2557 				window->wd_dosync = B_TRUE;
2558 			}
2559 			DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *,
2560 			    dma->dp_dip);
2561 
2562 		/* else this cookie fits in the current window */
2563 		} else {
2564 			window->wd_cookie_cnt++;
2565 			window->wd_size += dmac_size;
2566 		}
2567 
2568 		/* track our offset into the buffer, go to the next cookie */
2569 		ASSERT(dmac_size <= dma->dp_dma.dmao_size);
2570 		ASSERT(cookie->dmac_size <= dmac_size);
2571 		cur_offset += dmac_size;
2572 		cookie++;
2573 	}
2574 
2575 	/* if we ended up with a zero sized window in the end, clean it up */
2576 	if (window->wd_size == 0) {
2577 		hp->dmai_nwin--;
2578 		window--;
2579 	}
2580 
2581 	ASSERT(window->wd_trim.tr_trim_last == B_FALSE);
2582 
2583 	if (!partial) {
2584 		return (DDI_DMA_MAPPED);
2585 	}
2586 
2587 	ASSERT(dma->dp_partial_required);
2588 	return (DDI_DMA_PARTIAL_MAP);
2589 }
2590 
2591 
2592 /*
2593  * rootnex_setup_copybuf()
2594  *    Called in bind slowpath. Figures out if we're going to use the copy
2595  *    buffer, and if we do, sets up the basic state to handle it.
2596  */
2597 static int
2598 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq,
2599     rootnex_dma_t *dma, ddi_dma_attr_t *attr)
2600 {
2601 	rootnex_sglinfo_t *sinfo;
2602 	ddi_dma_attr_t lattr;
2603 	size_t max_copybuf;
2604 	int cansleep;
2605 	int e;
2606 #if !defined(__amd64)
2607 	int vmflag;
2608 #endif
2609 
2610 
2611 	sinfo = &dma->dp_sglinfo;
2612 
2613 	/*
2614 	 * read this first so it's consistent through the routine so we can
2615 	 * patch it on the fly.
2616 	 */
2617 	max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK;
2618 
2619 	/* We need to call into the rootnex on ddi_dma_sync() */
2620 	hp->dmai_rflags &= ~DMP_NOSYNC;
2621 
2622 	/* make sure the copybuf size <= the max size */
2623 	dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf);
2624 	ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0);
2625 
2626 #if !defined(__amd64)
2627 	/*
2628 	 * if we don't have kva space to copy to/from, allocate the KVA space
2629 	 * now. We only do this for the 32-bit kernel. We use seg kpm space for
2630 	 * the 64-bit kernel.
2631 	 */
2632 	if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) ||
2633 	    (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) {
2634 
2635 		/* convert the sleep flags */
2636 		if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2637 			vmflag = VM_SLEEP;
2638 		} else {
2639 			vmflag = VM_NOSLEEP;
2640 		}
2641 
2642 		/* allocate Kernel VA space that we can bcopy to/from */
2643 		dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size,
2644 		    vmflag);
2645 		if (dma->dp_kva == NULL) {
2646 			return (DDI_DMA_NORESOURCES);
2647 		}
2648 	}
2649 #endif
2650 
2651 	/* convert the sleep flags */
2652 	if (dmareq->dmar_fp == DDI_DMA_SLEEP) {
2653 		cansleep = 1;
2654 	} else {
2655 		cansleep = 0;
2656 	}
2657 
2658 	/*
2659 	 * Allocated the actual copy buffer. This needs to fit within the DMA
2660 	 * engines limits, so we can't use kmem_alloc...
2661 	 */
2662 	lattr = *attr;
2663 	lattr.dma_attr_align = MMU_PAGESIZE;
2664 	e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep,
2665 	    0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL);
2666 	if (e != DDI_SUCCESS) {
2667 #if !defined(__amd64)
2668 		if (dma->dp_kva != NULL) {
2669 			vmem_free(heap_arena, dma->dp_kva,
2670 			    dma->dp_copybuf_size);
2671 		}
2672 #endif
2673 		return (DDI_DMA_NORESOURCES);
2674 	}
2675 
2676 	DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip,
2677 	    size_t, dma->dp_copybuf_size);
2678 
2679 	return (DDI_SUCCESS);
2680 }
2681 
2682 
2683 /*
2684  * rootnex_setup_windows()
2685  *    Called in bind slowpath to setup the window state. We always have windows
2686  *    in the slowpath. Even if the window count = 1.
2687  */
2688 static int
2689 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
2690     ddi_dma_attr_t *attr, int kmflag)
2691 {
2692 	rootnex_window_t *windowp;
2693 	rootnex_sglinfo_t *sinfo;
2694 	size_t copy_state_size;
2695 	size_t win_state_size;
2696 	size_t state_available;
2697 	size_t space_needed;
2698 	uint_t copybuf_win;
2699 	uint_t maxxfer_win;
2700 	size_t space_used;
2701 	uint_t sglwin;
2702 
2703 
2704 	sinfo = &dma->dp_sglinfo;
2705 
2706 	dma->dp_current_win = 0;
2707 	hp->dmai_nwin = 0;
2708 
2709 	/* If we don't need to do a partial, we only have one window */
2710 	if (!dma->dp_partial_required) {
2711 		dma->dp_max_win = 1;
2712 
2713 	/*
2714 	 * we need multiple windows, need to figure out the worse case number
2715 	 * of windows.
2716 	 */
2717 	} else {
2718 		/*
2719 		 * if we need windows because we need more copy buffer that
2720 		 * we allow, the worse case number of windows we could need
2721 		 * here would be (copybuf space required / copybuf space that
2722 		 * we have) plus one for remainder, and plus 2 to handle the
2723 		 * extra pages on the trim for the first and last pages of the
2724 		 * buffer (a page is the minimum window size so under the right
2725 		 * attr settings, you could have a window for each page).
2726 		 * The last page will only be hit here if the size is not a
2727 		 * multiple of the granularity (which theoretically shouldn't
2728 		 * be the case but never has been enforced, so we could have
2729 		 * broken things without it).
2730 		 */
2731 		if (sinfo->si_copybuf_req > dma->dp_copybuf_size) {
2732 			ASSERT(dma->dp_copybuf_size > 0);
2733 			copybuf_win = (sinfo->si_copybuf_req /
2734 			    dma->dp_copybuf_size) + 1 + 2;
2735 		} else {
2736 			copybuf_win = 0;
2737 		}
2738 
2739 		/*
2740 		 * if we need windows because we have more cookies than the H/W
2741 		 * can handle, the number of windows we would need here would
2742 		 * be (cookie count / cookies count H/W supports) plus one for
2743 		 * remainder, and plus 2 to handle the extra pages on the trim
2744 		 * (see above comment about trim)
2745 		 */
2746 		if (attr->dma_attr_sgllen < sinfo->si_sgl_size) {
2747 			sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen)
2748 			    + 1) + 2;
2749 		} else {
2750 			sglwin = 0;
2751 		}
2752 
2753 		/*
2754 		 * if we need windows because we're binding more memory than the
2755 		 * H/W can transfer at once, the number of windows we would need
2756 		 * here would be (xfer count / max xfer H/W supports) plus one
2757 		 * for remainder, and plus 2 to handle the extra pages on the
2758 		 * trim (see above comment about trim)
2759 		 */
2760 		if (dma->dp_dma.dmao_size > dma->dp_maxxfer) {
2761 			maxxfer_win = (dma->dp_dma.dmao_size /
2762 			    dma->dp_maxxfer) + 1 + 2;
2763 		} else {
2764 			maxxfer_win = 0;
2765 		}
2766 		dma->dp_max_win =  copybuf_win + sglwin + maxxfer_win;
2767 		ASSERT(dma->dp_max_win > 0);
2768 	}
2769 	win_state_size = dma->dp_max_win * sizeof (rootnex_window_t);
2770 
2771 	/*
2772 	 * Get space for window and potential copy buffer state. Before we
2773 	 * go and allocate memory, see if we can get away with using what's
2774 	 * left in the pre-allocted state or the dynamically allocated sgl.
2775 	 */
2776 	space_used = (uintptr_t)(sinfo->si_sgl_size *
2777 	    sizeof (ddi_dma_cookie_t));
2778 
2779 	/* if we dynamically allocated space for the cookies */
2780 	if (dma->dp_need_to_free_cookie) {
2781 		/* if we have more space in the pre-allocted buffer, use it */
2782 		ASSERT(space_used <= dma->dp_cookie_size);
2783 		if ((dma->dp_cookie_size - space_used) <=
2784 		    rootnex_state->r_prealloc_size) {
2785 			state_available = rootnex_state->r_prealloc_size;
2786 			windowp = (rootnex_window_t *)dma->dp_prealloc_buffer;
2787 
2788 		/*
2789 		 * else, we have more free space in the dynamically allocated
2790 		 * buffer, i.e. the buffer wasn't worse case fragmented so we
2791 		 * didn't need a lot of cookies.
2792 		 */
2793 		} else {
2794 			state_available = dma->dp_cookie_size - space_used;
2795 			windowp = (rootnex_window_t *)
2796 			    &dma->dp_cookies[sinfo->si_sgl_size];
2797 		}
2798 
2799 	/* we used the pre-alloced buffer */
2800 	} else {
2801 		ASSERT(space_used <= rootnex_state->r_prealloc_size);
2802 		state_available = rootnex_state->r_prealloc_size - space_used;
2803 		windowp = (rootnex_window_t *)
2804 		    &dma->dp_cookies[sinfo->si_sgl_size];
2805 	}
2806 
2807 	/*
2808 	 * figure out how much state we need to track the copy buffer. Add an
2809 	 * addition 8 bytes for pointer alignemnt later.
2810 	 */
2811 	if (dma->dp_copybuf_size > 0) {
2812 		copy_state_size = sinfo->si_max_pages *
2813 		    sizeof (rootnex_pgmap_t);
2814 	} else {
2815 		copy_state_size = 0;
2816 	}
2817 	/* add an additional 8 bytes for pointer alignment */
2818 	space_needed = win_state_size + copy_state_size + 0x8;
2819 
2820 	/* if we have enough space already, use it */
2821 	if (state_available >= space_needed) {
2822 		dma->dp_window = windowp;
2823 		dma->dp_need_to_free_window = B_FALSE;
2824 
2825 	/* not enough space, need to allocate more. */
2826 	} else {
2827 		dma->dp_window = kmem_alloc(space_needed, kmflag);
2828 		if (dma->dp_window == NULL) {
2829 			return (DDI_DMA_NORESOURCES);
2830 		}
2831 		dma->dp_need_to_free_window = B_TRUE;
2832 		dma->dp_window_size = space_needed;
2833 		DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *,
2834 		    dma->dp_dip, size_t, space_needed);
2835 	}
2836 
2837 	/*
2838 	 * we allocate copy buffer state and window state at the same time.
2839 	 * setup our copy buffer state pointers. Make sure it's aligned.
2840 	 */
2841 	if (dma->dp_copybuf_size > 0) {
2842 		dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t)
2843 		    &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7);
2844 
2845 #if !defined(__amd64)
2846 		/*
2847 		 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to
2848 		 * false/NULL. Should be quicker to bzero vs loop and set.
2849 		 */
2850 		bzero(dma->dp_pgmap, copy_state_size);
2851 #endif
2852 	} else {
2853 		dma->dp_pgmap = NULL;
2854 	}
2855 
2856 	return (DDI_SUCCESS);
2857 }
2858 
2859 
2860 /*
2861  * rootnex_teardown_copybuf()
2862  *    cleans up after rootnex_setup_copybuf()
2863  */
2864 static void
2865 rootnex_teardown_copybuf(rootnex_dma_t *dma)
2866 {
2867 #if !defined(__amd64)
2868 	int i;
2869 
2870 	/*
2871 	 * if we allocated kernel heap VMEM space, go through all the pages and
2872 	 * map out any of the ones that we're mapped into the kernel heap VMEM
2873 	 * arena. Then free the VMEM space.
2874 	 */
2875 	if (dma->dp_kva != NULL) {
2876 		for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) {
2877 			if (dma->dp_pgmap[i].pm_mapped) {
2878 				hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr,
2879 				    MMU_PAGESIZE, HAT_UNLOAD);
2880 				dma->dp_pgmap[i].pm_mapped = B_FALSE;
2881 			}
2882 		}
2883 
2884 		vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size);
2885 	}
2886 
2887 #endif
2888 
2889 	/* if we allocated a copy buffer, free it */
2890 	if (dma->dp_cbaddr != NULL) {
2891 		i_ddi_mem_free(dma->dp_cbaddr, NULL);
2892 	}
2893 }
2894 
2895 
2896 /*
2897  * rootnex_teardown_windows()
2898  *    cleans up after rootnex_setup_windows()
2899  */
2900 static void
2901 rootnex_teardown_windows(rootnex_dma_t *dma)
2902 {
2903 	/*
2904 	 * if we had to allocate window state on the last bind (because we
2905 	 * didn't have enough pre-allocated space in the handle), free it.
2906 	 */
2907 	if (dma->dp_need_to_free_window) {
2908 		kmem_free(dma->dp_window, dma->dp_window_size);
2909 	}
2910 }
2911 
2912 
2913 /*
2914  * rootnex_init_win()
2915  *    Called in bind slow path during creation of a new window. Initializes
2916  *    window state to default values.
2917  */
2918 /*ARGSUSED*/
2919 static void
2920 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
2921     rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset)
2922 {
2923 	hp->dmai_nwin++;
2924 	window->wd_dosync = B_FALSE;
2925 	window->wd_offset = cur_offset;
2926 	window->wd_size = 0;
2927 	window->wd_first_cookie = cookie;
2928 	window->wd_cookie_cnt = 0;
2929 	window->wd_trim.tr_trim_first = B_FALSE;
2930 	window->wd_trim.tr_trim_last = B_FALSE;
2931 	window->wd_trim.tr_first_copybuf_win = B_FALSE;
2932 	window->wd_trim.tr_last_copybuf_win = B_FALSE;
2933 #if !defined(__amd64)
2934 	window->wd_remap_copybuf = dma->dp_cb_remaping;
2935 #endif
2936 }
2937 
2938 
2939 /*
2940  * rootnex_setup_cookie()
2941  *    Called in the bind slow path when the sgl uses the copy buffer. If any of
2942  *    the sgl uses the copy buffer, we need to go through each cookie, figure
2943  *    out if it uses the copy buffer, and if it does, save away everything we'll
2944  *    need during sync.
2945  */
2946 static void
2947 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma,
2948     ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used,
2949     page_t **cur_pp)
2950 {
2951 	boolean_t copybuf_sz_power_2;
2952 	rootnex_sglinfo_t *sinfo;
2953 	uint_t pidx;
2954 	uint_t pcnt;
2955 	off_t poff;
2956 #if defined(__amd64)
2957 	pfn_t pfn;
2958 #else
2959 	page_t **pplist;
2960 #endif
2961 
2962 	sinfo = &dma->dp_sglinfo;
2963 
2964 	/*
2965 	 * Calculate the page index relative to the start of the buffer. The
2966 	 * index to the current page for our buffer is the offset into the
2967 	 * first page of the buffer plus our current offset into the buffer
2968 	 * itself, shifted of course...
2969 	 */
2970 	pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT;
2971 	ASSERT(pidx < sinfo->si_max_pages);
2972 
2973 	/* if this cookie uses the copy buffer */
2974 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
2975 		/*
2976 		 * NOTE: we know that since this cookie uses the copy buffer, it
2977 		 * is <= MMU_PAGESIZE.
2978 		 */
2979 
2980 		/*
2981 		 * get the offset into the page. For the 64-bit kernel, get the
2982 		 * pfn which we'll use with seg kpm.
2983 		 */
2984 		poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET;
2985 #if defined(__amd64)
2986 		pfn = cookie->_dmu._dmac_ll >> MMU_PAGESHIFT;
2987 #endif
2988 
2989 		/* figure out if the copybuf size is a power of 2 */
2990 		if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) {
2991 			copybuf_sz_power_2 = B_FALSE;
2992 		} else {
2993 			copybuf_sz_power_2 = B_TRUE;
2994 		}
2995 
2996 		/* This page uses the copy buffer */
2997 		dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE;
2998 
2999 		/*
3000 		 * save the copy buffer KVA that we'll use with this page.
3001 		 * if we still fit within the copybuf, it's a simple add.
3002 		 * otherwise, we need to wrap over using & or % accordingly.
3003 		 */
3004 		if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) {
3005 			dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr +
3006 			    *copybuf_used;
3007 		} else {
3008 			if (copybuf_sz_power_2) {
3009 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3010 				    (uintptr_t)dma->dp_cbaddr +
3011 				    (*copybuf_used &
3012 				    (dma->dp_copybuf_size - 1)));
3013 			} else {
3014 				dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)(
3015 				    (uintptr_t)dma->dp_cbaddr +
3016 				    (*copybuf_used % dma->dp_copybuf_size));
3017 			}
3018 		}
3019 
3020 		/*
3021 		 * over write the cookie physical address with the address of
3022 		 * the physical address of the copy buffer page that we will
3023 		 * use.
3024 		 */
3025 		cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat,
3026 		    dma->dp_pgmap[pidx].pm_cbaddr)) + poff;
3027 
3028 		/* if we have a kernel VA, it's easy, just save that address */
3029 		if ((dmar_object->dmao_type != DMA_OTYP_PAGES) &&
3030 		    (sinfo->si_asp == &kas)) {
3031 			/*
3032 			 * save away the page aligned virtual address of the
3033 			 * driver buffer. Offsets are handled in the sync code.
3034 			 */
3035 			dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t)
3036 			    dmar_object->dmao_obj.virt_obj.v_addr + cur_offset)
3037 			    & MMU_PAGEMASK);
3038 #if !defined(__amd64)
3039 			/*
3040 			 * we didn't need to, and will never need to map this
3041 			 * page.
3042 			 */
3043 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3044 #endif
3045 
3046 		/* we don't have a kernel VA. We need one for the bcopy. */
3047 		} else {
3048 #if defined(__amd64)
3049 			/*
3050 			 * for the 64-bit kernel, it's easy. We use seg kpm to
3051 			 * get a Kernel VA for the corresponding pfn.
3052 			 */
3053 			dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn);
3054 #else
3055 			/*
3056 			 * for the 32-bit kernel, this is a pain. First we'll
3057 			 * save away the page_t or user VA for this page. This
3058 			 * is needed in rootnex_dma_win() when we switch to a
3059 			 * new window which requires us to re-map the copy
3060 			 * buffer.
3061 			 */
3062 			pplist = dmar_object->dmao_obj.virt_obj.v_priv;
3063 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3064 				dma->dp_pgmap[pidx].pm_pp = *cur_pp;
3065 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3066 			} else if (pplist != NULL) {
3067 				dma->dp_pgmap[pidx].pm_pp = pplist[pidx];
3068 				dma->dp_pgmap[pidx].pm_vaddr = NULL;
3069 			} else {
3070 				dma->dp_pgmap[pidx].pm_pp = NULL;
3071 				dma->dp_pgmap[pidx].pm_vaddr = (caddr_t)
3072 				    (((uintptr_t)
3073 				    dmar_object->dmao_obj.virt_obj.v_addr +
3074 				    cur_offset) & MMU_PAGEMASK);
3075 			}
3076 
3077 			/*
3078 			 * save away the page aligned virtual address which was
3079 			 * allocated from the kernel heap arena (taking into
3080 			 * account if we need more copy buffer than we alloced
3081 			 * and use multiple windows to handle this, i.e. &,%).
3082 			 * NOTE: there isn't and physical memory backing up this
3083 			 * virtual address space currently.
3084 			 */
3085 			if ((*copybuf_used + MMU_PAGESIZE) <=
3086 			    dma->dp_copybuf_size) {
3087 				dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3088 				    (((uintptr_t)dma->dp_kva + *copybuf_used) &
3089 				    MMU_PAGEMASK);
3090 			} else {
3091 				if (copybuf_sz_power_2) {
3092 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3093 					    (((uintptr_t)dma->dp_kva +
3094 					    (*copybuf_used &
3095 					    (dma->dp_copybuf_size - 1))) &
3096 					    MMU_PAGEMASK);
3097 				} else {
3098 					dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)
3099 					    (((uintptr_t)dma->dp_kva +
3100 					    (*copybuf_used %
3101 					    dma->dp_copybuf_size)) &
3102 					    MMU_PAGEMASK);
3103 				}
3104 			}
3105 
3106 			/*
3107 			 * if we haven't used up the available copy buffer yet,
3108 			 * map the kva to the physical page.
3109 			 */
3110 			if (!dma->dp_cb_remaping && ((*copybuf_used +
3111 			    MMU_PAGESIZE) <= dma->dp_copybuf_size)) {
3112 				dma->dp_pgmap[pidx].pm_mapped = B_TRUE;
3113 				if (dma->dp_pgmap[pidx].pm_pp != NULL) {
3114 					i86_pp_map(dma->dp_pgmap[pidx].pm_pp,
3115 					    dma->dp_pgmap[pidx].pm_kaddr);
3116 				} else {
3117 					i86_va_map(dma->dp_pgmap[pidx].pm_vaddr,
3118 					    sinfo->si_asp,
3119 					    dma->dp_pgmap[pidx].pm_kaddr);
3120 				}
3121 
3122 			/*
3123 			 * we've used up the available copy buffer, this page
3124 			 * will have to be mapped during rootnex_dma_win() when
3125 			 * we switch to a new window which requires a re-map
3126 			 * the copy buffer. (32-bit kernel only)
3127 			 */
3128 			} else {
3129 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3130 			}
3131 #endif
3132 			/* go to the next page_t */
3133 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3134 				*cur_pp = (*cur_pp)->p_next;
3135 			}
3136 		}
3137 
3138 		/* add to the copy buffer count */
3139 		*copybuf_used += MMU_PAGESIZE;
3140 
3141 	/*
3142 	 * This cookie doesn't use the copy buffer. Walk through the pages this
3143 	 * cookie occupies to reflect this.
3144 	 */
3145 	} else {
3146 		/*
3147 		 * figure out how many pages the cookie occupies. We need to
3148 		 * use the original page offset of the buffer and the cookies
3149 		 * offset in the buffer to do this.
3150 		 */
3151 		poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET;
3152 		pcnt = mmu_btopr(cookie->dmac_size + poff);
3153 
3154 		while (pcnt > 0) {
3155 #if !defined(__amd64)
3156 			/*
3157 			 * the 32-bit kernel doesn't have seg kpm, so we need
3158 			 * to map in the driver buffer (if it didn't come down
3159 			 * with a kernel VA) on the fly. Since this page doesn't
3160 			 * use the copy buffer, it's not, or will it ever, have
3161 			 * to be mapped in.
3162 			 */
3163 			dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3164 #endif
3165 			dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE;
3166 
3167 			/*
3168 			 * we need to update pidx and cur_pp or we'll loose
3169 			 * track of where we are.
3170 			 */
3171 			if (dmar_object->dmao_type == DMA_OTYP_PAGES) {
3172 				*cur_pp = (*cur_pp)->p_next;
3173 			}
3174 			pidx++;
3175 			pcnt--;
3176 		}
3177 	}
3178 }
3179 
3180 
3181 /*
3182  * rootnex_sgllen_window_boundary()
3183  *    Called in the bind slow path when the next cookie causes us to exceed (in
3184  *    this case == since we start at 0 and sgllen starts at 1) the maximum sgl
3185  *    length supported by the DMA H/W.
3186  */
3187 static int
3188 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3189     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr,
3190     off_t cur_offset)
3191 {
3192 	off_t new_offset;
3193 	size_t trim_sz;
3194 	off_t coffset;
3195 
3196 
3197 	/*
3198 	 * if we know we'll never have to trim, it's pretty easy. Just move to
3199 	 * the next window and init it. We're done.
3200 	 */
3201 	if (!dma->dp_trim_required) {
3202 		(*windowp)++;
3203 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3204 		(*windowp)->wd_cookie_cnt++;
3205 		(*windowp)->wd_size = cookie->dmac_size;
3206 		return (DDI_SUCCESS);
3207 	}
3208 
3209 	/* figure out how much we need to trim from the window */
3210 	ASSERT(attr->dma_attr_granular != 0);
3211 	if (dma->dp_granularity_power_2) {
3212 		trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1);
3213 	} else {
3214 		trim_sz = (*windowp)->wd_size % attr->dma_attr_granular;
3215 	}
3216 
3217 	/* The window's a whole multiple of granularity. We're done */
3218 	if (trim_sz == 0) {
3219 		(*windowp)++;
3220 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3221 		(*windowp)->wd_cookie_cnt++;
3222 		(*windowp)->wd_size = cookie->dmac_size;
3223 		return (DDI_SUCCESS);
3224 	}
3225 
3226 	/*
3227 	 * The window's not a whole multiple of granularity, since we know this
3228 	 * is due to the sgllen, we need to go back to the last cookie and trim
3229 	 * that one, add the left over part of the old cookie into the new
3230 	 * window, and then add in the new cookie into the new window.
3231 	 */
3232 
3233 	/*
3234 	 * make sure the driver isn't making us do something bad... Trimming and
3235 	 * sgllen == 1 don't go together.
3236 	 */
3237 	if (attr->dma_attr_sgllen == 1) {
3238 		return (DDI_DMA_NOMAPPING);
3239 	}
3240 
3241 	/*
3242 	 * first, setup the current window to account for the trim. Need to go
3243 	 * back to the last cookie for this.
3244 	 */
3245 	cookie--;
3246 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3247 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3248 	(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3249 	ASSERT(cookie->dmac_size > trim_sz);
3250 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3251 	(*windowp)->wd_size -= trim_sz;
3252 
3253 	/* save the buffer offsets for the next window */
3254 	coffset = cookie->dmac_size - trim_sz;
3255 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3256 
3257 	/*
3258 	 * set this now in case this is the first window. all other cases are
3259 	 * set in dma_win()
3260 	 */
3261 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3262 
3263 	/*
3264 	 * initialize the next window using what's left over in the previous
3265 	 * cookie.
3266 	 */
3267 	(*windowp)++;
3268 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3269 	(*windowp)->wd_cookie_cnt++;
3270 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3271 	(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset;
3272 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3273 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3274 		(*windowp)->wd_dosync = B_TRUE;
3275 	}
3276 
3277 	/*
3278 	 * now go back to the current cookie and add it to the new window. set
3279 	 * the new window size to the what was left over from the previous
3280 	 * cookie and what's in the current cookie.
3281 	 */
3282 	cookie++;
3283 	(*windowp)->wd_cookie_cnt++;
3284 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3285 
3286 	/*
3287 	 * trim plus the next cookie could put us over maxxfer (a cookie can be
3288 	 * a max size of maxxfer). Handle that case.
3289 	 */
3290 	if ((*windowp)->wd_size > dma->dp_maxxfer) {
3291 		/*
3292 		 * maxxfer is already a whole multiple of granularity, and this
3293 		 * trim will be <= the previous trim (since a cookie can't be
3294 		 * larger than maxxfer). Make things simple here.
3295 		 */
3296 		trim_sz = (*windowp)->wd_size - dma->dp_maxxfer;
3297 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3298 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3299 		(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3300 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3301 		(*windowp)->wd_size -= trim_sz;
3302 		ASSERT((*windowp)->wd_size == dma->dp_maxxfer);
3303 
3304 		/* save the buffer offsets for the next window */
3305 		coffset = cookie->dmac_size - trim_sz;
3306 		new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3307 
3308 		/* setup the next window */
3309 		(*windowp)++;
3310 		rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3311 		(*windowp)->wd_cookie_cnt++;
3312 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3313 		(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll +
3314 		    coffset;
3315 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3316 	}
3317 
3318 	return (DDI_SUCCESS);
3319 }
3320 
3321 
3322 /*
3323  * rootnex_copybuf_window_boundary()
3324  *    Called in bind slowpath when we get to a window boundary because we used
3325  *    up all the copy buffer that we have.
3326  */
3327 static int
3328 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3329     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset,
3330     size_t *copybuf_used)
3331 {
3332 	rootnex_sglinfo_t *sinfo;
3333 	off_t new_offset;
3334 	size_t trim_sz;
3335 	off_t coffset;
3336 	uint_t pidx;
3337 	off_t poff;
3338 
3339 
3340 	sinfo = &dma->dp_sglinfo;
3341 
3342 	/*
3343 	 * the copy buffer should be a whole multiple of page size. We know that
3344 	 * this cookie is <= MMU_PAGESIZE.
3345 	 */
3346 	ASSERT(cookie->dmac_size <= MMU_PAGESIZE);
3347 
3348 	/*
3349 	 * from now on, all new windows in this bind need to be re-mapped during
3350 	 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf
3351 	 * space...
3352 	 */
3353 #if !defined(__amd64)
3354 	dma->dp_cb_remaping = B_TRUE;
3355 #endif
3356 
3357 	/* reset copybuf used */
3358 	*copybuf_used = 0;
3359 
3360 	/*
3361 	 * if we don't have to trim (since granularity is set to 1), go to the
3362 	 * next window and add the current cookie to it. We know the current
3363 	 * cookie uses the copy buffer since we're in this code path.
3364 	 */
3365 	if (!dma->dp_trim_required) {
3366 		(*windowp)++;
3367 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3368 
3369 		/* Add this cookie to the new window */
3370 		(*windowp)->wd_cookie_cnt++;
3371 		(*windowp)->wd_size += cookie->dmac_size;
3372 		*copybuf_used += MMU_PAGESIZE;
3373 		return (DDI_SUCCESS);
3374 	}
3375 
3376 	/*
3377 	 * *** may need to trim, figure it out.
3378 	 */
3379 
3380 	/* figure out how much we need to trim from the window */
3381 	if (dma->dp_granularity_power_2) {
3382 		trim_sz = (*windowp)->wd_size &
3383 		    (hp->dmai_attr.dma_attr_granular - 1);
3384 	} else {
3385 		trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular;
3386 	}
3387 
3388 	/*
3389 	 * if the window's a whole multiple of granularity, go to the next
3390 	 * window, init it, then add in the current cookie. We know the current
3391 	 * cookie uses the copy buffer since we're in this code path.
3392 	 */
3393 	if (trim_sz == 0) {
3394 		(*windowp)++;
3395 		rootnex_init_win(hp, dma, *windowp, cookie, cur_offset);
3396 
3397 		/* Add this cookie to the new window */
3398 		(*windowp)->wd_cookie_cnt++;
3399 		(*windowp)->wd_size += cookie->dmac_size;
3400 		*copybuf_used += MMU_PAGESIZE;
3401 		return (DDI_SUCCESS);
3402 	}
3403 
3404 	/*
3405 	 * *** We figured it out, we definitly need to trim
3406 	 */
3407 
3408 	/*
3409 	 * make sure the driver isn't making us do something bad...
3410 	 * Trimming and sgllen == 1 don't go together.
3411 	 */
3412 	if (hp->dmai_attr.dma_attr_sgllen == 1) {
3413 		return (DDI_DMA_NOMAPPING);
3414 	}
3415 
3416 	/*
3417 	 * first, setup the current window to account for the trim. Need to go
3418 	 * back to the last cookie for this. Some of the last cookie will be in
3419 	 * the current window, and some of the last cookie will be in the new
3420 	 * window. All of the current cookie will be in the new window.
3421 	 */
3422 	cookie--;
3423 	(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3424 	(*windowp)->wd_trim.tr_last_cookie = cookie;
3425 	(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3426 	ASSERT(cookie->dmac_size > trim_sz);
3427 	(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3428 	(*windowp)->wd_size -= trim_sz;
3429 
3430 	/*
3431 	 * we're trimming the last cookie (not the current cookie). So that
3432 	 * last cookie may have or may not have been using the copy buffer (
3433 	 * we know the cookie passed in uses the copy buffer since we're in
3434 	 * this code path).
3435 	 *
3436 	 * If the last cookie doesn't use the copy buffer, nothing special to
3437 	 * do. However, if it does uses the copy buffer, it will be both the
3438 	 * last page in the current window and the first page in the next
3439 	 * window. Since we are reusing the copy buffer (and KVA space on the
3440 	 * 32-bit kernel), this page will use the end of the copy buffer in the
3441 	 * current window, and the start of the copy buffer in the next window.
3442 	 * Track that info... The cookie physical address was already set to
3443 	 * the copy buffer physical address in setup_cookie..
3444 	 */
3445 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3446 		pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset +
3447 		    (*windowp)->wd_size) >> MMU_PAGESHIFT;
3448 		(*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE;
3449 		(*windowp)->wd_trim.tr_last_pidx = pidx;
3450 		(*windowp)->wd_trim.tr_last_cbaddr =
3451 		    dma->dp_pgmap[pidx].pm_cbaddr;
3452 #if !defined(__amd64)
3453 		(*windowp)->wd_trim.tr_last_kaddr =
3454 		    dma->dp_pgmap[pidx].pm_kaddr;
3455 #endif
3456 	}
3457 
3458 	/* save the buffer offsets for the next window */
3459 	coffset = cookie->dmac_size - trim_sz;
3460 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3461 
3462 	/*
3463 	 * set this now in case this is the first window. all other cases are
3464 	 * set in dma_win()
3465 	 */
3466 	cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3467 
3468 	/*
3469 	 * initialize the next window using what's left over in the previous
3470 	 * cookie.
3471 	 */
3472 	(*windowp)++;
3473 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3474 	(*windowp)->wd_cookie_cnt++;
3475 	(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3476 	(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset;
3477 	(*windowp)->wd_trim.tr_first_size = trim_sz;
3478 
3479 	/*
3480 	 * again, we're tracking if the last cookie uses the copy buffer.
3481 	 * read the comment above for more info on why we need to track
3482 	 * additional state.
3483 	 *
3484 	 * For the first cookie in the new window, we need reset the physical
3485 	 * address to DMA into to the start of the copy buffer plus any
3486 	 * initial page offset which may be present.
3487 	 */
3488 	if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) {
3489 		(*windowp)->wd_dosync = B_TRUE;
3490 		(*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE;
3491 		(*windowp)->wd_trim.tr_first_pidx = pidx;
3492 		(*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr;
3493 		poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET;
3494 		(*windowp)->wd_trim.tr_first_paddr = ptob64(hat_getpfnum(
3495 		    kas.a_hat, dma->dp_cbaddr)) + poff;
3496 #if !defined(__amd64)
3497 		(*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva;
3498 #endif
3499 		/* account for the cookie copybuf usage in the new window */
3500 		*copybuf_used += MMU_PAGESIZE;
3501 
3502 		/*
3503 		 * every piece of code has to have a hack, and here is this
3504 		 * ones :-)
3505 		 *
3506 		 * There is a complex interaction between setup_cookie and the
3507 		 * copybuf window boundary. The complexity had to be in either
3508 		 * the maxxfer window, or the copybuf window, and I chose the
3509 		 * copybuf code.
3510 		 *
3511 		 * So in this code path, we have taken the last cookie,
3512 		 * virtually broken it in half due to the trim, and it happens
3513 		 * to use the copybuf which further complicates life. At the
3514 		 * same time, we have already setup the current cookie, which
3515 		 * is now wrong. More background info: the current cookie uses
3516 		 * the copybuf, so it is only a page long max. So we need to
3517 		 * fix the current cookies copy buffer address, physical
3518 		 * address, and kva for the 32-bit kernel. We due this by
3519 		 * bumping them by page size (of course, we can't due this on
3520 		 * the physical address since the copy buffer may not be
3521 		 * physically contiguous).
3522 		 */
3523 		cookie++;
3524 		dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE;
3525 		poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET;
3526 		cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat,
3527 		    dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff;
3528 #if !defined(__amd64)
3529 		ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE);
3530 		dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE;
3531 #endif
3532 	} else {
3533 		/* go back to the current cookie */
3534 		cookie++;
3535 	}
3536 
3537 	/*
3538 	 * add the current cookie to the new window. set the new window size to
3539 	 * the what was left over from the previous cookie and what's in the
3540 	 * current cookie.
3541 	 */
3542 	(*windowp)->wd_cookie_cnt++;
3543 	(*windowp)->wd_size = trim_sz + cookie->dmac_size;
3544 	ASSERT((*windowp)->wd_size < dma->dp_maxxfer);
3545 
3546 	/*
3547 	 * we know that the cookie passed in always uses the copy buffer. We
3548 	 * wouldn't be here if it didn't.
3549 	 */
3550 	*copybuf_used += MMU_PAGESIZE;
3551 
3552 	return (DDI_SUCCESS);
3553 }
3554 
3555 
3556 /*
3557  * rootnex_maxxfer_window_boundary()
3558  *    Called in bind slowpath when we get to a window boundary because we will
3559  *    go over maxxfer.
3560  */
3561 static int
3562 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma,
3563     rootnex_window_t **windowp, ddi_dma_cookie_t *cookie)
3564 {
3565 	size_t dmac_size;
3566 	off_t new_offset;
3567 	size_t trim_sz;
3568 	off_t coffset;
3569 
3570 
3571 	/*
3572 	 * calculate how much we have to trim off of the current cookie to equal
3573 	 * maxxfer. We don't have to account for granularity here since our
3574 	 * maxxfer already takes that into account.
3575 	 */
3576 	trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer;
3577 	ASSERT(trim_sz <= cookie->dmac_size);
3578 	ASSERT(trim_sz <= dma->dp_maxxfer);
3579 
3580 	/* save cookie size since we need it later and we might change it */
3581 	dmac_size = cookie->dmac_size;
3582 
3583 	/*
3584 	 * if we're not trimming the entire cookie, setup the current window to
3585 	 * account for the trim.
3586 	 */
3587 	if (trim_sz < cookie->dmac_size) {
3588 		(*windowp)->wd_cookie_cnt++;
3589 		(*windowp)->wd_trim.tr_trim_last = B_TRUE;
3590 		(*windowp)->wd_trim.tr_last_cookie = cookie;
3591 		(*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll;
3592 		(*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz;
3593 		(*windowp)->wd_size = dma->dp_maxxfer;
3594 
3595 		/*
3596 		 * set the adjusted cookie size now in case this is the first
3597 		 * window. All other windows are taken care of in get win
3598 		 */
3599 		cookie->dmac_size = (*windowp)->wd_trim.tr_last_size;
3600 	}
3601 
3602 	/*
3603 	 * coffset is the current offset within the cookie, new_offset is the
3604 	 * current offset with the entire buffer.
3605 	 */
3606 	coffset = dmac_size - trim_sz;
3607 	new_offset = (*windowp)->wd_offset + (*windowp)->wd_size;
3608 
3609 	/* initialize the next window */
3610 	(*windowp)++;
3611 	rootnex_init_win(hp, dma, *windowp, cookie, new_offset);
3612 	(*windowp)->wd_cookie_cnt++;
3613 	(*windowp)->wd_size = trim_sz;
3614 	if (trim_sz < dmac_size) {
3615 		(*windowp)->wd_trim.tr_trim_first = B_TRUE;
3616 		(*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll +
3617 		    coffset;
3618 		(*windowp)->wd_trim.tr_first_size = trim_sz;
3619 	}
3620 
3621 	return (DDI_SUCCESS);
3622 }
3623 
3624 
3625 /*
3626  * rootnex_dma_sync()
3627  *    called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags.
3628  *    We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC
3629  *    is set, ddi_dma_sync() returns immediately passing back success.
3630  */
3631 /*ARGSUSED*/
3632 static int
3633 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
3634     off_t off, size_t len, uint_t cache_flags)
3635 {
3636 	rootnex_sglinfo_t *sinfo;
3637 	rootnex_pgmap_t *cbpage;
3638 	rootnex_window_t *win;
3639 	ddi_dma_impl_t *hp;
3640 	rootnex_dma_t *dma;
3641 	caddr_t fromaddr;
3642 	caddr_t toaddr;
3643 	uint_t psize;
3644 	off_t offset;
3645 	uint_t pidx;
3646 	size_t size;
3647 	off_t poff;
3648 	int e;
3649 
3650 
3651 	hp = (ddi_dma_impl_t *)handle;
3652 	dma = (rootnex_dma_t *)hp->dmai_private;
3653 	sinfo = &dma->dp_sglinfo;
3654 
3655 	/*
3656 	 * if we don't have any windows, we don't need to sync. A copybuf
3657 	 * will cause us to have at least one window.
3658 	 */
3659 	if (dma->dp_window == NULL) {
3660 		return (DDI_SUCCESS);
3661 	}
3662 
3663 	/* This window may not need to be sync'd */
3664 	win = &dma->dp_window[dma->dp_current_win];
3665 	if (!win->wd_dosync) {
3666 		return (DDI_SUCCESS);
3667 	}
3668 
3669 	/* handle off and len special cases */
3670 	if ((off == 0) || (rootnex_sync_ignore_params)) {
3671 		offset = win->wd_offset;
3672 	} else {
3673 		offset = off;
3674 	}
3675 	if ((len == 0) || (rootnex_sync_ignore_params)) {
3676 		size = win->wd_size;
3677 	} else {
3678 		size = len;
3679 	}
3680 
3681 	/* check the sync args to make sure they make a little sense */
3682 	if (rootnex_sync_check_parms) {
3683 		e = rootnex_valid_sync_parms(hp, win, offset, size,
3684 		    cache_flags);
3685 		if (e != DDI_SUCCESS) {
3686 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]);
3687 			return (DDI_FAILURE);
3688 		}
3689 	}
3690 
3691 	/*
3692 	 * special case the first page to handle the offset into the page. The
3693 	 * offset to the current page for our buffer is the offset into the
3694 	 * first page of the buffer plus our current offset into the buffer
3695 	 * itself, masked of course.
3696 	 */
3697 	poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET;
3698 	psize = MIN((MMU_PAGESIZE - poff), size);
3699 
3700 	/* go through all the pages that we want to sync */
3701 	while (size > 0) {
3702 		/*
3703 		 * Calculate the page index relative to the start of the buffer.
3704 		 * The index to the current page for our buffer is the offset
3705 		 * into the first page of the buffer plus our current offset
3706 		 * into the buffer itself, shifted of course...
3707 		 */
3708 		pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT;
3709 		ASSERT(pidx < sinfo->si_max_pages);
3710 
3711 		/*
3712 		 * if this page uses the copy buffer, we need to sync it,
3713 		 * otherwise, go on to the next page.
3714 		 */
3715 		cbpage = &dma->dp_pgmap[pidx];
3716 		ASSERT((cbpage->pm_uses_copybuf == B_TRUE) ||
3717 		    (cbpage->pm_uses_copybuf == B_FALSE));
3718 		if (cbpage->pm_uses_copybuf) {
3719 			/* cbaddr and kaddr should be page aligned */
3720 			ASSERT(((uintptr_t)cbpage->pm_cbaddr &
3721 			    MMU_PAGEOFFSET) == 0);
3722 			ASSERT(((uintptr_t)cbpage->pm_kaddr &
3723 			    MMU_PAGEOFFSET) == 0);
3724 
3725 			/*
3726 			 * if we're copying for the device, we are going to
3727 			 * copy from the drivers buffer and to the rootnex
3728 			 * allocated copy buffer.
3729 			 */
3730 			if (cache_flags == DDI_DMA_SYNC_FORDEV) {
3731 				fromaddr = cbpage->pm_kaddr + poff;
3732 				toaddr = cbpage->pm_cbaddr + poff;
3733 				DTRACE_PROBE2(rootnex__sync__dev,
3734 				    dev_info_t *, dma->dp_dip, size_t, psize);
3735 
3736 			/*
3737 			 * if we're copying for the cpu/kernel, we are going to
3738 			 * copy from the rootnex allocated copy buffer to the
3739 			 * drivers buffer.
3740 			 */
3741 			} else {
3742 				fromaddr = cbpage->pm_cbaddr + poff;
3743 				toaddr = cbpage->pm_kaddr + poff;
3744 				DTRACE_PROBE2(rootnex__sync__cpu,
3745 				    dev_info_t *, dma->dp_dip, size_t, psize);
3746 			}
3747 
3748 			bcopy(fromaddr, toaddr, psize);
3749 		}
3750 
3751 		/*
3752 		 * decrement size until we're done, update our offset into the
3753 		 * buffer, and get the next page size.
3754 		 */
3755 		size -= psize;
3756 		offset += psize;
3757 		psize = MIN(MMU_PAGESIZE, size);
3758 
3759 		/* page offset is zero for the rest of this loop */
3760 		poff = 0;
3761 	}
3762 
3763 	return (DDI_SUCCESS);
3764 }
3765 
3766 
3767 /*
3768  * rootnex_valid_sync_parms()
3769  *    checks the parameters passed to sync to verify they are correct.
3770  */
3771 static int
3772 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win,
3773     off_t offset, size_t size, uint_t cache_flags)
3774 {
3775 	off_t woffset;
3776 
3777 
3778 	/*
3779 	 * the first part of the test to make sure the offset passed in is
3780 	 * within the window.
3781 	 */
3782 	if (offset < win->wd_offset) {
3783 		return (DDI_FAILURE);
3784 	}
3785 
3786 	/*
3787 	 * second and last part of the test to make sure the offset and length
3788 	 * passed in is within the window.
3789 	 */
3790 	woffset = offset - win->wd_offset;
3791 	if ((woffset + size) > win->wd_size) {
3792 		return (DDI_FAILURE);
3793 	}
3794 
3795 	/*
3796 	 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should
3797 	 * be set too.
3798 	 */
3799 	if ((cache_flags == DDI_DMA_SYNC_FORDEV) &&
3800 	    (hp->dmai_rflags & DDI_DMA_WRITE)) {
3801 		return (DDI_SUCCESS);
3802 	}
3803 
3804 	/*
3805 	 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL
3806 	 * should be set. Also DDI_DMA_READ should be set in the flags.
3807 	 */
3808 	if (((cache_flags == DDI_DMA_SYNC_FORCPU) ||
3809 	    (cache_flags == DDI_DMA_SYNC_FORKERNEL)) &&
3810 	    (hp->dmai_rflags & DDI_DMA_READ)) {
3811 		return (DDI_SUCCESS);
3812 	}
3813 
3814 	return (DDI_FAILURE);
3815 }
3816 
3817 
3818 /*
3819  * rootnex_dma_win()
3820  *    called from ddi_dma_getwin()
3821  */
3822 /*ARGSUSED*/
3823 static int
3824 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
3825     uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep,
3826     uint_t *ccountp)
3827 {
3828 	rootnex_window_t *window;
3829 	rootnex_trim_t *trim;
3830 	ddi_dma_impl_t *hp;
3831 	rootnex_dma_t *dma;
3832 #if !defined(__amd64)
3833 	rootnex_sglinfo_t *sinfo;
3834 	rootnex_pgmap_t *pmap;
3835 	uint_t pidx;
3836 	uint_t pcnt;
3837 	off_t poff;
3838 	int i;
3839 #endif
3840 
3841 
3842 	hp = (ddi_dma_impl_t *)handle;
3843 	dma = (rootnex_dma_t *)hp->dmai_private;
3844 #if !defined(__amd64)
3845 	sinfo = &dma->dp_sglinfo;
3846 #endif
3847 
3848 	/* If we try and get a window which doesn't exist, return failure */
3849 	if (win >= hp->dmai_nwin) {
3850 		ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
3851 		return (DDI_FAILURE);
3852 	}
3853 
3854 	/*
3855 	 * if we don't have any windows, and they're asking for the first
3856 	 * window, setup the cookie pointer to the first cookie in the bind.
3857 	 * setup our return values, then increment the cookie since we return
3858 	 * the first cookie on the stack.
3859 	 */
3860 	if (dma->dp_window == NULL) {
3861 		if (win != 0) {
3862 			ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]);
3863 			return (DDI_FAILURE);
3864 		}
3865 		hp->dmai_cookie = dma->dp_cookies;
3866 		*offp = 0;
3867 		*lenp = dma->dp_dma.dmao_size;
3868 		*ccountp = dma->dp_sglinfo.si_sgl_size;
3869 		*cookiep = hp->dmai_cookie[0];
3870 		hp->dmai_cookie++;
3871 		return (DDI_SUCCESS);
3872 	}
3873 
3874 	/* sync the old window before moving on to the new one */
3875 	window = &dma->dp_window[dma->dp_current_win];
3876 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) {
3877 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
3878 		    DDI_DMA_SYNC_FORCPU);
3879 	}
3880 
3881 #if !defined(__amd64)
3882 	/*
3883 	 * before we move to the next window, if we need to re-map, unmap all
3884 	 * the pages in this window.
3885 	 */
3886 	if (dma->dp_cb_remaping) {
3887 		/*
3888 		 * If we switch to this window again, we'll need to map in
3889 		 * on the fly next time.
3890 		 */
3891 		window->wd_remap_copybuf = B_TRUE;
3892 
3893 		/*
3894 		 * calculate the page index into the buffer where this window
3895 		 * starts, and the number of pages this window takes up.
3896 		 */
3897 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
3898 		    MMU_PAGESHIFT;
3899 		poff = (sinfo->si_buf_offset + window->wd_offset) &
3900 		    MMU_PAGEOFFSET;
3901 		pcnt = mmu_btopr(window->wd_size + poff);
3902 		ASSERT((pidx + pcnt) <= sinfo->si_max_pages);
3903 
3904 		/* unmap pages which are currently mapped in this window */
3905 		for (i = 0; i < pcnt; i++) {
3906 			if (dma->dp_pgmap[pidx].pm_mapped) {
3907 				hat_unload(kas.a_hat,
3908 				    dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE,
3909 				    HAT_UNLOAD);
3910 				dma->dp_pgmap[pidx].pm_mapped = B_FALSE;
3911 			}
3912 			pidx++;
3913 		}
3914 	}
3915 #endif
3916 
3917 	/*
3918 	 * Move to the new window.
3919 	 * NOTE: current_win must be set for sync to work right
3920 	 */
3921 	dma->dp_current_win = win;
3922 	window = &dma->dp_window[win];
3923 
3924 	/* if needed, adjust the first and/or last cookies for trim */
3925 	trim = &window->wd_trim;
3926 	if (trim->tr_trim_first) {
3927 		window->wd_first_cookie->_dmu._dmac_ll = trim->tr_first_paddr;
3928 		window->wd_first_cookie->dmac_size = trim->tr_first_size;
3929 #if !defined(__amd64)
3930 		window->wd_first_cookie->dmac_type =
3931 		    (window->wd_first_cookie->dmac_type &
3932 		    ROOTNEX_USES_COPYBUF) + window->wd_offset;
3933 #endif
3934 		if (trim->tr_first_copybuf_win) {
3935 			dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr =
3936 			    trim->tr_first_cbaddr;
3937 #if !defined(__amd64)
3938 			dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr =
3939 			    trim->tr_first_kaddr;
3940 #endif
3941 		}
3942 	}
3943 	if (trim->tr_trim_last) {
3944 		trim->tr_last_cookie->_dmu._dmac_ll = trim->tr_last_paddr;
3945 		trim->tr_last_cookie->dmac_size = trim->tr_last_size;
3946 		if (trim->tr_last_copybuf_win) {
3947 			dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr =
3948 			    trim->tr_last_cbaddr;
3949 #if !defined(__amd64)
3950 			dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr =
3951 			    trim->tr_last_kaddr;
3952 #endif
3953 		}
3954 	}
3955 
3956 	/*
3957 	 * setup the cookie pointer to the first cookie in the window. setup
3958 	 * our return values, then increment the cookie since we return the
3959 	 * first cookie on the stack.
3960 	 */
3961 	hp->dmai_cookie = window->wd_first_cookie;
3962 	*offp = window->wd_offset;
3963 	*lenp = window->wd_size;
3964 	*ccountp = window->wd_cookie_cnt;
3965 	*cookiep = hp->dmai_cookie[0];
3966 	hp->dmai_cookie++;
3967 
3968 #if !defined(__amd64)
3969 	/* re-map copybuf if required for this window */
3970 	if (dma->dp_cb_remaping) {
3971 		/*
3972 		 * calculate the page index into the buffer where this
3973 		 * window starts.
3974 		 */
3975 		pidx = (sinfo->si_buf_offset + window->wd_offset) >>
3976 		    MMU_PAGESHIFT;
3977 		ASSERT(pidx < sinfo->si_max_pages);
3978 
3979 		/*
3980 		 * the first page can get unmapped if it's shared with the
3981 		 * previous window. Even if the rest of this window is already
3982 		 * mapped in, we need to still check this one.
3983 		 */
3984 		pmap = &dma->dp_pgmap[pidx];
3985 		if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) {
3986 			if (pmap->pm_pp != NULL) {
3987 				pmap->pm_mapped = B_TRUE;
3988 				i86_pp_map(pmap->pm_pp, pmap->pm_kaddr);
3989 			} else if (pmap->pm_vaddr != NULL) {
3990 				pmap->pm_mapped = B_TRUE;
3991 				i86_va_map(pmap->pm_vaddr, sinfo->si_asp,
3992 				    pmap->pm_kaddr);
3993 			}
3994 		}
3995 		pidx++;
3996 
3997 		/* map in the rest of the pages if required */
3998 		if (window->wd_remap_copybuf) {
3999 			window->wd_remap_copybuf = B_FALSE;
4000 
4001 			/* figure out many pages this window takes up */
4002 			poff = (sinfo->si_buf_offset + window->wd_offset) &
4003 			    MMU_PAGEOFFSET;
4004 			pcnt = mmu_btopr(window->wd_size + poff);
4005 			ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages);
4006 
4007 			/* map pages which require it */
4008 			for (i = 1; i < pcnt; i++) {
4009 				pmap = &dma->dp_pgmap[pidx];
4010 				if (pmap->pm_uses_copybuf) {
4011 					ASSERT(pmap->pm_mapped == B_FALSE);
4012 					if (pmap->pm_pp != NULL) {
4013 						pmap->pm_mapped = B_TRUE;
4014 						i86_pp_map(pmap->pm_pp,
4015 						    pmap->pm_kaddr);
4016 					} else if (pmap->pm_vaddr != NULL) {
4017 						pmap->pm_mapped = B_TRUE;
4018 						i86_va_map(pmap->pm_vaddr,
4019 						    sinfo->si_asp,
4020 						    pmap->pm_kaddr);
4021 					}
4022 				}
4023 				pidx++;
4024 			}
4025 		}
4026 	}
4027 #endif
4028 
4029 	/* if the new window uses the copy buffer, sync it for the device */
4030 	if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) {
4031 		(void) rootnex_dma_sync(dip, rdip, handle, 0, 0,
4032 		    DDI_DMA_SYNC_FORDEV);
4033 	}
4034 
4035 	return (DDI_SUCCESS);
4036 }
4037 
4038 
4039 
4040 /*
4041  * ************************
4042  *  obsoleted dma routines
4043  * ************************
4044  */
4045 
4046 /*
4047  * rootnex_dma_map()
4048  *    called from ddi_dma_setup()
4049  */
4050 /* ARGSUSED */
4051 static int
4052 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq,
4053     ddi_dma_handle_t *handlep)
4054 {
4055 #if defined(__amd64)
4056 	/*
4057 	 * this interface is not supported in 64-bit x86 kernel. See comment in
4058 	 * rootnex_dma_mctl()
4059 	 */
4060 	return (DDI_DMA_NORESOURCES);
4061 
4062 #else /* 32-bit x86 kernel */
4063 	ddi_dma_handle_t *lhandlep;
4064 	ddi_dma_handle_t lhandle;
4065 	ddi_dma_cookie_t cookie;
4066 	ddi_dma_attr_t dma_attr;
4067 	ddi_dma_lim_t *dma_lim;
4068 	uint_t ccnt;
4069 	int e;
4070 
4071 
4072 	/*
4073 	 * if the driver is just testing to see if it's possible to do the bind,
4074 	 * we'll use local state. Otherwise, use the handle pointer passed in.
4075 	 */
4076 	if (handlep == NULL) {
4077 		lhandlep = &lhandle;
4078 	} else {
4079 		lhandlep = handlep;
4080 	}
4081 
4082 	/* convert the limit structure to a dma_attr one */
4083 	dma_lim = dmareq->dmar_limits;
4084 	dma_attr.dma_attr_version = DMA_ATTR_V0;
4085 	dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo;
4086 	dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi;
4087 	dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer;
4088 	dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max;
4089 	dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max;
4090 	dma_attr.dma_attr_granular = dma_lim->dlim_granular;
4091 	dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen;
4092 	dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize;
4093 	dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes;
4094 	dma_attr.dma_attr_align = MMU_PAGESIZE;
4095 	dma_attr.dma_attr_flags = 0;
4096 
4097 	e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp,
4098 	    dmareq->dmar_arg, lhandlep);
4099 	if (e != DDI_SUCCESS) {
4100 		return (e);
4101 	}
4102 
4103 	e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt);
4104 	if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) {
4105 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4106 		return (e);
4107 	}
4108 
4109 	/*
4110 	 * if the driver is just testing to see if it's possible to do the bind,
4111 	 * free up the local state and return the result.
4112 	 */
4113 	if (handlep == NULL) {
4114 		(void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep);
4115 		(void) rootnex_dma_freehdl(dip, rdip, *lhandlep);
4116 		if (e == DDI_DMA_MAPPED) {
4117 			return (DDI_DMA_MAPOK);
4118 		} else {
4119 			return (DDI_DMA_NOMAPPING);
4120 		}
4121 	}
4122 
4123 	return (e);
4124 #endif /* defined(__amd64) */
4125 }
4126 
4127 
4128 /*
4129  * rootnex_dma_mctl()
4130  *
4131  */
4132 /* ARGSUSED */
4133 static int
4134 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
4135     enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp,
4136     uint_t cache_flags)
4137 {
4138 #if defined(__amd64)
4139 	/*
4140 	 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a
4141 	 * common implementation in genunix, so they no longer have x86
4142 	 * specific functionality which called into dma_ctl.
4143 	 *
4144 	 * The rest of the obsoleted interfaces were never supported in the
4145 	 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface
4146 	 * was not ported to the x86 64-bit kernel do to serious x86 rootnex
4147 	 * implementation issues.
4148 	 *
4149 	 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and
4150 	 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we
4151 	 * reflect that now too...
4152 	 *
4153 	 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are
4154 	 * not going to put this functionality into the 64-bit x86 kernel now.
4155 	 * It wasn't ported to the 64-bit kernel for s10, no reason to change
4156 	 * that in a future release.
4157 	 */
4158 	return (DDI_FAILURE);
4159 
4160 #else /* 32-bit x86 kernel */
4161 	ddi_dma_cookie_t lcookie;
4162 	ddi_dma_cookie_t *cookie;
4163 	rootnex_window_t *window;
4164 	ddi_dma_impl_t *hp;
4165 	rootnex_dma_t *dma;
4166 	uint_t nwin;
4167 	uint_t ccnt;
4168 	size_t len;
4169 	off_t off;
4170 	int e;
4171 
4172 
4173 	/*
4174 	 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little
4175 	 * hacky since were optimizing for the current interfaces and so we can
4176 	 * cleanup the mess in genunix. Hopefully we will remove the this
4177 	 * obsoleted routines someday soon.
4178 	 */
4179 
4180 	switch (request) {
4181 
4182 	case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */
4183 		hp = (ddi_dma_impl_t *)handle;
4184 		cookie = (ddi_dma_cookie_t *)objpp;
4185 
4186 		/*
4187 		 * convert segment to cookie. We don't distinguish between the
4188 		 * two :-)
4189 		 */
4190 		*cookie = *hp->dmai_cookie;
4191 		*lenp = cookie->dmac_size;
4192 		*offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF;
4193 		return (DDI_SUCCESS);
4194 
4195 	case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */
4196 		hp = (ddi_dma_impl_t *)handle;
4197 		dma = (rootnex_dma_t *)hp->dmai_private;
4198 
4199 		if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) {
4200 			return (DDI_DMA_STALE);
4201 		}
4202 
4203 		/* handle the case where we don't have any windows */
4204 		if (dma->dp_window == NULL) {
4205 			/*
4206 			 * if seg == NULL, and we don't have any windows,
4207 			 * return the first cookie in the sgl.
4208 			 */
4209 			if (*lenp == NULL) {
4210 				dma->dp_current_cookie = 0;
4211 				hp->dmai_cookie = dma->dp_cookies;
4212 				*objpp = (caddr_t)handle;
4213 				return (DDI_SUCCESS);
4214 
4215 			/* if we have more cookies, go to the next cookie */
4216 			} else {
4217 				if ((dma->dp_current_cookie + 1) >=
4218 				    dma->dp_sglinfo.si_sgl_size) {
4219 					return (DDI_DMA_DONE);
4220 				}
4221 				dma->dp_current_cookie++;
4222 				hp->dmai_cookie++;
4223 				return (DDI_SUCCESS);
4224 			}
4225 		}
4226 
4227 		/* We have one or more windows */
4228 		window = &dma->dp_window[dma->dp_current_win];
4229 
4230 		/*
4231 		 * if seg == NULL, return the first cookie in the current
4232 		 * window
4233 		 */
4234 		if (*lenp == NULL) {
4235 			dma->dp_current_cookie = 0;
4236 			hp->dmai_cookie = window->wd_first_cookie;
4237 
4238 		/*
4239 		 * go to the next cookie in the window then see if we done with
4240 		 * this window.
4241 		 */
4242 		} else {
4243 			if ((dma->dp_current_cookie + 1) >=
4244 			    window->wd_cookie_cnt) {
4245 				return (DDI_DMA_DONE);
4246 			}
4247 			dma->dp_current_cookie++;
4248 			hp->dmai_cookie++;
4249 		}
4250 		*objpp = (caddr_t)handle;
4251 		return (DDI_SUCCESS);
4252 
4253 	case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */
4254 		hp = (ddi_dma_impl_t *)handle;
4255 		dma = (rootnex_dma_t *)hp->dmai_private;
4256 
4257 		if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) {
4258 			return (DDI_DMA_STALE);
4259 		}
4260 
4261 		/* if win == NULL, return the first window in the bind */
4262 		if (*offp == NULL) {
4263 			nwin = 0;
4264 
4265 		/*
4266 		 * else, go to the next window then see if we're done with all
4267 		 * the windows.
4268 		 */
4269 		} else {
4270 			nwin = dma->dp_current_win + 1;
4271 			if (nwin >= hp->dmai_nwin) {
4272 				return (DDI_DMA_DONE);
4273 			}
4274 		}
4275 
4276 		/* switch to the next window */
4277 		e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len,
4278 		    &lcookie, &ccnt);
4279 		ASSERT(e == DDI_SUCCESS);
4280 		if (e != DDI_SUCCESS) {
4281 			return (DDI_DMA_STALE);
4282 		}
4283 
4284 		/* reset the cookie back to the first cookie in the window */
4285 		if (dma->dp_window != NULL) {
4286 			window = &dma->dp_window[dma->dp_current_win];
4287 			hp->dmai_cookie = window->wd_first_cookie;
4288 		} else {
4289 			hp->dmai_cookie = dma->dp_cookies;
4290 		}
4291 
4292 		*objpp = (caddr_t)handle;
4293 		return (DDI_SUCCESS);
4294 
4295 	case DDI_DMA_FREE: /* ddi_dma_free() */
4296 		(void) rootnex_dma_unbindhdl(dip, rdip, handle);
4297 		(void) rootnex_dma_freehdl(dip, rdip, handle);
4298 		if (rootnex_state->r_dvma_call_list_id) {
4299 			ddi_run_callback(&rootnex_state->r_dvma_call_list_id);
4300 		}
4301 		return (DDI_SUCCESS);
4302 
4303 	case DDI_DMA_IOPB_ALLOC:	/* get contiguous DMA-able memory */
4304 	case DDI_DMA_SMEM_ALLOC:	/* get contiguous DMA-able memory */
4305 		/* should never get here, handled in genunix */
4306 		ASSERT(0);
4307 		return (DDI_FAILURE);
4308 
4309 	case DDI_DMA_KVADDR:
4310 	case DDI_DMA_GETERR:
4311 	case DDI_DMA_COFF:
4312 		return (DDI_FAILURE);
4313 	}
4314 
4315 	return (DDI_FAILURE);
4316 #endif /* defined(__amd64) */
4317 }
4318 
4319 
4320 /*
4321  * *********
4322  *  FMA Code
4323  * *********
4324  */
4325 
4326 /*
4327  * rootnex_fm_init()
4328  *    FMA init busop
4329  */
4330 /* ARGSUSED */
4331 static int
4332 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap,
4333     ddi_iblock_cookie_t *ibc)
4334 {
4335 	*ibc = rootnex_state->r_err_ibc;
4336 
4337 	return (ddi_system_fmcap);
4338 }
4339 
4340 /*
4341  * rootnex_dma_check()
4342  *    Function called after a dma fault occurred to find out whether the
4343  *    fault address is associated with a driver that is able to handle faults
4344  *    and recover from faults.
4345  */
4346 /* ARGSUSED */
4347 static int
4348 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr,
4349     const void *not_used)
4350 {
4351 	rootnex_window_t *window;
4352 	uint64_t start_addr;
4353 	uint64_t fault_addr;
4354 	ddi_dma_impl_t *hp;
4355 	rootnex_dma_t *dma;
4356 	uint64_t end_addr;
4357 	size_t csize;
4358 	int i;
4359 	int j;
4360 
4361 
4362 	/* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */
4363 	hp = (ddi_dma_impl_t *)handle;
4364 	ASSERT(hp);
4365 
4366 	dma = (rootnex_dma_t *)hp->dmai_private;
4367 
4368 	/* Get the address that we need to search for */
4369 	fault_addr = *(uint64_t *)addr;
4370 
4371 	/*
4372 	 * if we don't have any windows, we can just walk through all the
4373 	 * cookies.
4374 	 */
4375 	if (dma->dp_window == NULL) {
4376 		/* for each cookie */
4377 		for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) {
4378 			/*
4379 			 * if the faulted address is within the physical address
4380 			 * range of the cookie, return DDI_FM_NONFATAL.
4381 			 */
4382 			if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) &&
4383 			    (fault_addr <= (dma->dp_cookies[i].dmac_laddress +
4384 			    dma->dp_cookies[i].dmac_size))) {
4385 				return (DDI_FM_NONFATAL);
4386 			}
4387 		}
4388 
4389 		/* fault_addr not within this DMA handle */
4390 		return (DDI_FM_UNKNOWN);
4391 	}
4392 
4393 	/* we have mutiple windows, walk through each window */
4394 	for (i = 0; i < hp->dmai_nwin; i++) {
4395 		window = &dma->dp_window[i];
4396 
4397 		/* Go through all the cookies in the window */
4398 		for (j = 0; j < window->wd_cookie_cnt; j++) {
4399 
4400 			start_addr = window->wd_first_cookie[j].dmac_laddress;
4401 			csize = window->wd_first_cookie[j].dmac_size;
4402 
4403 			/*
4404 			 * if we are trimming the first cookie in the window,
4405 			 * and this is the first cookie, adjust the start
4406 			 * address and size of the cookie to account for the
4407 			 * trim.
4408 			 */
4409 			if (window->wd_trim.tr_trim_first && (j == 0)) {
4410 				start_addr = window->wd_trim.tr_first_paddr;
4411 				csize = window->wd_trim.tr_first_size;
4412 			}
4413 
4414 			/*
4415 			 * if we are trimming the last cookie in the window,
4416 			 * and this is the last cookie, adjust the start
4417 			 * address and size of the cookie to account for the
4418 			 * trim.
4419 			 */
4420 			if (window->wd_trim.tr_trim_last &&
4421 			    (j == (window->wd_cookie_cnt - 1))) {
4422 				start_addr = window->wd_trim.tr_last_paddr;
4423 				csize = window->wd_trim.tr_last_size;
4424 			}
4425 
4426 			end_addr = start_addr + csize;
4427 
4428 			/*
4429 			 * if the faulted address is within the physical address
4430 			 * range of the cookie, return DDI_FM_NONFATAL.
4431 			 */
4432 			if ((fault_addr >= start_addr) &&
4433 			    (fault_addr <= end_addr)) {
4434 				return (DDI_FM_NONFATAL);
4435 			}
4436 		}
4437 	}
4438 
4439 	/* fault_addr not within this DMA handle */
4440 	return (DDI_FM_UNKNOWN);
4441 }
4442