1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * x86 root nexus driver 31 */ 32 33 #include <sys/sysmacros.h> 34 #include <sys/conf.h> 35 #include <sys/autoconf.h> 36 #include <sys/sysmacros.h> 37 #include <sys/debug.h> 38 #include <sys/psw.h> 39 #include <sys/ddidmareq.h> 40 #include <sys/promif.h> 41 #include <sys/devops.h> 42 #include <sys/kmem.h> 43 #include <sys/cmn_err.h> 44 #include <vm/seg.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_dev.h> 47 #include <sys/vmem.h> 48 #include <sys/mman.h> 49 #include <vm/hat.h> 50 #include <vm/as.h> 51 #include <vm/page.h> 52 #include <sys/avintr.h> 53 #include <sys/errno.h> 54 #include <sys/modctl.h> 55 #include <sys/ddi_impldefs.h> 56 #include <sys/sunddi.h> 57 #include <sys/sunndi.h> 58 #include <sys/psm.h> 59 #include <sys/ontrap.h> 60 #include <sys/atomic.h> 61 #include <sys/sdt.h> 62 #include <sys/rootnex.h> 63 #include <vm/hat_i86.h> 64 65 66 /* 67 * enable/disable extra checking of function parameters. Useful for debugging 68 * drivers. 69 */ 70 #ifdef DEBUG 71 int rootnex_alloc_check_parms = 1; 72 int rootnex_bind_check_parms = 1; 73 int rootnex_bind_check_inuse = 1; 74 int rootnex_unbind_verify_buffer = 0; 75 int rootnex_sync_check_parms = 1; 76 #else 77 int rootnex_alloc_check_parms = 0; 78 int rootnex_bind_check_parms = 0; 79 int rootnex_bind_check_inuse = 0; 80 int rootnex_unbind_verify_buffer = 0; 81 int rootnex_sync_check_parms = 0; 82 #endif 83 84 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 85 int rootnex_bind_fail = 1; 86 int rootnex_bind_warn = 1; 87 uint8_t *rootnex_warn_list; 88 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 89 #define ROOTNEX_BIND_WARNING (0x1 << 0) 90 91 /* 92 * revert back to old broken behavior of always sync'ing entire copy buffer. 93 * This is useful if be have a buggy driver which doesn't correctly pass in 94 * the offset and size into ddi_dma_sync(). 95 */ 96 int rootnex_sync_ignore_params = 0; 97 98 /* 99 * maximum size that we will allow for a copy buffer. Can be patched on the 100 * fly 101 */ 102 size_t rootnex_max_copybuf_size = 0x100000; 103 104 /* 105 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 106 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 107 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 108 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 109 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 110 * (< 8K). We will still need to allocate the copy buffer during bind though 111 * (if we need one). These can only be modified in /etc/system before rootnex 112 * attach. 113 */ 114 #if defined(__amd64) 115 int rootnex_prealloc_cookies = 65; 116 int rootnex_prealloc_windows = 4; 117 int rootnex_prealloc_copybuf = 2; 118 #else 119 int rootnex_prealloc_cookies = 33; 120 int rootnex_prealloc_windows = 4; 121 int rootnex_prealloc_copybuf = 2; 122 #endif 123 124 /* driver global state */ 125 static rootnex_state_t *rootnex_state; 126 127 /* shortcut to rootnex counters */ 128 static uint64_t *rootnex_cnt; 129 130 /* 131 * XXX - does x86 even need these or are they left over from the SPARC days? 132 */ 133 /* statically defined integer/boolean properties for the root node */ 134 static rootnex_intprop_t rootnex_intprp[] = { 135 { "PAGESIZE", PAGESIZE }, 136 { "MMU_PAGESIZE", MMU_PAGESIZE }, 137 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 138 { DDI_RELATIVE_ADDRESSING, 1 }, 139 }; 140 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 141 142 143 static struct cb_ops rootnex_cb_ops = { 144 nodev, /* open */ 145 nodev, /* close */ 146 nodev, /* strategy */ 147 nodev, /* print */ 148 nodev, /* dump */ 149 nodev, /* read */ 150 nodev, /* write */ 151 nodev, /* ioctl */ 152 nodev, /* devmap */ 153 nodev, /* mmap */ 154 nodev, /* segmap */ 155 nochpoll, /* chpoll */ 156 ddi_prop_op, /* cb_prop_op */ 157 NULL, /* struct streamtab */ 158 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 159 CB_REV, /* Rev */ 160 nodev, /* cb_aread */ 161 nodev /* cb_awrite */ 162 }; 163 164 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 165 off_t offset, off_t len, caddr_t *vaddrp); 166 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 167 struct hat *hat, struct seg *seg, caddr_t addr, 168 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 169 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 170 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 171 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 172 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 173 ddi_dma_handle_t *handlep); 174 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 175 ddi_dma_handle_t handle); 176 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 177 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 178 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 179 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 180 ddi_dma_handle_t handle); 181 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 182 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 183 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 184 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 185 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 186 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 187 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 188 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 189 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 190 ddi_ctl_enum_t ctlop, void *arg, void *result); 191 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 192 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 193 194 195 static struct bus_ops rootnex_bus_ops = { 196 BUSO_REV, 197 rootnex_map, 198 NULL, 199 NULL, 200 NULL, 201 rootnex_map_fault, 202 rootnex_dma_map, 203 rootnex_dma_allochdl, 204 rootnex_dma_freehdl, 205 rootnex_dma_bindhdl, 206 rootnex_dma_unbindhdl, 207 rootnex_dma_sync, 208 rootnex_dma_win, 209 rootnex_dma_mctl, 210 rootnex_ctlops, 211 ddi_bus_prop_op, 212 i_ddi_rootnex_get_eventcookie, 213 i_ddi_rootnex_add_eventcall, 214 i_ddi_rootnex_remove_eventcall, 215 i_ddi_rootnex_post_event, 216 0, /* bus_intr_ctl */ 217 0, /* bus_config */ 218 0, /* bus_unconfig */ 219 NULL, /* bus_fm_init */ 220 NULL, /* bus_fm_fini */ 221 NULL, /* bus_fm_access_enter */ 222 NULL, /* bus_fm_access_exit */ 223 NULL, /* bus_powr */ 224 rootnex_intr_ops /* bus_intr_op */ 225 }; 226 227 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 228 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 229 230 static struct dev_ops rootnex_ops = { 231 DEVO_REV, 232 0, 233 ddi_no_info, 234 nulldev, 235 nulldev, 236 rootnex_attach, 237 rootnex_detach, 238 nulldev, 239 &rootnex_cb_ops, 240 &rootnex_bus_ops 241 }; 242 243 static struct modldrv rootnex_modldrv = { 244 &mod_driverops, 245 "i86pc root nexus %I%", 246 &rootnex_ops 247 }; 248 249 static struct modlinkage rootnex_modlinkage = { 250 MODREV_1, 251 (void *)&rootnex_modldrv, 252 NULL 253 }; 254 255 256 /* 257 * extern hacks 258 */ 259 extern struct seg_ops segdev_ops; 260 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 261 #ifdef DDI_MAP_DEBUG 262 extern int ddi_map_debug_flag; 263 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 264 #endif 265 #define ptob64(x) (((uint64_t)(x)) << MMU_PAGESHIFT) 266 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 267 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 268 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 269 psm_intr_op_t, int *); 270 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 271 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 272 /* 273 * Use device arena to use for device control register mappings. 274 * Various kernel memory walkers (debugger, dtrace) need to know 275 * to avoid this address range to prevent undesired device activity. 276 */ 277 extern void *device_arena_alloc(size_t size, int vm_flag); 278 extern void device_arena_free(void * vaddr, size_t size); 279 280 281 /* 282 * Internal functions 283 */ 284 static int rootnex_dma_init(); 285 static void rootnex_add_props(dev_info_t *); 286 static int rootnex_ctl_reportdev(dev_info_t *dip); 287 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 288 static int rootnex_ctlops_poke(peekpoke_ctlops_t *in_args); 289 static int rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result); 290 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 291 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 292 static int rootnex_map_handle(ddi_map_req_t *mp); 293 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 294 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 295 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 296 ddi_dma_attr_t *attr); 297 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 298 rootnex_sglinfo_t *sglinfo); 299 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 300 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 301 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 302 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 303 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 304 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 305 ddi_dma_attr_t *attr, int kmflag); 306 static void rootnex_teardown_windows(rootnex_dma_t *dma); 307 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 308 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 309 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 310 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 311 size_t *copybuf_used, page_t **cur_pp); 312 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 313 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 314 ddi_dma_attr_t *attr, off_t cur_offset); 315 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 316 rootnex_dma_t *dma, rootnex_window_t **windowp, 317 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 318 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 319 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 320 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 321 off_t offset, size_t size, uint_t cache_flags); 322 static int rootnex_verify_buffer(rootnex_dma_t *dma); 323 324 325 /* 326 * _init() 327 * 328 */ 329 int 330 _init(void) 331 { 332 333 rootnex_state = NULL; 334 return (mod_install(&rootnex_modlinkage)); 335 } 336 337 338 /* 339 * _info() 340 * 341 */ 342 int 343 _info(struct modinfo *modinfop) 344 { 345 return (mod_info(&rootnex_modlinkage, modinfop)); 346 } 347 348 349 /* 350 * _fini() 351 * 352 */ 353 int 354 _fini(void) 355 { 356 return (EBUSY); 357 } 358 359 360 /* 361 * rootnex_attach() 362 * 363 */ 364 static int 365 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 366 { 367 int e; 368 369 370 switch (cmd) { 371 case DDI_ATTACH: 372 break; 373 case DDI_RESUME: 374 return (DDI_SUCCESS); 375 default: 376 return (DDI_FAILURE); 377 } 378 379 /* 380 * We should only have one instance of rootnex. Save it away since we 381 * don't have an easy way to get it back later. 382 */ 383 ASSERT(rootnex_state == NULL); 384 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 385 386 rootnex_state->r_dip = dip; 387 rootnex_state->r_reserved_msg_printed = B_FALSE; 388 rootnex_cnt = &rootnex_state->r_counters[0]; 389 390 mutex_init(&rootnex_state->r_peekpoke_mutex, NULL, MUTEX_SPIN, 391 (void *)ipltospl(15)); 392 393 /* initialize DMA related state */ 394 e = rootnex_dma_init(); 395 if (e != DDI_SUCCESS) { 396 mutex_destroy(&rootnex_state->r_peekpoke_mutex); 397 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 398 return (DDI_FAILURE); 399 } 400 401 /* Add static root node properties */ 402 rootnex_add_props(dip); 403 404 /* since we can't call ddi_report_dev() */ 405 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 406 407 /* Initialize rootnex event handle */ 408 i_ddi_rootnex_init_events(dip); 409 410 return (DDI_SUCCESS); 411 } 412 413 414 /* 415 * rootnex_detach() 416 * 417 */ 418 /*ARGSUSED*/ 419 static int 420 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 421 { 422 switch (cmd) { 423 case DDI_SUSPEND: 424 break; 425 default: 426 return (DDI_FAILURE); 427 } 428 429 return (DDI_SUCCESS); 430 } 431 432 433 /* 434 * rootnex_dma_init() 435 * 436 */ 437 /*ARGSUSED*/ 438 static int 439 rootnex_dma_init() 440 { 441 size_t bufsize; 442 443 444 /* 445 * size of our cookie/window/copybuf state needed in dma bind that we 446 * pre-alloc in dma_alloc_handle 447 */ 448 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 449 rootnex_state->r_prealloc_size = 450 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 451 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 452 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 453 454 /* 455 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 456 * allocate 16 extra bytes for struct pointer alignment 457 * (p->dmai_private & dma->dp_prealloc_buffer) 458 */ 459 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 460 rootnex_state->r_prealloc_size + 0x10; 461 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 462 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 463 if (rootnex_state->r_dmahdl_cache == NULL) { 464 return (DDI_FAILURE); 465 } 466 467 /* 468 * allocate array to track which major numbers we have printed warnings 469 * for. 470 */ 471 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 472 KM_SLEEP); 473 474 return (DDI_SUCCESS); 475 } 476 477 478 /* 479 * rootnex_add_props() 480 * 481 */ 482 static void 483 rootnex_add_props(dev_info_t *dip) 484 { 485 rootnex_intprop_t *rpp; 486 int i; 487 488 /* Add static integer/boolean properties to the root node */ 489 rpp = rootnex_intprp; 490 for (i = 0; i < NROOT_INTPROPS; i++) { 491 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 492 rpp[i].prop_name, rpp[i].prop_value); 493 } 494 } 495 496 497 498 /* 499 * ************************* 500 * ctlops related routines 501 * ************************* 502 */ 503 504 /* 505 * rootnex_ctlops() 506 * 507 */ 508 /*ARGSUSED*/ 509 static int 510 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 511 void *arg, void *result) 512 { 513 int n, *ptr; 514 struct ddi_parent_private_data *pdp; 515 516 switch (ctlop) { 517 case DDI_CTLOPS_DMAPMAPC: 518 /* 519 * Return 'partial' to indicate that dma mapping 520 * has to be done in the main MMU. 521 */ 522 return (DDI_DMA_PARTIAL); 523 524 case DDI_CTLOPS_BTOP: 525 /* 526 * Convert byte count input to physical page units. 527 * (byte counts that are not a page-size multiple 528 * are rounded down) 529 */ 530 *(ulong_t *)result = btop(*(ulong_t *)arg); 531 return (DDI_SUCCESS); 532 533 case DDI_CTLOPS_PTOB: 534 /* 535 * Convert size in physical pages to bytes 536 */ 537 *(ulong_t *)result = ptob(*(ulong_t *)arg); 538 return (DDI_SUCCESS); 539 540 case DDI_CTLOPS_BTOPR: 541 /* 542 * Convert byte count input to physical page units 543 * (byte counts that are not a page-size multiple 544 * are rounded up) 545 */ 546 *(ulong_t *)result = btopr(*(ulong_t *)arg); 547 return (DDI_SUCCESS); 548 549 case DDI_CTLOPS_POKE: 550 return (rootnex_ctlops_poke((peekpoke_ctlops_t *)arg)); 551 552 case DDI_CTLOPS_PEEK: 553 return (rootnex_ctlops_peek((peekpoke_ctlops_t *)arg, result)); 554 555 case DDI_CTLOPS_INITCHILD: 556 return (impl_ddi_sunbus_initchild(arg)); 557 558 case DDI_CTLOPS_UNINITCHILD: 559 impl_ddi_sunbus_removechild(arg); 560 return (DDI_SUCCESS); 561 562 case DDI_CTLOPS_REPORTDEV: 563 return (rootnex_ctl_reportdev(rdip)); 564 565 case DDI_CTLOPS_IOMIN: 566 /* 567 * Nothing to do here but reflect back.. 568 */ 569 return (DDI_SUCCESS); 570 571 case DDI_CTLOPS_REGSIZE: 572 case DDI_CTLOPS_NREGS: 573 break; 574 575 case DDI_CTLOPS_SIDDEV: 576 if (ndi_dev_is_prom_node(rdip)) 577 return (DDI_SUCCESS); 578 if (ndi_dev_is_persistent_node(rdip)) 579 return (DDI_SUCCESS); 580 return (DDI_FAILURE); 581 582 case DDI_CTLOPS_POWER: 583 return ((*pm_platform_power)((power_req_t *)arg)); 584 585 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 586 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 587 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 588 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 589 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 590 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 591 if (!rootnex_state->r_reserved_msg_printed) { 592 rootnex_state->r_reserved_msg_printed = B_TRUE; 593 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 594 "1 or more reserved/obsolete operations."); 595 } 596 return (DDI_FAILURE); 597 598 default: 599 return (DDI_FAILURE); 600 } 601 /* 602 * The rest are for "hardware" properties 603 */ 604 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 605 return (DDI_FAILURE); 606 607 if (ctlop == DDI_CTLOPS_NREGS) { 608 ptr = (int *)result; 609 *ptr = pdp->par_nreg; 610 } else { 611 off_t *size = (off_t *)result; 612 613 ptr = (int *)arg; 614 n = *ptr; 615 if (n >= pdp->par_nreg) { 616 return (DDI_FAILURE); 617 } 618 *size = (off_t)pdp->par_reg[n].regspec_size; 619 } 620 return (DDI_SUCCESS); 621 } 622 623 624 /* 625 * rootnex_ctl_reportdev() 626 * 627 */ 628 static int 629 rootnex_ctl_reportdev(dev_info_t *dev) 630 { 631 int i, n, len, f_len = 0; 632 char *buf; 633 634 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 635 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 636 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 637 len = strlen(buf); 638 639 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 640 641 struct regspec *rp = sparc_pd_getreg(dev, i); 642 643 if (i == 0) 644 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 645 ": "); 646 else 647 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 648 " and "); 649 len = strlen(buf); 650 651 switch (rp->regspec_bustype) { 652 653 case BTEISA: 654 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 655 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 656 break; 657 658 case BTISA: 659 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 660 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 661 break; 662 663 default: 664 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 665 "space %x offset %x", 666 rp->regspec_bustype, rp->regspec_addr); 667 break; 668 } 669 len = strlen(buf); 670 } 671 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 672 int pri; 673 674 if (i != 0) { 675 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 676 ","); 677 len = strlen(buf); 678 } 679 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 680 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 681 " sparc ipl %d", pri); 682 len = strlen(buf); 683 } 684 #ifdef DEBUG 685 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 686 cmn_err(CE_NOTE, "next message is truncated: " 687 "printed length 1024, real length %d", f_len); 688 } 689 #endif /* DEBUG */ 690 cmn_err(CE_CONT, "?%s\n", buf); 691 kmem_free(buf, REPORTDEV_BUFSIZE); 692 return (DDI_SUCCESS); 693 } 694 695 696 /* 697 * rootnex_ctlops_poke() 698 * 699 */ 700 static int 701 rootnex_ctlops_poke(peekpoke_ctlops_t *in_args) 702 { 703 int err = DDI_SUCCESS; 704 on_trap_data_t otd; 705 706 /* Cautious access not supported. */ 707 if (in_args->handle != NULL) 708 return (DDI_FAILURE); 709 710 mutex_enter(&rootnex_state->r_peekpoke_mutex); 711 712 /* Set up protected environment. */ 713 if (!on_trap(&otd, OT_DATA_ACCESS)) { 714 switch (in_args->size) { 715 case sizeof (uint8_t): 716 *(uint8_t *)in_args->dev_addr = *(uint8_t *) 717 in_args->host_addr; 718 break; 719 720 case sizeof (uint16_t): 721 *(uint16_t *)in_args->dev_addr = 722 *(uint16_t *)in_args->host_addr; 723 break; 724 725 case sizeof (uint32_t): 726 *(uint32_t *)in_args->dev_addr = 727 *(uint32_t *)in_args->host_addr; 728 break; 729 730 case sizeof (uint64_t): 731 *(uint64_t *)in_args->dev_addr = 732 *(uint64_t *)in_args->host_addr; 733 break; 734 735 default: 736 err = DDI_FAILURE; 737 break; 738 } 739 } else 740 err = DDI_FAILURE; 741 742 /* Take down protected environment. */ 743 no_trap(); 744 mutex_exit(&rootnex_state->r_peekpoke_mutex); 745 746 return (err); 747 } 748 749 750 /* 751 * rootnex_ctlops_peek() 752 * 753 */ 754 static int 755 rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result) 756 { 757 int err = DDI_SUCCESS; 758 on_trap_data_t otd; 759 760 /* Cautious access not supported. */ 761 if (in_args->handle != NULL) 762 return (DDI_FAILURE); 763 764 mutex_enter(&rootnex_state->r_peekpoke_mutex); 765 766 if (!on_trap(&otd, OT_DATA_ACCESS)) { 767 switch (in_args->size) { 768 case sizeof (uint8_t): 769 *(uint8_t *)in_args->host_addr = 770 *(uint8_t *)in_args->dev_addr; 771 break; 772 773 case sizeof (uint16_t): 774 *(uint16_t *)in_args->host_addr = 775 *(uint16_t *)in_args->dev_addr; 776 break; 777 778 case sizeof (uint32_t): 779 *(uint32_t *)in_args->host_addr = 780 *(uint32_t *)in_args->dev_addr; 781 break; 782 783 case sizeof (uint64_t): 784 *(uint64_t *)in_args->host_addr = 785 *(uint64_t *)in_args->dev_addr; 786 break; 787 788 default: 789 err = DDI_FAILURE; 790 break; 791 } 792 result = (void *)in_args->host_addr; 793 } else 794 err = DDI_FAILURE; 795 796 no_trap(); 797 mutex_exit(&rootnex_state->r_peekpoke_mutex); 798 799 return (err); 800 } 801 802 803 804 /* 805 * ****************** 806 * map related code 807 * ****************** 808 */ 809 810 /* 811 * rootnex_map() 812 * 813 */ 814 static int 815 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 816 off_t len, caddr_t *vaddrp) 817 { 818 struct regspec *rp, tmp_reg; 819 ddi_map_req_t mr = *mp; /* Get private copy of request */ 820 int error; 821 822 mp = &mr; 823 824 switch (mp->map_op) { 825 case DDI_MO_MAP_LOCKED: 826 case DDI_MO_UNMAP: 827 case DDI_MO_MAP_HANDLE: 828 break; 829 default: 830 #ifdef DDI_MAP_DEBUG 831 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 832 mp->map_op); 833 #endif /* DDI_MAP_DEBUG */ 834 return (DDI_ME_UNIMPLEMENTED); 835 } 836 837 if (mp->map_flags & DDI_MF_USER_MAPPING) { 838 #ifdef DDI_MAP_DEBUG 839 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 840 #endif /* DDI_MAP_DEBUG */ 841 return (DDI_ME_UNIMPLEMENTED); 842 } 843 844 /* 845 * First, if given an rnumber, convert it to a regspec... 846 * (Presumably, this is on behalf of a child of the root node?) 847 */ 848 849 if (mp->map_type == DDI_MT_RNUMBER) { 850 851 int rnumber = mp->map_obj.rnumber; 852 #ifdef DDI_MAP_DEBUG 853 static char *out_of_range = 854 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 855 #endif /* DDI_MAP_DEBUG */ 856 857 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 858 if (rp == NULL) { 859 #ifdef DDI_MAP_DEBUG 860 cmn_err(CE_WARN, out_of_range, rnumber, 861 ddi_get_name(rdip)); 862 #endif /* DDI_MAP_DEBUG */ 863 return (DDI_ME_RNUMBER_RANGE); 864 } 865 866 /* 867 * Convert the given ddi_map_req_t from rnumber to regspec... 868 */ 869 870 mp->map_type = DDI_MT_REGSPEC; 871 mp->map_obj.rp = rp; 872 } 873 874 /* 875 * Adjust offset and length correspnding to called values... 876 * XXX: A non-zero length means override the one in the regspec 877 * XXX: (regardless of what's in the parent's range?) 878 */ 879 880 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 881 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 882 883 #ifdef DDI_MAP_DEBUG 884 cmn_err(CE_CONT, 885 "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d>" 886 " offset %d len %d handle 0x%x\n", 887 ddi_get_name(dip), ddi_get_name(rdip), 888 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 889 offset, len, mp->map_handlep); 890 #endif /* DDI_MAP_DEBUG */ 891 892 /* 893 * I/O or memory mapping: 894 * 895 * <bustype=0, addr=x, len=x>: memory 896 * <bustype=1, addr=x, len=x>: i/o 897 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 898 */ 899 900 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 901 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 902 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 903 ddi_get_name(rdip), rp->regspec_bustype, 904 rp->regspec_addr, rp->regspec_size); 905 return (DDI_ME_INVAL); 906 } 907 908 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 909 /* 910 * compatibility i/o mapping 911 */ 912 rp->regspec_bustype += (uint_t)offset; 913 } else { 914 /* 915 * Normal memory or i/o mapping 916 */ 917 rp->regspec_addr += (uint_t)offset; 918 } 919 920 if (len != 0) 921 rp->regspec_size = (uint_t)len; 922 923 #ifdef DDI_MAP_DEBUG 924 cmn_err(CE_CONT, 925 " <%s,%s> <0x%x, 0x%x, 0x%d>" 926 " offset %d len %d handle 0x%x\n", 927 ddi_get_name(dip), ddi_get_name(rdip), 928 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 929 offset, len, mp->map_handlep); 930 #endif /* DDI_MAP_DEBUG */ 931 932 /* 933 * Apply any parent ranges at this level, if applicable. 934 * (This is where nexus specific regspec translation takes place. 935 * Use of this function is implicit agreement that translation is 936 * provided via ddi_apply_range.) 937 */ 938 939 #ifdef DDI_MAP_DEBUG 940 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 941 ddi_get_name(dip), ddi_get_name(rdip)); 942 #endif /* DDI_MAP_DEBUG */ 943 944 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 945 return (error); 946 947 switch (mp->map_op) { 948 case DDI_MO_MAP_LOCKED: 949 950 /* 951 * Set up the locked down kernel mapping to the regspec... 952 */ 953 954 return (rootnex_map_regspec(mp, vaddrp)); 955 956 case DDI_MO_UNMAP: 957 958 /* 959 * Release mapping... 960 */ 961 962 return (rootnex_unmap_regspec(mp, vaddrp)); 963 964 case DDI_MO_MAP_HANDLE: 965 966 return (rootnex_map_handle(mp)); 967 968 default: 969 return (DDI_ME_UNIMPLEMENTED); 970 } 971 } 972 973 974 /* 975 * rootnex_map_fault() 976 * 977 * fault in mappings for requestors 978 */ 979 /*ARGSUSED*/ 980 static int 981 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 982 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 983 uint_t lock) 984 { 985 986 #ifdef DDI_MAP_DEBUG 987 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 988 ddi_map_debug(" Seg <%s>\n", 989 seg->s_ops == &segdev_ops ? "segdev" : 990 seg == &kvseg ? "segkmem" : "NONE!"); 991 #endif /* DDI_MAP_DEBUG */ 992 993 /* 994 * This is all terribly broken, but it is a start 995 * 996 * XXX Note that this test means that segdev_ops 997 * must be exported from seg_dev.c. 998 * XXX What about devices with their own segment drivers? 999 */ 1000 if (seg->s_ops == &segdev_ops) { 1001 struct segdev_data *sdp = 1002 (struct segdev_data *)seg->s_data; 1003 1004 if (hat == NULL) { 1005 /* 1006 * This is one plausible interpretation of 1007 * a null hat i.e. use the first hat on the 1008 * address space hat list which by convention is 1009 * the hat of the system MMU. At alternative 1010 * would be to panic .. this might well be better .. 1011 */ 1012 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1013 hat = seg->s_as->a_hat; 1014 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1015 } 1016 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1017 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1018 } else if (seg == &kvseg && dp == NULL) { 1019 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1020 HAT_LOAD_LOCK); 1021 } else 1022 return (DDI_FAILURE); 1023 return (DDI_SUCCESS); 1024 } 1025 1026 1027 /* 1028 * rootnex_map_regspec() 1029 * we don't support mapping of I/O cards above 4Gb 1030 */ 1031 static int 1032 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1033 { 1034 ulong_t base; 1035 void *cvaddr; 1036 uint_t npages, pgoffset; 1037 struct regspec *rp; 1038 ddi_acc_hdl_t *hp; 1039 ddi_acc_impl_t *ap; 1040 uint_t hat_acc_flags; 1041 1042 rp = mp->map_obj.rp; 1043 hp = mp->map_handlep; 1044 1045 #ifdef DDI_MAP_DEBUG 1046 ddi_map_debug( 1047 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1048 rp->regspec_bustype, rp->regspec_addr, 1049 rp->regspec_size, mp->map_handlep); 1050 #endif /* DDI_MAP_DEBUG */ 1051 1052 /* 1053 * I/O or memory mapping 1054 * 1055 * <bustype=0, addr=x, len=x>: memory 1056 * <bustype=1, addr=x, len=x>: i/o 1057 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1058 */ 1059 1060 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1061 cmn_err(CE_WARN, "rootnex: invalid register spec" 1062 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1063 rp->regspec_addr, rp->regspec_size); 1064 return (DDI_FAILURE); 1065 } 1066 1067 if (rp->regspec_bustype != 0) { 1068 /* 1069 * I/O space - needs a handle. 1070 */ 1071 if (hp == NULL) { 1072 return (DDI_FAILURE); 1073 } 1074 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1075 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1076 impl_acc_hdl_init(hp); 1077 1078 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1079 #ifdef DDI_MAP_DEBUG 1080 ddi_map_debug("rootnex_map_regspec: mmap() \ 1081 to I/O space is not supported.\n"); 1082 #endif /* DDI_MAP_DEBUG */ 1083 return (DDI_ME_INVAL); 1084 } else { 1085 /* 1086 * 1275-compliant vs. compatibility i/o mapping 1087 */ 1088 *vaddrp = 1089 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1090 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1091 ((caddr_t)(uintptr_t)rp->regspec_addr); 1092 } 1093 1094 #ifdef DDI_MAP_DEBUG 1095 ddi_map_debug( 1096 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1097 rp->regspec_size, *vaddrp); 1098 #endif /* DDI_MAP_DEBUG */ 1099 return (DDI_SUCCESS); 1100 } 1101 1102 /* 1103 * Memory space 1104 */ 1105 1106 if (hp != NULL) { 1107 /* 1108 * hat layer ignores 1109 * hp->ah_acc.devacc_attr_endian_flags. 1110 */ 1111 switch (hp->ah_acc.devacc_attr_dataorder) { 1112 case DDI_STRICTORDER_ACC: 1113 hat_acc_flags = HAT_STRICTORDER; 1114 break; 1115 case DDI_UNORDERED_OK_ACC: 1116 hat_acc_flags = HAT_UNORDERED_OK; 1117 break; 1118 case DDI_MERGING_OK_ACC: 1119 hat_acc_flags = HAT_MERGING_OK; 1120 break; 1121 case DDI_LOADCACHING_OK_ACC: 1122 hat_acc_flags = HAT_LOADCACHING_OK; 1123 break; 1124 case DDI_STORECACHING_OK_ACC: 1125 hat_acc_flags = HAT_STORECACHING_OK; 1126 break; 1127 } 1128 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1129 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1130 impl_acc_hdl_init(hp); 1131 hp->ah_hat_flags = hat_acc_flags; 1132 } else { 1133 hat_acc_flags = HAT_STRICTORDER; 1134 } 1135 1136 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1137 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1138 1139 if (rp->regspec_size == 0) { 1140 #ifdef DDI_MAP_DEBUG 1141 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1142 #endif /* DDI_MAP_DEBUG */ 1143 return (DDI_ME_INVAL); 1144 } 1145 1146 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1147 *vaddrp = (caddr_t)mmu_btop(base); 1148 } else { 1149 npages = mmu_btopr(rp->regspec_size + pgoffset); 1150 1151 #ifdef DDI_MAP_DEBUG 1152 ddi_map_debug("rootnex_map_regspec: Mapping %d pages \ 1153 physical %x ", 1154 npages, base); 1155 #endif /* DDI_MAP_DEBUG */ 1156 1157 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1158 if (cvaddr == NULL) 1159 return (DDI_ME_NORESOURCES); 1160 1161 /* 1162 * Now map in the pages we've allocated... 1163 */ 1164 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base), 1165 mp->map_prot | hat_acc_flags, HAT_LOAD_LOCK); 1166 *vaddrp = (caddr_t)cvaddr + pgoffset; 1167 } 1168 1169 #ifdef DDI_MAP_DEBUG 1170 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1171 #endif /* DDI_MAP_DEBUG */ 1172 return (DDI_SUCCESS); 1173 } 1174 1175 1176 /* 1177 * rootnex_unmap_regspec() 1178 * 1179 */ 1180 static int 1181 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1182 { 1183 caddr_t addr = (caddr_t)*vaddrp; 1184 uint_t npages, pgoffset; 1185 struct regspec *rp; 1186 1187 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1188 return (0); 1189 1190 rp = mp->map_obj.rp; 1191 1192 if (rp->regspec_size == 0) { 1193 #ifdef DDI_MAP_DEBUG 1194 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1195 #endif /* DDI_MAP_DEBUG */ 1196 return (DDI_ME_INVAL); 1197 } 1198 1199 /* 1200 * I/O or memory mapping: 1201 * 1202 * <bustype=0, addr=x, len=x>: memory 1203 * <bustype=1, addr=x, len=x>: i/o 1204 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1205 */ 1206 if (rp->regspec_bustype != 0) { 1207 /* 1208 * This is I/O space, which requires no particular 1209 * processing on unmap since it isn't mapped in the 1210 * first place. 1211 */ 1212 return (DDI_SUCCESS); 1213 } 1214 1215 /* 1216 * Memory space 1217 */ 1218 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1219 npages = mmu_btopr(rp->regspec_size + pgoffset); 1220 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1221 device_arena_free(addr - pgoffset, ptob(npages)); 1222 1223 /* 1224 * Destroy the pointer - the mapping has logically gone 1225 */ 1226 *vaddrp = NULL; 1227 1228 return (DDI_SUCCESS); 1229 } 1230 1231 1232 /* 1233 * rootnex_map_handle() 1234 * 1235 */ 1236 static int 1237 rootnex_map_handle(ddi_map_req_t *mp) 1238 { 1239 ddi_acc_hdl_t *hp; 1240 ulong_t base; 1241 uint_t pgoffset; 1242 struct regspec *rp; 1243 1244 rp = mp->map_obj.rp; 1245 1246 #ifdef DDI_MAP_DEBUG 1247 ddi_map_debug( 1248 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1249 rp->regspec_bustype, rp->regspec_addr, 1250 rp->regspec_size, mp->map_handlep); 1251 #endif /* DDI_MAP_DEBUG */ 1252 1253 /* 1254 * I/O or memory mapping: 1255 * 1256 * <bustype=0, addr=x, len=x>: memory 1257 * <bustype=1, addr=x, len=x>: i/o 1258 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1259 */ 1260 if (rp->regspec_bustype != 0) { 1261 /* 1262 * This refers to I/O space, and we don't support "mapping" 1263 * I/O space to a user. 1264 */ 1265 return (DDI_FAILURE); 1266 } 1267 1268 /* 1269 * Set up the hat_flags for the mapping. 1270 */ 1271 hp = mp->map_handlep; 1272 1273 switch (hp->ah_acc.devacc_attr_endian_flags) { 1274 case DDI_NEVERSWAP_ACC: 1275 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1276 break; 1277 case DDI_STRUCTURE_LE_ACC: 1278 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1279 break; 1280 case DDI_STRUCTURE_BE_ACC: 1281 return (DDI_FAILURE); 1282 default: 1283 return (DDI_REGS_ACC_CONFLICT); 1284 } 1285 1286 switch (hp->ah_acc.devacc_attr_dataorder) { 1287 case DDI_STRICTORDER_ACC: 1288 break; 1289 case DDI_UNORDERED_OK_ACC: 1290 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1291 break; 1292 case DDI_MERGING_OK_ACC: 1293 hp->ah_hat_flags |= HAT_MERGING_OK; 1294 break; 1295 case DDI_LOADCACHING_OK_ACC: 1296 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1297 break; 1298 case DDI_STORECACHING_OK_ACC: 1299 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1300 break; 1301 default: 1302 return (DDI_FAILURE); 1303 } 1304 1305 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1306 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1307 1308 if (rp->regspec_size == 0) 1309 return (DDI_ME_INVAL); 1310 1311 hp->ah_pfn = mmu_btop(base); 1312 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1313 1314 return (DDI_SUCCESS); 1315 } 1316 1317 1318 1319 /* 1320 * ************************ 1321 * interrupt related code 1322 * ************************ 1323 */ 1324 1325 /* 1326 * rootnex_intr_ops() 1327 * bus_intr_op() function for interrupt support 1328 */ 1329 /* ARGSUSED */ 1330 static int 1331 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1332 ddi_intr_handle_impl_t *hdlp, void *result) 1333 { 1334 struct intrspec *ispec; 1335 struct ddi_parent_private_data *pdp; 1336 1337 DDI_INTR_NEXDBG((CE_CONT, 1338 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1339 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1340 1341 /* Process the interrupt operation */ 1342 switch (intr_op) { 1343 case DDI_INTROP_GETCAP: 1344 /* First check with pcplusmp */ 1345 if (psm_intr_ops == NULL) 1346 return (DDI_FAILURE); 1347 1348 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1349 *(int *)result = 0; 1350 return (DDI_FAILURE); 1351 } 1352 break; 1353 case DDI_INTROP_SETCAP: 1354 if (psm_intr_ops == NULL) 1355 return (DDI_FAILURE); 1356 1357 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1358 return (DDI_FAILURE); 1359 break; 1360 case DDI_INTROP_ALLOC: 1361 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1362 return (DDI_FAILURE); 1363 hdlp->ih_pri = ispec->intrspec_pri; 1364 *(int *)result = hdlp->ih_scratch1; 1365 break; 1366 case DDI_INTROP_FREE: 1367 pdp = ddi_get_parent_data(rdip); 1368 /* 1369 * Special case for 'pcic' driver' only. 1370 * If an intrspec was created for it, clean it up here 1371 * See detailed comments on this in the function 1372 * rootnex_get_ispec(). 1373 */ 1374 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1375 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1376 pdp->par_nintr); 1377 /* 1378 * Set it to zero; so that 1379 * DDI framework doesn't free it again 1380 */ 1381 pdp->par_intr = NULL; 1382 pdp->par_nintr = 0; 1383 } 1384 break; 1385 case DDI_INTROP_GETPRI: 1386 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1387 return (DDI_FAILURE); 1388 *(int *)result = ispec->intrspec_pri; 1389 break; 1390 case DDI_INTROP_SETPRI: 1391 /* Validate the interrupt priority passed to us */ 1392 if (*(int *)result > LOCK_LEVEL) 1393 return (DDI_FAILURE); 1394 1395 /* Ensure that PSM is all initialized and ispec is ok */ 1396 if ((psm_intr_ops == NULL) || 1397 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1398 return (DDI_FAILURE); 1399 1400 /* Change the priority */ 1401 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1402 PSM_FAILURE) 1403 return (DDI_FAILURE); 1404 1405 /* update the ispec with the new priority */ 1406 ispec->intrspec_pri = *(int *)result; 1407 break; 1408 case DDI_INTROP_ADDISR: 1409 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1410 return (DDI_FAILURE); 1411 ispec->intrspec_func = hdlp->ih_cb_func; 1412 break; 1413 case DDI_INTROP_REMISR: 1414 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1415 return (DDI_FAILURE); 1416 ispec->intrspec_func = (uint_t (*)()) 0; 1417 break; 1418 case DDI_INTROP_ENABLE: 1419 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1420 return (DDI_FAILURE); 1421 1422 /* Call psmi to translate irq with the dip */ 1423 if (psm_intr_ops == NULL) 1424 return (DDI_FAILURE); 1425 1426 hdlp->ih_private = (void *)ispec; 1427 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1428 (int *)&hdlp->ih_vector); 1429 1430 /* Add the interrupt handler */ 1431 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1432 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1433 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, rdip)) 1434 return (DDI_FAILURE); 1435 break; 1436 case DDI_INTROP_DISABLE: 1437 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1438 return (DDI_FAILURE); 1439 1440 /* Call psm_ops() to translate irq with the dip */ 1441 if (psm_intr_ops == NULL) 1442 return (DDI_FAILURE); 1443 1444 hdlp->ih_private = (void *)ispec; 1445 (void) (*psm_intr_ops)(rdip, hdlp, 1446 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1447 1448 /* Remove the interrupt handler */ 1449 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1450 hdlp->ih_cb_func, hdlp->ih_vector); 1451 break; 1452 case DDI_INTROP_SETMASK: 1453 if (psm_intr_ops == NULL) 1454 return (DDI_FAILURE); 1455 1456 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1457 return (DDI_FAILURE); 1458 break; 1459 case DDI_INTROP_CLRMASK: 1460 if (psm_intr_ops == NULL) 1461 return (DDI_FAILURE); 1462 1463 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1464 return (DDI_FAILURE); 1465 break; 1466 case DDI_INTROP_GETPENDING: 1467 if (psm_intr_ops == NULL) 1468 return (DDI_FAILURE); 1469 1470 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1471 result)) { 1472 *(int *)result = 0; 1473 return (DDI_FAILURE); 1474 } 1475 break; 1476 case DDI_INTROP_NINTRS: 1477 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 1478 return (DDI_FAILURE); 1479 *(int *)result = pdp->par_nintr; 1480 if (pdp->par_nintr == 0) { 1481 /* 1482 * Special case for 'pcic' driver' only. This driver 1483 * driver is a child of 'isa' and 'rootnex' drivers. 1484 * 1485 * See detailed comments on this in the function 1486 * rootnex_get_ispec(). 1487 * 1488 * Children of 'pcic' send 'NINITR' request all the 1489 * way to rootnex driver. But, the 'pdp->par_nintr' 1490 * field may not initialized. So, we fake it here 1491 * to return 1 (a la what PCMCIA nexus does). 1492 */ 1493 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1494 *(int *)result = 1; 1495 } 1496 break; 1497 case DDI_INTROP_SUPPORTED_TYPES: 1498 *(int *)result = 0; 1499 *(int *)result |= DDI_INTR_TYPE_FIXED; /* Always ... */ 1500 break; 1501 case DDI_INTROP_NAVAIL: 1502 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1503 return (DDI_FAILURE); 1504 1505 if (psm_intr_ops == NULL) { 1506 *(int *)result = 1; 1507 break; 1508 } 1509 1510 /* Priority in the handle not initialized yet */ 1511 hdlp->ih_pri = ispec->intrspec_pri; 1512 (void) (*psm_intr_ops)(rdip, hdlp, 1513 PSM_INTR_OP_NAVAIL_VECTORS, result); 1514 break; 1515 default: 1516 return (DDI_FAILURE); 1517 } 1518 1519 return (DDI_SUCCESS); 1520 } 1521 1522 1523 /* 1524 * rootnex_get_ispec() 1525 * convert an interrupt number to an interrupt specification. 1526 * The interrupt number determines which interrupt spec will be 1527 * returned if more than one exists. 1528 * 1529 * Look into the parent private data area of the 'rdip' to find out 1530 * the interrupt specification. First check to make sure there is 1531 * one that matchs "inumber" and then return a pointer to it. 1532 * 1533 * Return NULL if one could not be found. 1534 * 1535 * NOTE: This is needed for rootnex_intr_ops() 1536 */ 1537 static struct intrspec * 1538 rootnex_get_ispec(dev_info_t *rdip, int inum) 1539 { 1540 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1541 1542 /* 1543 * Special case handling for drivers that provide their own 1544 * intrspec structures instead of relying on the DDI framework. 1545 * 1546 * A broken hardware driver in ON could potentially provide its 1547 * own intrspec structure, instead of relying on the hardware. 1548 * If these drivers are children of 'rootnex' then we need to 1549 * continue to provide backward compatibility to them here. 1550 * 1551 * Following check is a special case for 'pcic' driver which 1552 * was found to have broken hardwre andby provides its own intrspec. 1553 * 1554 * Verbatim comments from this driver are shown here: 1555 * "Don't use the ddi_add_intr since we don't have a 1556 * default intrspec in all cases." 1557 * 1558 * Since an 'ispec' may not be always created for it, 1559 * check for that and create one if so. 1560 * 1561 * NOTE: Currently 'pcic' is the only driver found to do this. 1562 */ 1563 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1564 pdp->par_nintr = 1; 1565 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1566 pdp->par_nintr, KM_SLEEP); 1567 } 1568 1569 /* Validate the interrupt number */ 1570 if (inum >= pdp->par_nintr) 1571 return (NULL); 1572 1573 /* Get the interrupt structure pointer and return that */ 1574 return ((struct intrspec *)&pdp->par_intr[inum]); 1575 } 1576 1577 1578 /* 1579 * ****************** 1580 * dma related code 1581 * ****************** 1582 */ 1583 1584 /* 1585 * rootnex_dma_allochdl() 1586 * called from ddi_dma_alloc_handle(). 1587 */ 1588 /*ARGSUSED*/ 1589 static int 1590 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1591 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1592 { 1593 uint64_t maxsegmentsize_ll; 1594 uint_t maxsegmentsize; 1595 ddi_dma_impl_t *hp; 1596 rootnex_dma_t *dma; 1597 uint64_t count_max; 1598 uint64_t seg; 1599 int kmflag; 1600 int e; 1601 1602 1603 /* convert our sleep flags */ 1604 if (waitfp == DDI_DMA_SLEEP) { 1605 kmflag = KM_SLEEP; 1606 } else { 1607 kmflag = KM_NOSLEEP; 1608 } 1609 1610 /* 1611 * We try to do only one memory allocation here. We'll do a little 1612 * pointer manipulation later. If the bind ends up taking more than 1613 * our prealloc's space, we'll have to allocate more memory in the 1614 * bind operation. Not great, but much better than before and the 1615 * best we can do with the current bind interfaces. 1616 */ 1617 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1618 if (hp == NULL) { 1619 if (waitfp != DDI_DMA_DONTWAIT) { 1620 ddi_set_callback(waitfp, arg, 1621 &rootnex_state->r_dvma_call_list_id); 1622 } 1623 return (DDI_DMA_NORESOURCES); 1624 } 1625 1626 /* Do our pointer manipulation now, align the structures */ 1627 hp->dmai_private = (void *)(((uintptr_t)hp + 1628 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1629 dma = (rootnex_dma_t *)hp->dmai_private; 1630 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1631 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1632 1633 /* setup the handle */ 1634 rootnex_clean_dmahdl(hp); 1635 dma->dp_dip = rdip; 1636 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1637 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1638 hp->dmai_minxfer = attr->dma_attr_minxfer; 1639 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1640 hp->dmai_rdip = rdip; 1641 hp->dmai_attr = *attr; 1642 1643 /* we don't need to worry about the SPL since we do a tryenter */ 1644 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1645 1646 /* 1647 * Figure out our maximum segment size. If the segment size is greater 1648 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1649 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1650 * dma_attr_count_max are size-1 type values. 1651 * 1652 * Maximum segment size is the largest physically contiguous chunk of 1653 * memory that we can return from a bind (i.e. the maximum size of a 1654 * single cookie). 1655 */ 1656 1657 /* handle the rollover cases */ 1658 seg = attr->dma_attr_seg + 1; 1659 if (seg < attr->dma_attr_seg) { 1660 seg = attr->dma_attr_seg; 1661 } 1662 count_max = attr->dma_attr_count_max + 1; 1663 if (count_max < attr->dma_attr_count_max) { 1664 count_max = attr->dma_attr_count_max; 1665 } 1666 1667 /* 1668 * granularity may or may not be a power of two. If it isn't, we can't 1669 * use a simple mask. 1670 */ 1671 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1672 dma->dp_granularity_power_2 = B_FALSE; 1673 } else { 1674 dma->dp_granularity_power_2 = B_TRUE; 1675 } 1676 1677 /* 1678 * maxxfer should be a whole multiple of granularity. If we're going to 1679 * break up a window because we're greater than maxxfer, we might as 1680 * well make sure it's maxxfer is a whole multiple so we don't have to 1681 * worry about triming the window later on for this case. 1682 */ 1683 if (attr->dma_attr_granular > 1) { 1684 if (dma->dp_granularity_power_2) { 1685 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1686 (attr->dma_attr_maxxfer & 1687 (attr->dma_attr_granular - 1)); 1688 } else { 1689 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1690 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1691 } 1692 } else { 1693 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1694 } 1695 1696 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1697 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1698 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1699 maxsegmentsize = 0xFFFFFFFF; 1700 } else { 1701 maxsegmentsize = maxsegmentsize_ll; 1702 } 1703 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1704 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1705 1706 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1707 if (rootnex_alloc_check_parms) { 1708 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1709 if (e != DDI_SUCCESS) { 1710 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1711 (void) rootnex_dma_freehdl(dip, rdip, 1712 (ddi_dma_handle_t)hp); 1713 return (e); 1714 } 1715 } 1716 1717 *handlep = (ddi_dma_handle_t)hp; 1718 1719 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1720 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1721 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1722 1723 return (DDI_SUCCESS); 1724 } 1725 1726 1727 /* 1728 * rootnex_dma_freehdl() 1729 * called from ddi_dma_free_handle(). 1730 */ 1731 /*ARGSUSED*/ 1732 static int 1733 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1734 { 1735 ddi_dma_impl_t *hp; 1736 rootnex_dma_t *dma; 1737 1738 1739 hp = (ddi_dma_impl_t *)handle; 1740 dma = (rootnex_dma_t *)hp->dmai_private; 1741 1742 /* unbind should have been called first */ 1743 ASSERT(!dma->dp_inuse); 1744 1745 mutex_destroy(&dma->dp_mutex); 1746 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1747 1748 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1749 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1750 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1751 1752 if (rootnex_state->r_dvma_call_list_id) 1753 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1754 1755 return (DDI_SUCCESS); 1756 } 1757 1758 1759 /* 1760 * rootnex_dma_bindhdl() 1761 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1762 */ 1763 /*ARGSUSED*/ 1764 static int 1765 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1766 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1767 { 1768 rootnex_sglinfo_t *sinfo; 1769 ddi_dma_attr_t *attr; 1770 ddi_dma_impl_t *hp; 1771 rootnex_dma_t *dma; 1772 int kmflag; 1773 int e; 1774 1775 1776 hp = (ddi_dma_impl_t *)handle; 1777 dma = (rootnex_dma_t *)hp->dmai_private; 1778 sinfo = &dma->dp_sglinfo; 1779 attr = &hp->dmai_attr; 1780 1781 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1782 1783 /* 1784 * This is useful for debugging a driver. Not as useful in a production 1785 * system. The only time this will fail is if you have a driver bug. 1786 */ 1787 if (rootnex_bind_check_inuse) { 1788 /* 1789 * No one else should ever have this lock unless someone else 1790 * is trying to use this handle. So contention on the lock 1791 * is the same as inuse being set. 1792 */ 1793 e = mutex_tryenter(&dma->dp_mutex); 1794 if (e == 0) { 1795 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1796 return (DDI_DMA_INUSE); 1797 } 1798 if (dma->dp_inuse) { 1799 mutex_exit(&dma->dp_mutex); 1800 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1801 return (DDI_DMA_INUSE); 1802 } 1803 dma->dp_inuse = B_TRUE; 1804 mutex_exit(&dma->dp_mutex); 1805 } 1806 1807 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1808 if (rootnex_bind_check_parms) { 1809 e = rootnex_valid_bind_parms(dmareq, attr); 1810 if (e != DDI_SUCCESS) { 1811 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1812 rootnex_clean_dmahdl(hp); 1813 return (e); 1814 } 1815 } 1816 1817 /* save away the original bind info */ 1818 dma->dp_dma = dmareq->dmar_object; 1819 1820 /* 1821 * Figure out a rough estimate of what maximum number of pages this 1822 * buffer could use (a high estimate of course). 1823 */ 1824 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1825 1826 /* 1827 * We'll use the pre-allocated cookies for any bind that will *always* 1828 * fit (more important to be consistent, we don't want to create 1829 * additional degenerate cases). 1830 */ 1831 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1832 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1833 dma->dp_need_to_free_cookie = B_FALSE; 1834 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1835 uint_t, sinfo->si_max_pages); 1836 1837 /* 1838 * For anything larger than that, we'll go ahead and allocate the 1839 * maximum number of pages we expect to see. Hopefuly, we won't be 1840 * seeing this path in the fast path for high performance devices very 1841 * frequently. 1842 * 1843 * a ddi bind interface that allowed the driver to provide storage to 1844 * the bind interface would speed this case up. 1845 */ 1846 } else { 1847 /* convert the sleep flags */ 1848 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1849 kmflag = KM_SLEEP; 1850 } else { 1851 kmflag = KM_NOSLEEP; 1852 } 1853 1854 /* 1855 * Save away how much memory we allocated. If we're doing a 1856 * nosleep, the alloc could fail... 1857 */ 1858 dma->dp_cookie_size = sinfo->si_max_pages * 1859 sizeof (ddi_dma_cookie_t); 1860 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1861 if (dma->dp_cookies == NULL) { 1862 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1863 rootnex_clean_dmahdl(hp); 1864 return (DDI_DMA_NORESOURCES); 1865 } 1866 dma->dp_need_to_free_cookie = B_TRUE; 1867 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1868 sinfo->si_max_pages); 1869 } 1870 hp->dmai_cookie = dma->dp_cookies; 1871 1872 /* 1873 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1874 * looking at the contraints in the dma structure. It will then put some 1875 * additional state about the sgl in the dma struct (i.e. is the sgl 1876 * clean, or do we need to do some munging; how many pages need to be 1877 * copied, etc.) 1878 */ 1879 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1880 &dma->dp_sglinfo); 1881 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1882 1883 /* if we don't need a copy buffer, we don't need to sync */ 1884 if (sinfo->si_copybuf_req == 0) { 1885 hp->dmai_rflags |= DMP_NOSYNC; 1886 } 1887 1888 /* 1889 * if we don't need the copybuf and we don't need to do a partial, we 1890 * hit the fast path. All the high performance devices should be trying 1891 * to hit this path. To hit this path, a device should be able to reach 1892 * all of memory, shouldn't try to bind more than it can transfer, and 1893 * the buffer shouldn't require more cookies than the driver/device can 1894 * handle [sgllen]). 1895 */ 1896 if ((sinfo->si_copybuf_req == 0) && 1897 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1898 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1899 /* 1900 * copy out the first cookie and ccountp, set the cookie 1901 * pointer to the second cookie. The first cookie is passed 1902 * back on the stack. Additional cookies are accessed via 1903 * ddi_dma_nextcookie() 1904 */ 1905 *cookiep = dma->dp_cookies[0]; 1906 *ccountp = sinfo->si_sgl_size; 1907 hp->dmai_cookie++; 1908 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1909 hp->dmai_nwin = 1; 1910 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1911 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1912 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1913 dma->dp_dma.dmao_size); 1914 return (DDI_DMA_MAPPED); 1915 } 1916 1917 /* 1918 * go to the slow path, we may need to alloc more memory, create 1919 * multiple windows, and munge up a sgl to make the device happy. 1920 */ 1921 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 1922 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 1923 if (dma->dp_need_to_free_cookie) { 1924 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1925 } 1926 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1927 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 1928 return (e); 1929 } 1930 1931 /* if the first window uses the copy buffer, sync it for the device */ 1932 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 1933 (hp->dmai_rflags & DDI_DMA_WRITE)) { 1934 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1935 DDI_DMA_SYNC_FORDEV); 1936 } 1937 1938 /* 1939 * copy out the first cookie and ccountp, set the cookie pointer to the 1940 * second cookie. Make sure the partial flag is set/cleared correctly. 1941 * If we have a partial map (i.e. multiple windows), the number of 1942 * cookies we return is the number of cookies in the first window. 1943 */ 1944 if (e == DDI_DMA_MAPPED) { 1945 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1946 *ccountp = sinfo->si_sgl_size; 1947 } else { 1948 hp->dmai_rflags |= DDI_DMA_PARTIAL; 1949 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 1950 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 1951 } 1952 *cookiep = dma->dp_cookies[0]; 1953 hp->dmai_cookie++; 1954 1955 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1956 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 1957 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1958 dma->dp_dma.dmao_size); 1959 return (e); 1960 } 1961 1962 1963 /* 1964 * rootnex_dma_unbindhdl() 1965 * called from ddi_dma_unbind_handle() 1966 */ 1967 /*ARGSUSED*/ 1968 static int 1969 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 1970 ddi_dma_handle_t handle) 1971 { 1972 ddi_dma_impl_t *hp; 1973 rootnex_dma_t *dma; 1974 int e; 1975 1976 1977 hp = (ddi_dma_impl_t *)handle; 1978 dma = (rootnex_dma_t *)hp->dmai_private; 1979 1980 /* make sure the buffer wasn't free'd before calling unbind */ 1981 if (rootnex_unbind_verify_buffer) { 1982 e = rootnex_verify_buffer(dma); 1983 if (e != DDI_SUCCESS) { 1984 ASSERT(0); 1985 return (DDI_FAILURE); 1986 } 1987 } 1988 1989 /* sync the current window before unbinding the buffer */ 1990 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 1991 (hp->dmai_rflags & DDI_DMA_READ)) { 1992 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1993 DDI_DMA_SYNC_FORCPU); 1994 } 1995 1996 /* 1997 * cleanup and copy buffer or window state. if we didn't use the copy 1998 * buffer or windows, there won't be much to do :-) 1999 */ 2000 rootnex_teardown_copybuf(dma); 2001 rootnex_teardown_windows(dma); 2002 2003 /* 2004 * If we had to allocate space to for the worse case sgl (it didn't 2005 * fit into our pre-allocate buffer), free that up now 2006 */ 2007 if (dma->dp_need_to_free_cookie) { 2008 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2009 } 2010 2011 /* 2012 * clean up the handle so it's ready for the next bind (i.e. if the 2013 * handle is reused). 2014 */ 2015 rootnex_clean_dmahdl(hp); 2016 2017 if (rootnex_state->r_dvma_call_list_id) 2018 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2019 2020 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2021 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2022 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2023 2024 return (DDI_SUCCESS); 2025 } 2026 2027 2028 /* 2029 * rootnex_verify_buffer() 2030 * verify buffer wasn't free'd 2031 */ 2032 static int 2033 rootnex_verify_buffer(rootnex_dma_t *dma) 2034 { 2035 peekpoke_ctlops_t peek; 2036 page_t **pplist; 2037 caddr_t vaddr; 2038 uint_t pcnt; 2039 uint_t poff; 2040 page_t *pp; 2041 uint8_t b; 2042 int i; 2043 int e; 2044 2045 2046 /* Figure out how many pages this buffer occupies */ 2047 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2048 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2049 } else { 2050 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2051 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2052 } 2053 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2054 2055 switch (dma->dp_dma.dmao_type) { 2056 case DMA_OTYP_PAGES: 2057 /* 2058 * for a linked list of pp's walk through them to make sure 2059 * they're locked and not free. 2060 */ 2061 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2062 for (i = 0; i < pcnt; i++) { 2063 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2064 return (DDI_FAILURE); 2065 } 2066 pp = pp->p_next; 2067 } 2068 break; 2069 2070 case DMA_OTYP_VADDR: 2071 case DMA_OTYP_BUFVADDR: 2072 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2073 /* 2074 * for an array of pp's walk through them to make sure they're 2075 * not free. It's possible that they may not be locked. 2076 */ 2077 if (pplist) { 2078 for (i = 0; i < pcnt; i++) { 2079 if (PP_ISFREE(pplist[i])) { 2080 return (DDI_FAILURE); 2081 } 2082 } 2083 2084 /* For a virtual address, try to peek at each page */ 2085 } else { 2086 if (dma->dp_sglinfo.si_asp == &kas) { 2087 bzero(&peek, sizeof (peekpoke_ctlops_t)); 2088 peek.host_addr = (uintptr_t)&b; 2089 peek.size = sizeof (uint8_t); 2090 peek.dev_addr = (uintptr_t)vaddr; 2091 for (i = 0; i < pcnt; i++) { 2092 e = rootnex_ctlops_peek(&peek, &b); 2093 if (e != DDI_SUCCESS) { 2094 return (DDI_FAILURE); 2095 } 2096 peek.dev_addr += MMU_PAGESIZE; 2097 } 2098 } 2099 } 2100 break; 2101 2102 default: 2103 ASSERT(0); 2104 break; 2105 } 2106 2107 return (DDI_SUCCESS); 2108 } 2109 2110 2111 /* 2112 * rootnex_clean_dmahdl() 2113 * Clean the dma handle. This should be called on a handle alloc and an 2114 * unbind handle. Set the handle state to the default settings. 2115 */ 2116 static void 2117 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2118 { 2119 rootnex_dma_t *dma; 2120 2121 2122 dma = (rootnex_dma_t *)hp->dmai_private; 2123 2124 hp->dmai_nwin = 0; 2125 dma->dp_current_cookie = 0; 2126 dma->dp_copybuf_size = 0; 2127 dma->dp_window = NULL; 2128 dma->dp_cbaddr = NULL; 2129 dma->dp_inuse = B_FALSE; 2130 dma->dp_need_to_free_cookie = B_FALSE; 2131 dma->dp_need_to_free_window = B_FALSE; 2132 dma->dp_partial_required = B_FALSE; 2133 dma->dp_trim_required = B_FALSE; 2134 dma->dp_sglinfo.si_copybuf_req = 0; 2135 #if !defined(__amd64) 2136 dma->dp_cb_remaping = B_FALSE; 2137 dma->dp_kva = NULL; 2138 #endif 2139 2140 /* FMA related initialization */ 2141 hp->dmai_fault = 0; 2142 hp->dmai_fault_check = NULL; 2143 hp->dmai_fault_notify = NULL; 2144 hp->dmai_error.err_ena = 0; 2145 hp->dmai_error.err_status = DDI_FM_OK; 2146 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2147 hp->dmai_error.err_ontrap = NULL; 2148 hp->dmai_error.err_fep = NULL; 2149 } 2150 2151 2152 /* 2153 * rootnex_valid_alloc_parms() 2154 * Called in ddi_dma_alloc_handle path to validate its parameters. 2155 */ 2156 static int 2157 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2158 { 2159 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2160 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2161 (attr->dma_attr_granular > MMU_PAGESIZE) || 2162 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2163 return (DDI_DMA_BADATTR); 2164 } 2165 2166 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2167 return (DDI_DMA_BADATTR); 2168 } 2169 2170 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2171 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2172 attr->dma_attr_sgllen <= 0) { 2173 return (DDI_DMA_BADATTR); 2174 } 2175 2176 /* We should be able to DMA into every byte offset in a page */ 2177 if (maxsegmentsize < MMU_PAGESIZE) { 2178 return (DDI_DMA_BADATTR); 2179 } 2180 2181 return (DDI_SUCCESS); 2182 } 2183 2184 2185 /* 2186 * rootnex_valid_bind_parms() 2187 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2188 */ 2189 /* ARGSUSED */ 2190 static int 2191 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2192 { 2193 #if !defined(__amd64) 2194 /* 2195 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2196 * we can track the offset for the obsoleted interfaces. 2197 */ 2198 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2199 return (DDI_DMA_TOOBIG); 2200 } 2201 #endif 2202 2203 return (DDI_SUCCESS); 2204 } 2205 2206 2207 /* 2208 * rootnex_get_sgl() 2209 * Called in bind fastpath to get the sgl. Most of this will be replaced 2210 * with a call to the vm layer when vm2.0 comes around... 2211 */ 2212 static void 2213 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2214 rootnex_sglinfo_t *sglinfo) 2215 { 2216 ddi_dma_atyp_t buftype; 2217 uint64_t last_page; 2218 uint64_t offset; 2219 uint64_t addrhi; 2220 uint64_t addrlo; 2221 uint64_t maxseg; 2222 page_t **pplist; 2223 uint64_t paddr; 2224 uint32_t psize; 2225 uint32_t size; 2226 caddr_t vaddr; 2227 uint_t pcnt; 2228 page_t *pp; 2229 uint_t cnt; 2230 2231 2232 /* shortcuts */ 2233 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2234 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2235 maxseg = sglinfo->si_max_cookie_size; 2236 buftype = dmar_object->dmao_type; 2237 addrhi = sglinfo->si_max_addr; 2238 addrlo = sglinfo->si_min_addr; 2239 size = dmar_object->dmao_size; 2240 2241 pcnt = 0; 2242 cnt = 0; 2243 2244 /* 2245 * if we were passed down a linked list of pages, i.e. pointer to 2246 * page_t, use this to get our physical address and buf offset. 2247 */ 2248 if (buftype == DMA_OTYP_PAGES) { 2249 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2250 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2251 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2252 MMU_PAGEOFFSET; 2253 paddr = ptob64(pp->p_pagenum) + offset; 2254 psize = MIN(size, (MMU_PAGESIZE - offset)); 2255 pp = pp->p_next; 2256 sglinfo->si_asp = NULL; 2257 2258 /* 2259 * We weren't passed down a linked list of pages, but if we were passed 2260 * down an array of pages, use this to get our physical address and buf 2261 * offset. 2262 */ 2263 } else if (pplist != NULL) { 2264 ASSERT((buftype == DMA_OTYP_VADDR) || 2265 (buftype == DMA_OTYP_BUFVADDR)); 2266 2267 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2268 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2269 if (sglinfo->si_asp == NULL) { 2270 sglinfo->si_asp = &kas; 2271 } 2272 2273 ASSERT(!PP_ISFREE(pplist[pcnt])); 2274 paddr = ptob64(pplist[pcnt]->p_pagenum); 2275 paddr += offset; 2276 psize = MIN(size, (MMU_PAGESIZE - offset)); 2277 pcnt++; 2278 2279 /* 2280 * All we have is a virtual address, we'll need to call into the VM 2281 * to get the physical address. 2282 */ 2283 } else { 2284 ASSERT((buftype == DMA_OTYP_VADDR) || 2285 (buftype == DMA_OTYP_BUFVADDR)); 2286 2287 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2288 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2289 if (sglinfo->si_asp == NULL) { 2290 sglinfo->si_asp = &kas; 2291 } 2292 2293 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2294 paddr += offset; 2295 psize = MIN(size, (MMU_PAGESIZE - offset)); 2296 vaddr += psize; 2297 } 2298 2299 /* 2300 * Setup the first cookie with the physical address of the page and the 2301 * size of the page (which takes into account the initial offset into 2302 * the page. 2303 */ 2304 sgl[cnt].dmac_laddress = paddr; 2305 sgl[cnt].dmac_size = psize; 2306 sgl[cnt].dmac_type = 0; 2307 2308 /* 2309 * Save away the buffer offset into the page. We'll need this later in 2310 * the copy buffer code to help figure out the page index within the 2311 * buffer and the offset into the current page. 2312 */ 2313 sglinfo->si_buf_offset = offset; 2314 2315 /* 2316 * If the DMA engine can't reach the physical address, increase how 2317 * much copy buffer we need. We always increase by pagesize so we don't 2318 * have to worry about converting offsets. Set a flag in the cookies 2319 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2320 * last cookie, go to the next cookie (since we separate each page which 2321 * uses the copy buffer in case the copy buffer is not physically 2322 * contiguous. 2323 */ 2324 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2325 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2326 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2327 if ((cnt + 1) < sglinfo->si_max_pages) { 2328 cnt++; 2329 sgl[cnt].dmac_laddress = 0; 2330 sgl[cnt].dmac_size = 0; 2331 sgl[cnt].dmac_type = 0; 2332 } 2333 } 2334 2335 /* 2336 * save this page's physical address so we can figure out if the next 2337 * page is physically contiguous. Keep decrementing size until we are 2338 * done with the buffer. 2339 */ 2340 last_page = paddr & MMU_PAGEMASK; 2341 size -= psize; 2342 2343 while (size > 0) { 2344 /* Get the size for this page (i.e. partial or full page) */ 2345 psize = MIN(size, MMU_PAGESIZE); 2346 2347 if (buftype == DMA_OTYP_PAGES) { 2348 /* get the paddr from the page_t */ 2349 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2350 paddr = ptob64(pp->p_pagenum); 2351 pp = pp->p_next; 2352 } else if (pplist != NULL) { 2353 /* index into the array of page_t's to get the paddr */ 2354 ASSERT(!PP_ISFREE(pplist[pcnt])); 2355 paddr = ptob64(pplist[pcnt]->p_pagenum); 2356 pcnt++; 2357 } else { 2358 /* call into the VM to get the paddr */ 2359 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, 2360 vaddr)); 2361 vaddr += psize; 2362 } 2363 2364 /* check to see if this page needs the copy buffer */ 2365 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2366 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2367 2368 /* 2369 * if there is something in the current cookie, go to 2370 * the next one. We only want one page in a cookie which 2371 * uses the copybuf since the copybuf doesn't have to 2372 * be physically contiguous. 2373 */ 2374 if (sgl[cnt].dmac_size != 0) { 2375 cnt++; 2376 } 2377 sgl[cnt].dmac_laddress = paddr; 2378 sgl[cnt].dmac_size = psize; 2379 #if defined(__amd64) 2380 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2381 #else 2382 /* 2383 * save the buf offset for 32-bit kernel. used in the 2384 * obsoleted interfaces. 2385 */ 2386 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2387 (dmar_object->dmao_size - size); 2388 #endif 2389 /* if this isn't the last cookie, go to the next one */ 2390 if ((cnt + 1) < sglinfo->si_max_pages) { 2391 cnt++; 2392 sgl[cnt].dmac_laddress = 0; 2393 sgl[cnt].dmac_size = 0; 2394 sgl[cnt].dmac_type = 0; 2395 } 2396 2397 /* 2398 * this page didn't need the copy buffer, if it's not physically 2399 * contiguous, or it would put us over a segment boundary, or it 2400 * puts us over the max cookie size, or the current sgl doesn't 2401 * have anything in it. 2402 */ 2403 } else if (((last_page + MMU_PAGESIZE) != paddr) || 2404 !(paddr & sglinfo->si_segmask) || 2405 ((sgl[cnt].dmac_size + psize) > maxseg) || 2406 (sgl[cnt].dmac_size == 0)) { 2407 /* 2408 * if we're not already in a new cookie, go to the next 2409 * cookie. 2410 */ 2411 if (sgl[cnt].dmac_size != 0) { 2412 cnt++; 2413 } 2414 2415 /* save the cookie information */ 2416 sgl[cnt].dmac_laddress = paddr; 2417 sgl[cnt].dmac_size = psize; 2418 #if defined(__amd64) 2419 sgl[cnt].dmac_type = 0; 2420 #else 2421 /* 2422 * save the buf offset for 32-bit kernel. used in the 2423 * obsoleted interfaces. 2424 */ 2425 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2426 #endif 2427 2428 /* 2429 * this page didn't need the copy buffer, it is physically 2430 * contiguous with the last page, and it's <= the max cookie 2431 * size. 2432 */ 2433 } else { 2434 sgl[cnt].dmac_size += psize; 2435 2436 /* 2437 * if this exactly == the maximum cookie size, and 2438 * it isn't the last cookie, go to the next cookie. 2439 */ 2440 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2441 ((cnt + 1) < sglinfo->si_max_pages)) { 2442 cnt++; 2443 sgl[cnt].dmac_laddress = 0; 2444 sgl[cnt].dmac_size = 0; 2445 sgl[cnt].dmac_type = 0; 2446 } 2447 } 2448 2449 /* 2450 * save this page's physical address so we can figure out if the 2451 * next page is physically contiguous. Keep decrementing size 2452 * until we are done with the buffer. 2453 */ 2454 last_page = paddr; 2455 size -= psize; 2456 } 2457 2458 /* we're done, save away how many cookies the sgl has */ 2459 if (sgl[cnt].dmac_size == 0) { 2460 ASSERT(cnt < sglinfo->si_max_pages); 2461 sglinfo->si_sgl_size = cnt; 2462 } else { 2463 sglinfo->si_sgl_size = cnt + 1; 2464 } 2465 } 2466 2467 2468 /* 2469 * rootnex_bind_slowpath() 2470 * Call in the bind path if the calling driver can't use the sgl without 2471 * modifying it. We either need to use the copy buffer and/or we will end up 2472 * with a partial bind. 2473 */ 2474 static int 2475 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2476 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2477 { 2478 rootnex_sglinfo_t *sinfo; 2479 rootnex_window_t *window; 2480 ddi_dma_cookie_t *cookie; 2481 size_t copybuf_used; 2482 size_t dmac_size; 2483 boolean_t partial; 2484 off_t cur_offset; 2485 page_t *cur_pp; 2486 major_t mnum; 2487 int e; 2488 int i; 2489 2490 2491 sinfo = &dma->dp_sglinfo; 2492 copybuf_used = 0; 2493 partial = B_FALSE; 2494 2495 /* 2496 * If we're using the copybuf, set the copybuf state in dma struct. 2497 * Needs to be first since it sets the copy buffer size. 2498 */ 2499 if (sinfo->si_copybuf_req != 0) { 2500 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2501 if (e != DDI_SUCCESS) { 2502 return (e); 2503 } 2504 } else { 2505 dma->dp_copybuf_size = 0; 2506 } 2507 2508 /* 2509 * Figure out if we need to do a partial mapping. If so, figure out 2510 * if we need to trim the buffers when we munge the sgl. 2511 */ 2512 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2513 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2514 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2515 dma->dp_partial_required = B_TRUE; 2516 if (attr->dma_attr_granular != 1) { 2517 dma->dp_trim_required = B_TRUE; 2518 } 2519 } else { 2520 dma->dp_partial_required = B_FALSE; 2521 dma->dp_trim_required = B_FALSE; 2522 } 2523 2524 /* If we need to do a partial bind, make sure the driver supports it */ 2525 if (dma->dp_partial_required && 2526 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2527 2528 mnum = ddi_driver_major(dma->dp_dip); 2529 /* 2530 * patchable which allows us to print one warning per major 2531 * number. 2532 */ 2533 if ((rootnex_bind_warn) && 2534 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2535 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2536 cmn_err(CE_WARN, "!%s: coding error detected, the " 2537 "driver is using ddi_dma_attr(9S) incorrectly. " 2538 "There is a small risk of data corruption in " 2539 "particular with large I/Os. The driver should be " 2540 "replaced with a corrected version for proper " 2541 "system operation. To disable this warning, add " 2542 "'set rootnex:rootnex_bind_warn=0' to " 2543 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2544 } 2545 return (DDI_DMA_TOOBIG); 2546 } 2547 2548 /* 2549 * we might need multiple windows, setup state to handle them. In this 2550 * code path, we will have at least one window. 2551 */ 2552 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2553 if (e != DDI_SUCCESS) { 2554 rootnex_teardown_copybuf(dma); 2555 return (e); 2556 } 2557 2558 window = &dma->dp_window[0]; 2559 cookie = &dma->dp_cookies[0]; 2560 cur_offset = 0; 2561 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2562 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2563 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2564 } 2565 2566 /* loop though all the cookies we got back from get_sgl() */ 2567 for (i = 0; i < sinfo->si_sgl_size; i++) { 2568 /* 2569 * If we're using the copy buffer, check this cookie and setup 2570 * its associated copy buffer state. If this cookie uses the 2571 * copy buffer, make sure we sync this window during dma_sync. 2572 */ 2573 if (dma->dp_copybuf_size > 0) { 2574 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2575 cur_offset, ©buf_used, &cur_pp); 2576 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2577 window->wd_dosync = B_TRUE; 2578 } 2579 } 2580 2581 /* 2582 * save away the cookie size, since it could be modified in 2583 * the windowing code. 2584 */ 2585 dmac_size = cookie->dmac_size; 2586 2587 /* if we went over max copybuf size */ 2588 if (dma->dp_copybuf_size && 2589 (copybuf_used > dma->dp_copybuf_size)) { 2590 partial = B_TRUE; 2591 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2592 cookie, cur_offset, ©buf_used); 2593 if (e != DDI_SUCCESS) { 2594 rootnex_teardown_copybuf(dma); 2595 rootnex_teardown_windows(dma); 2596 return (e); 2597 } 2598 2599 /* 2600 * if the coookie uses the copy buffer, make sure the 2601 * new window we just moved to is set to sync. 2602 */ 2603 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2604 window->wd_dosync = B_TRUE; 2605 } 2606 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2607 dma->dp_dip); 2608 2609 /* if the cookie cnt == max sgllen, move to the next window */ 2610 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2611 partial = B_TRUE; 2612 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2613 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2614 cookie, attr, cur_offset); 2615 if (e != DDI_SUCCESS) { 2616 rootnex_teardown_copybuf(dma); 2617 rootnex_teardown_windows(dma); 2618 return (e); 2619 } 2620 2621 /* 2622 * if the coookie uses the copy buffer, make sure the 2623 * new window we just moved to is set to sync. 2624 */ 2625 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2626 window->wd_dosync = B_TRUE; 2627 } 2628 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2629 dma->dp_dip); 2630 2631 /* else if we will be over maxxfer */ 2632 } else if ((window->wd_size + dmac_size) > 2633 dma->dp_maxxfer) { 2634 partial = B_TRUE; 2635 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2636 cookie); 2637 if (e != DDI_SUCCESS) { 2638 rootnex_teardown_copybuf(dma); 2639 rootnex_teardown_windows(dma); 2640 return (e); 2641 } 2642 2643 /* 2644 * if the coookie uses the copy buffer, make sure the 2645 * new window we just moved to is set to sync. 2646 */ 2647 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2648 window->wd_dosync = B_TRUE; 2649 } 2650 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2651 dma->dp_dip); 2652 2653 /* else this cookie fits in the current window */ 2654 } else { 2655 window->wd_cookie_cnt++; 2656 window->wd_size += dmac_size; 2657 } 2658 2659 /* track our offset into the buffer, go to the next cookie */ 2660 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2661 ASSERT(cookie->dmac_size <= dmac_size); 2662 cur_offset += dmac_size; 2663 cookie++; 2664 } 2665 2666 /* if we ended up with a zero sized window in the end, clean it up */ 2667 if (window->wd_size == 0) { 2668 hp->dmai_nwin--; 2669 window--; 2670 } 2671 2672 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2673 2674 if (!partial) { 2675 return (DDI_DMA_MAPPED); 2676 } 2677 2678 ASSERT(dma->dp_partial_required); 2679 return (DDI_DMA_PARTIAL_MAP); 2680 } 2681 2682 2683 /* 2684 * rootnex_setup_copybuf() 2685 * Called in bind slowpath. Figures out if we're going to use the copy 2686 * buffer, and if we do, sets up the basic state to handle it. 2687 */ 2688 static int 2689 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2690 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2691 { 2692 rootnex_sglinfo_t *sinfo; 2693 ddi_dma_attr_t lattr; 2694 size_t max_copybuf; 2695 int cansleep; 2696 int e; 2697 #if !defined(__amd64) 2698 int vmflag; 2699 #endif 2700 2701 2702 sinfo = &dma->dp_sglinfo; 2703 2704 /* 2705 * read this first so it's consistent through the routine so we can 2706 * patch it on the fly. 2707 */ 2708 max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK; 2709 2710 /* We need to call into the rootnex on ddi_dma_sync() */ 2711 hp->dmai_rflags &= ~DMP_NOSYNC; 2712 2713 /* make sure the copybuf size <= the max size */ 2714 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2715 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2716 2717 #if !defined(__amd64) 2718 /* 2719 * if we don't have kva space to copy to/from, allocate the KVA space 2720 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2721 * the 64-bit kernel. 2722 */ 2723 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2724 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2725 2726 /* convert the sleep flags */ 2727 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2728 vmflag = VM_SLEEP; 2729 } else { 2730 vmflag = VM_NOSLEEP; 2731 } 2732 2733 /* allocate Kernel VA space that we can bcopy to/from */ 2734 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2735 vmflag); 2736 if (dma->dp_kva == NULL) { 2737 return (DDI_DMA_NORESOURCES); 2738 } 2739 } 2740 #endif 2741 2742 /* convert the sleep flags */ 2743 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2744 cansleep = 1; 2745 } else { 2746 cansleep = 0; 2747 } 2748 2749 /* 2750 * Allocated the actual copy buffer. This needs to fit within the DMA 2751 * engines limits, so we can't use kmem_alloc... 2752 */ 2753 lattr = *attr; 2754 lattr.dma_attr_align = MMU_PAGESIZE; 2755 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2756 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2757 if (e != DDI_SUCCESS) { 2758 #if !defined(__amd64) 2759 if (dma->dp_kva != NULL) { 2760 vmem_free(heap_arena, dma->dp_kva, 2761 dma->dp_copybuf_size); 2762 } 2763 #endif 2764 return (DDI_DMA_NORESOURCES); 2765 } 2766 2767 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2768 size_t, dma->dp_copybuf_size); 2769 2770 return (DDI_SUCCESS); 2771 } 2772 2773 2774 /* 2775 * rootnex_setup_windows() 2776 * Called in bind slowpath to setup the window state. We always have windows 2777 * in the slowpath. Even if the window count = 1. 2778 */ 2779 static int 2780 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2781 ddi_dma_attr_t *attr, int kmflag) 2782 { 2783 rootnex_window_t *windowp; 2784 rootnex_sglinfo_t *sinfo; 2785 size_t copy_state_size; 2786 size_t win_state_size; 2787 size_t state_available; 2788 size_t space_needed; 2789 uint_t copybuf_win; 2790 uint_t maxxfer_win; 2791 size_t space_used; 2792 uint_t sglwin; 2793 2794 2795 sinfo = &dma->dp_sglinfo; 2796 2797 dma->dp_current_win = 0; 2798 hp->dmai_nwin = 0; 2799 2800 /* If we don't need to do a partial, we only have one window */ 2801 if (!dma->dp_partial_required) { 2802 dma->dp_max_win = 1; 2803 2804 /* 2805 * we need multiple windows, need to figure out the worse case number 2806 * of windows. 2807 */ 2808 } else { 2809 /* 2810 * if we need windows because we need more copy buffer that 2811 * we allow, the worse case number of windows we could need 2812 * here would be (copybuf space required / copybuf space that 2813 * we have) plus one for remainder, and plus 2 to handle the 2814 * extra pages on the trim for the first and last pages of the 2815 * buffer (a page is the minimum window size so under the right 2816 * attr settings, you could have a window for each page). 2817 * The last page will only be hit here if the size is not a 2818 * multiple of the granularity (which theoretically shouldn't 2819 * be the case but never has been enforced, so we could have 2820 * broken things without it). 2821 */ 2822 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2823 ASSERT(dma->dp_copybuf_size > 0); 2824 copybuf_win = (sinfo->si_copybuf_req / 2825 dma->dp_copybuf_size) + 1 + 2; 2826 } else { 2827 copybuf_win = 0; 2828 } 2829 2830 /* 2831 * if we need windows because we have more cookies than the H/W 2832 * can handle, the number of windows we would need here would 2833 * be (cookie count / cookies count H/W supports) plus one for 2834 * remainder, and plus 2 to handle the extra pages on the trim 2835 * (see above comment about trim) 2836 */ 2837 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2838 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2839 + 1) + 2; 2840 } else { 2841 sglwin = 0; 2842 } 2843 2844 /* 2845 * if we need windows because we're binding more memory than the 2846 * H/W can transfer at once, the number of windows we would need 2847 * here would be (xfer count / max xfer H/W supports) plus one 2848 * for remainder, and plus 2 to handle the extra pages on the 2849 * trim (see above comment about trim) 2850 */ 2851 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 2852 maxxfer_win = (dma->dp_dma.dmao_size / 2853 dma->dp_maxxfer) + 1 + 2; 2854 } else { 2855 maxxfer_win = 0; 2856 } 2857 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 2858 ASSERT(dma->dp_max_win > 0); 2859 } 2860 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 2861 2862 /* 2863 * Get space for window and potential copy buffer state. Before we 2864 * go and allocate memory, see if we can get away with using what's 2865 * left in the pre-allocted state or the dynamically allocated sgl. 2866 */ 2867 space_used = (uintptr_t)(sinfo->si_sgl_size * 2868 sizeof (ddi_dma_cookie_t)); 2869 2870 /* if we dynamically allocated space for the cookies */ 2871 if (dma->dp_need_to_free_cookie) { 2872 /* if we have more space in the pre-allocted buffer, use it */ 2873 ASSERT(space_used <= dma->dp_cookie_size); 2874 if ((dma->dp_cookie_size - space_used) <= 2875 rootnex_state->r_prealloc_size) { 2876 state_available = rootnex_state->r_prealloc_size; 2877 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 2878 2879 /* 2880 * else, we have more free space in the dynamically allocated 2881 * buffer, i.e. the buffer wasn't worse case fragmented so we 2882 * didn't need a lot of cookies. 2883 */ 2884 } else { 2885 state_available = dma->dp_cookie_size - space_used; 2886 windowp = (rootnex_window_t *) 2887 &dma->dp_cookies[sinfo->si_sgl_size]; 2888 } 2889 2890 /* we used the pre-alloced buffer */ 2891 } else { 2892 ASSERT(space_used <= rootnex_state->r_prealloc_size); 2893 state_available = rootnex_state->r_prealloc_size - space_used; 2894 windowp = (rootnex_window_t *) 2895 &dma->dp_cookies[sinfo->si_sgl_size]; 2896 } 2897 2898 /* 2899 * figure out how much state we need to track the copy buffer. Add an 2900 * addition 8 bytes for pointer alignemnt later. 2901 */ 2902 if (dma->dp_copybuf_size > 0) { 2903 copy_state_size = sinfo->si_max_pages * 2904 sizeof (rootnex_pgmap_t); 2905 } else { 2906 copy_state_size = 0; 2907 } 2908 /* add an additional 8 bytes for pointer alignment */ 2909 space_needed = win_state_size + copy_state_size + 0x8; 2910 2911 /* if we have enough space already, use it */ 2912 if (state_available >= space_needed) { 2913 dma->dp_window = windowp; 2914 dma->dp_need_to_free_window = B_FALSE; 2915 2916 /* not enough space, need to allocate more. */ 2917 } else { 2918 dma->dp_window = kmem_alloc(space_needed, kmflag); 2919 if (dma->dp_window == NULL) { 2920 return (DDI_DMA_NORESOURCES); 2921 } 2922 dma->dp_need_to_free_window = B_TRUE; 2923 dma->dp_window_size = space_needed; 2924 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 2925 dma->dp_dip, size_t, space_needed); 2926 } 2927 2928 /* 2929 * we allocate copy buffer state and window state at the same time. 2930 * setup our copy buffer state pointers. Make sure it's aligned. 2931 */ 2932 if (dma->dp_copybuf_size > 0) { 2933 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 2934 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 2935 2936 #if !defined(__amd64) 2937 /* 2938 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 2939 * false/NULL. Should be quicker to bzero vs loop and set. 2940 */ 2941 bzero(dma->dp_pgmap, copy_state_size); 2942 #endif 2943 } else { 2944 dma->dp_pgmap = NULL; 2945 } 2946 2947 return (DDI_SUCCESS); 2948 } 2949 2950 2951 /* 2952 * rootnex_teardown_copybuf() 2953 * cleans up after rootnex_setup_copybuf() 2954 */ 2955 static void 2956 rootnex_teardown_copybuf(rootnex_dma_t *dma) 2957 { 2958 #if !defined(__amd64) 2959 int i; 2960 2961 /* 2962 * if we allocated kernel heap VMEM space, go through all the pages and 2963 * map out any of the ones that we're mapped into the kernel heap VMEM 2964 * arena. Then free the VMEM space. 2965 */ 2966 if (dma->dp_kva != NULL) { 2967 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 2968 if (dma->dp_pgmap[i].pm_mapped) { 2969 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 2970 MMU_PAGESIZE, HAT_UNLOAD); 2971 dma->dp_pgmap[i].pm_mapped = B_FALSE; 2972 } 2973 } 2974 2975 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 2976 } 2977 2978 #endif 2979 2980 /* if we allocated a copy buffer, free it */ 2981 if (dma->dp_cbaddr != NULL) { 2982 i_ddi_mem_free(dma->dp_cbaddr, 0); 2983 } 2984 } 2985 2986 2987 /* 2988 * rootnex_teardown_windows() 2989 * cleans up after rootnex_setup_windows() 2990 */ 2991 static void 2992 rootnex_teardown_windows(rootnex_dma_t *dma) 2993 { 2994 /* 2995 * if we had to allocate window state on the last bind (because we 2996 * didn't have enough pre-allocated space in the handle), free it. 2997 */ 2998 if (dma->dp_need_to_free_window) { 2999 kmem_free(dma->dp_window, dma->dp_window_size); 3000 } 3001 } 3002 3003 3004 /* 3005 * rootnex_init_win() 3006 * Called in bind slow path during creation of a new window. Initializes 3007 * window state to default values. 3008 */ 3009 /*ARGSUSED*/ 3010 static void 3011 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3012 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3013 { 3014 hp->dmai_nwin++; 3015 window->wd_dosync = B_FALSE; 3016 window->wd_offset = cur_offset; 3017 window->wd_size = 0; 3018 window->wd_first_cookie = cookie; 3019 window->wd_cookie_cnt = 0; 3020 window->wd_trim.tr_trim_first = B_FALSE; 3021 window->wd_trim.tr_trim_last = B_FALSE; 3022 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3023 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3024 #if !defined(__amd64) 3025 window->wd_remap_copybuf = dma->dp_cb_remaping; 3026 #endif 3027 } 3028 3029 3030 /* 3031 * rootnex_setup_cookie() 3032 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3033 * the sgl uses the copy buffer, we need to go through each cookie, figure 3034 * out if it uses the copy buffer, and if it does, save away everything we'll 3035 * need during sync. 3036 */ 3037 static void 3038 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3039 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3040 page_t **cur_pp) 3041 { 3042 boolean_t copybuf_sz_power_2; 3043 rootnex_sglinfo_t *sinfo; 3044 uint_t pidx; 3045 uint_t pcnt; 3046 off_t poff; 3047 #if defined(__amd64) 3048 pfn_t pfn; 3049 #else 3050 page_t **pplist; 3051 #endif 3052 3053 sinfo = &dma->dp_sglinfo; 3054 3055 /* 3056 * Calculate the page index relative to the start of the buffer. The 3057 * index to the current page for our buffer is the offset into the 3058 * first page of the buffer plus our current offset into the buffer 3059 * itself, shifted of course... 3060 */ 3061 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3062 ASSERT(pidx < sinfo->si_max_pages); 3063 3064 /* if this cookie uses the copy buffer */ 3065 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3066 /* 3067 * NOTE: we know that since this cookie uses the copy buffer, it 3068 * is <= MMU_PAGESIZE. 3069 */ 3070 3071 /* 3072 * get the offset into the page. For the 64-bit kernel, get the 3073 * pfn which we'll use with seg kpm. 3074 */ 3075 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3076 #if defined(__amd64) 3077 pfn = cookie->_dmu._dmac_ll >> MMU_PAGESHIFT; 3078 #endif 3079 3080 /* figure out if the copybuf size is a power of 2 */ 3081 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3082 copybuf_sz_power_2 = B_FALSE; 3083 } else { 3084 copybuf_sz_power_2 = B_TRUE; 3085 } 3086 3087 /* This page uses the copy buffer */ 3088 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3089 3090 /* 3091 * save the copy buffer KVA that we'll use with this page. 3092 * if we still fit within the copybuf, it's a simple add. 3093 * otherwise, we need to wrap over using & or % accordingly. 3094 */ 3095 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3096 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3097 *copybuf_used; 3098 } else { 3099 if (copybuf_sz_power_2) { 3100 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3101 (uintptr_t)dma->dp_cbaddr + 3102 (*copybuf_used & 3103 (dma->dp_copybuf_size - 1))); 3104 } else { 3105 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3106 (uintptr_t)dma->dp_cbaddr + 3107 (*copybuf_used % dma->dp_copybuf_size)); 3108 } 3109 } 3110 3111 /* 3112 * over write the cookie physical address with the address of 3113 * the physical address of the copy buffer page that we will 3114 * use. 3115 */ 3116 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3117 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3118 3119 /* if we have a kernel VA, it's easy, just save that address */ 3120 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3121 (sinfo->si_asp == &kas)) { 3122 /* 3123 * save away the page aligned virtual address of the 3124 * driver buffer. Offsets are handled in the sync code. 3125 */ 3126 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3127 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3128 & MMU_PAGEMASK); 3129 #if !defined(__amd64) 3130 /* 3131 * we didn't need to, and will never need to map this 3132 * page. 3133 */ 3134 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3135 #endif 3136 3137 /* we don't have a kernel VA. We need one for the bcopy. */ 3138 } else { 3139 #if defined(__amd64) 3140 /* 3141 * for the 64-bit kernel, it's easy. We use seg kpm to 3142 * get a Kernel VA for the corresponding pfn. 3143 */ 3144 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3145 #else 3146 /* 3147 * for the 32-bit kernel, this is a pain. First we'll 3148 * save away the page_t or user VA for this page. This 3149 * is needed in rootnex_dma_win() when we switch to a 3150 * new window which requires us to re-map the copy 3151 * buffer. 3152 */ 3153 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3154 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3155 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3156 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3157 } else if (pplist != NULL) { 3158 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3159 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3160 } else { 3161 dma->dp_pgmap[pidx].pm_pp = NULL; 3162 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3163 (((uintptr_t) 3164 dmar_object->dmao_obj.virt_obj.v_addr + 3165 cur_offset) & MMU_PAGEMASK); 3166 } 3167 3168 /* 3169 * save away the page aligned virtual address which was 3170 * allocated from the kernel heap arena (taking into 3171 * account if we need more copy buffer than we alloced 3172 * and use multiple windows to handle this, i.e. &,%). 3173 * NOTE: there isn't and physical memory backing up this 3174 * virtual address space currently. 3175 */ 3176 if ((*copybuf_used + MMU_PAGESIZE) <= 3177 dma->dp_copybuf_size) { 3178 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3179 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3180 MMU_PAGEMASK); 3181 } else { 3182 if (copybuf_sz_power_2) { 3183 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3184 (((uintptr_t)dma->dp_kva + 3185 (*copybuf_used & 3186 (dma->dp_copybuf_size - 1))) & 3187 MMU_PAGEMASK); 3188 } else { 3189 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3190 (((uintptr_t)dma->dp_kva + 3191 (*copybuf_used % 3192 dma->dp_copybuf_size)) & 3193 MMU_PAGEMASK); 3194 } 3195 } 3196 3197 /* 3198 * if we haven't used up the available copy buffer yet, 3199 * map the kva to the physical page. 3200 */ 3201 if (!dma->dp_cb_remaping && ((*copybuf_used + 3202 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3203 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3204 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3205 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3206 dma->dp_pgmap[pidx].pm_kaddr); 3207 } else { 3208 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3209 sinfo->si_asp, 3210 dma->dp_pgmap[pidx].pm_kaddr); 3211 } 3212 3213 /* 3214 * we've used up the available copy buffer, this page 3215 * will have to be mapped during rootnex_dma_win() when 3216 * we switch to a new window which requires a re-map 3217 * the copy buffer. (32-bit kernel only) 3218 */ 3219 } else { 3220 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3221 } 3222 #endif 3223 /* go to the next page_t */ 3224 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3225 *cur_pp = (*cur_pp)->p_next; 3226 } 3227 } 3228 3229 /* add to the copy buffer count */ 3230 *copybuf_used += MMU_PAGESIZE; 3231 3232 /* 3233 * This cookie doesn't use the copy buffer. Walk through the pages this 3234 * cookie occupies to reflect this. 3235 */ 3236 } else { 3237 /* 3238 * figure out how many pages the cookie occupies. We need to 3239 * use the original page offset of the buffer and the cookies 3240 * offset in the buffer to do this. 3241 */ 3242 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3243 pcnt = mmu_btopr(cookie->dmac_size + poff); 3244 3245 while (pcnt > 0) { 3246 #if !defined(__amd64) 3247 /* 3248 * the 32-bit kernel doesn't have seg kpm, so we need 3249 * to map in the driver buffer (if it didn't come down 3250 * with a kernel VA) on the fly. Since this page doesn't 3251 * use the copy buffer, it's not, or will it ever, have 3252 * to be mapped in. 3253 */ 3254 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3255 #endif 3256 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3257 3258 /* 3259 * we need to update pidx and cur_pp or we'll loose 3260 * track of where we are. 3261 */ 3262 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3263 *cur_pp = (*cur_pp)->p_next; 3264 } 3265 pidx++; 3266 pcnt--; 3267 } 3268 } 3269 } 3270 3271 3272 /* 3273 * rootnex_sgllen_window_boundary() 3274 * Called in the bind slow path when the next cookie causes us to exceed (in 3275 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3276 * length supported by the DMA H/W. 3277 */ 3278 static int 3279 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3280 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3281 off_t cur_offset) 3282 { 3283 off_t new_offset; 3284 size_t trim_sz; 3285 off_t coffset; 3286 3287 3288 /* 3289 * if we know we'll never have to trim, it's pretty easy. Just move to 3290 * the next window and init it. We're done. 3291 */ 3292 if (!dma->dp_trim_required) { 3293 (*windowp)++; 3294 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3295 (*windowp)->wd_cookie_cnt++; 3296 (*windowp)->wd_size = cookie->dmac_size; 3297 return (DDI_SUCCESS); 3298 } 3299 3300 /* figure out how much we need to trim from the window */ 3301 ASSERT(attr->dma_attr_granular != 0); 3302 if (dma->dp_granularity_power_2) { 3303 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3304 } else { 3305 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3306 } 3307 3308 /* The window's a whole multiple of granularity. We're done */ 3309 if (trim_sz == 0) { 3310 (*windowp)++; 3311 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3312 (*windowp)->wd_cookie_cnt++; 3313 (*windowp)->wd_size = cookie->dmac_size; 3314 return (DDI_SUCCESS); 3315 } 3316 3317 /* 3318 * The window's not a whole multiple of granularity, since we know this 3319 * is due to the sgllen, we need to go back to the last cookie and trim 3320 * that one, add the left over part of the old cookie into the new 3321 * window, and then add in the new cookie into the new window. 3322 */ 3323 3324 /* 3325 * make sure the driver isn't making us do something bad... Trimming and 3326 * sgllen == 1 don't go together. 3327 */ 3328 if (attr->dma_attr_sgllen == 1) { 3329 return (DDI_DMA_NOMAPPING); 3330 } 3331 3332 /* 3333 * first, setup the current window to account for the trim. Need to go 3334 * back to the last cookie for this. 3335 */ 3336 cookie--; 3337 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3338 (*windowp)->wd_trim.tr_last_cookie = cookie; 3339 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3340 ASSERT(cookie->dmac_size > trim_sz); 3341 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3342 (*windowp)->wd_size -= trim_sz; 3343 3344 /* save the buffer offsets for the next window */ 3345 coffset = cookie->dmac_size - trim_sz; 3346 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3347 3348 /* 3349 * set this now in case this is the first window. all other cases are 3350 * set in dma_win() 3351 */ 3352 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3353 3354 /* 3355 * initialize the next window using what's left over in the previous 3356 * cookie. 3357 */ 3358 (*windowp)++; 3359 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3360 (*windowp)->wd_cookie_cnt++; 3361 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3362 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3363 (*windowp)->wd_trim.tr_first_size = trim_sz; 3364 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3365 (*windowp)->wd_dosync = B_TRUE; 3366 } 3367 3368 /* 3369 * now go back to the current cookie and add it to the new window. set 3370 * the new window size to the what was left over from the previous 3371 * cookie and what's in the current cookie. 3372 */ 3373 cookie++; 3374 (*windowp)->wd_cookie_cnt++; 3375 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3376 3377 /* 3378 * trim plus the next cookie could put us over maxxfer (a cookie can be 3379 * a max size of maxxfer). Handle that case. 3380 */ 3381 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3382 /* 3383 * maxxfer is already a whole multiple of granularity, and this 3384 * trim will be <= the previous trim (since a cookie can't be 3385 * larger than maxxfer). Make things simple here. 3386 */ 3387 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3388 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3389 (*windowp)->wd_trim.tr_last_cookie = cookie; 3390 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3391 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3392 (*windowp)->wd_size -= trim_sz; 3393 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3394 3395 /* save the buffer offsets for the next window */ 3396 coffset = cookie->dmac_size - trim_sz; 3397 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3398 3399 /* setup the next window */ 3400 (*windowp)++; 3401 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3402 (*windowp)->wd_cookie_cnt++; 3403 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3404 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3405 coffset; 3406 (*windowp)->wd_trim.tr_first_size = trim_sz; 3407 } 3408 3409 return (DDI_SUCCESS); 3410 } 3411 3412 3413 /* 3414 * rootnex_copybuf_window_boundary() 3415 * Called in bind slowpath when we get to a window boundary because we used 3416 * up all the copy buffer that we have. 3417 */ 3418 static int 3419 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3420 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3421 size_t *copybuf_used) 3422 { 3423 rootnex_sglinfo_t *sinfo; 3424 off_t new_offset; 3425 size_t trim_sz; 3426 off_t coffset; 3427 uint_t pidx; 3428 off_t poff; 3429 3430 3431 sinfo = &dma->dp_sglinfo; 3432 3433 /* 3434 * the copy buffer should be a whole multiple of page size. We know that 3435 * this cookie is <= MMU_PAGESIZE. 3436 */ 3437 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3438 3439 /* 3440 * from now on, all new windows in this bind need to be re-mapped during 3441 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3442 * space... 3443 */ 3444 #if !defined(__amd64) 3445 dma->dp_cb_remaping = B_TRUE; 3446 #endif 3447 3448 /* reset copybuf used */ 3449 *copybuf_used = 0; 3450 3451 /* 3452 * if we don't have to trim (since granularity is set to 1), go to the 3453 * next window and add the current cookie to it. We know the current 3454 * cookie uses the copy buffer since we're in this code path. 3455 */ 3456 if (!dma->dp_trim_required) { 3457 (*windowp)++; 3458 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3459 3460 /* Add this cookie to the new window */ 3461 (*windowp)->wd_cookie_cnt++; 3462 (*windowp)->wd_size += cookie->dmac_size; 3463 *copybuf_used += MMU_PAGESIZE; 3464 return (DDI_SUCCESS); 3465 } 3466 3467 /* 3468 * *** may need to trim, figure it out. 3469 */ 3470 3471 /* figure out how much we need to trim from the window */ 3472 if (dma->dp_granularity_power_2) { 3473 trim_sz = (*windowp)->wd_size & 3474 (hp->dmai_attr.dma_attr_granular - 1); 3475 } else { 3476 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3477 } 3478 3479 /* 3480 * if the window's a whole multiple of granularity, go to the next 3481 * window, init it, then add in the current cookie. We know the current 3482 * cookie uses the copy buffer since we're in this code path. 3483 */ 3484 if (trim_sz == 0) { 3485 (*windowp)++; 3486 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3487 3488 /* Add this cookie to the new window */ 3489 (*windowp)->wd_cookie_cnt++; 3490 (*windowp)->wd_size += cookie->dmac_size; 3491 *copybuf_used += MMU_PAGESIZE; 3492 return (DDI_SUCCESS); 3493 } 3494 3495 /* 3496 * *** We figured it out, we definitly need to trim 3497 */ 3498 3499 /* 3500 * make sure the driver isn't making us do something bad... 3501 * Trimming and sgllen == 1 don't go together. 3502 */ 3503 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3504 return (DDI_DMA_NOMAPPING); 3505 } 3506 3507 /* 3508 * first, setup the current window to account for the trim. Need to go 3509 * back to the last cookie for this. Some of the last cookie will be in 3510 * the current window, and some of the last cookie will be in the new 3511 * window. All of the current cookie will be in the new window. 3512 */ 3513 cookie--; 3514 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3515 (*windowp)->wd_trim.tr_last_cookie = cookie; 3516 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3517 ASSERT(cookie->dmac_size > trim_sz); 3518 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3519 (*windowp)->wd_size -= trim_sz; 3520 3521 /* 3522 * we're trimming the last cookie (not the current cookie). So that 3523 * last cookie may have or may not have been using the copy buffer ( 3524 * we know the cookie passed in uses the copy buffer since we're in 3525 * this code path). 3526 * 3527 * If the last cookie doesn't use the copy buffer, nothing special to 3528 * do. However, if it does uses the copy buffer, it will be both the 3529 * last page in the current window and the first page in the next 3530 * window. Since we are reusing the copy buffer (and KVA space on the 3531 * 32-bit kernel), this page will use the end of the copy buffer in the 3532 * current window, and the start of the copy buffer in the next window. 3533 * Track that info... The cookie physical address was already set to 3534 * the copy buffer physical address in setup_cookie.. 3535 */ 3536 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3537 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3538 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3539 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3540 (*windowp)->wd_trim.tr_last_pidx = pidx; 3541 (*windowp)->wd_trim.tr_last_cbaddr = 3542 dma->dp_pgmap[pidx].pm_cbaddr; 3543 #if !defined(__amd64) 3544 (*windowp)->wd_trim.tr_last_kaddr = 3545 dma->dp_pgmap[pidx].pm_kaddr; 3546 #endif 3547 } 3548 3549 /* save the buffer offsets for the next window */ 3550 coffset = cookie->dmac_size - trim_sz; 3551 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3552 3553 /* 3554 * set this now in case this is the first window. all other cases are 3555 * set in dma_win() 3556 */ 3557 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3558 3559 /* 3560 * initialize the next window using what's left over in the previous 3561 * cookie. 3562 */ 3563 (*windowp)++; 3564 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3565 (*windowp)->wd_cookie_cnt++; 3566 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3567 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3568 (*windowp)->wd_trim.tr_first_size = trim_sz; 3569 3570 /* 3571 * again, we're tracking if the last cookie uses the copy buffer. 3572 * read the comment above for more info on why we need to track 3573 * additional state. 3574 * 3575 * For the first cookie in the new window, we need reset the physical 3576 * address to DMA into to the start of the copy buffer plus any 3577 * initial page offset which may be present. 3578 */ 3579 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3580 (*windowp)->wd_dosync = B_TRUE; 3581 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3582 (*windowp)->wd_trim.tr_first_pidx = pidx; 3583 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3584 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3585 (*windowp)->wd_trim.tr_first_paddr = ptob64(hat_getpfnum( 3586 kas.a_hat, dma->dp_cbaddr)) + poff; 3587 #if !defined(__amd64) 3588 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3589 #endif 3590 /* account for the cookie copybuf usage in the new window */ 3591 *copybuf_used += MMU_PAGESIZE; 3592 3593 /* 3594 * every piece of code has to have a hack, and here is this 3595 * ones :-) 3596 * 3597 * There is a complex interaction between setup_cookie and the 3598 * copybuf window boundary. The complexity had to be in either 3599 * the maxxfer window, or the copybuf window, and I chose the 3600 * copybuf code. 3601 * 3602 * So in this code path, we have taken the last cookie, 3603 * virtually broken it in half due to the trim, and it happens 3604 * to use the copybuf which further complicates life. At the 3605 * same time, we have already setup the current cookie, which 3606 * is now wrong. More background info: the current cookie uses 3607 * the copybuf, so it is only a page long max. So we need to 3608 * fix the current cookies copy buffer address, physical 3609 * address, and kva for the 32-bit kernel. We due this by 3610 * bumping them by page size (of course, we can't due this on 3611 * the physical address since the copy buffer may not be 3612 * physically contiguous). 3613 */ 3614 cookie++; 3615 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3616 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3617 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3618 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3619 #if !defined(__amd64) 3620 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3621 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3622 #endif 3623 } else { 3624 /* go back to the current cookie */ 3625 cookie++; 3626 } 3627 3628 /* 3629 * add the current cookie to the new window. set the new window size to 3630 * the what was left over from the previous cookie and what's in the 3631 * current cookie. 3632 */ 3633 (*windowp)->wd_cookie_cnt++; 3634 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3635 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3636 3637 /* 3638 * we know that the cookie passed in always uses the copy buffer. We 3639 * wouldn't be here if it didn't. 3640 */ 3641 *copybuf_used += MMU_PAGESIZE; 3642 3643 return (DDI_SUCCESS); 3644 } 3645 3646 3647 /* 3648 * rootnex_maxxfer_window_boundary() 3649 * Called in bind slowpath when we get to a window boundary because we will 3650 * go over maxxfer. 3651 */ 3652 static int 3653 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3654 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3655 { 3656 size_t dmac_size; 3657 off_t new_offset; 3658 size_t trim_sz; 3659 off_t coffset; 3660 3661 3662 /* 3663 * calculate how much we have to trim off of the current cookie to equal 3664 * maxxfer. We don't have to account for granularity here since our 3665 * maxxfer already takes that into account. 3666 */ 3667 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3668 ASSERT(trim_sz <= cookie->dmac_size); 3669 ASSERT(trim_sz <= dma->dp_maxxfer); 3670 3671 /* save cookie size since we need it later and we might change it */ 3672 dmac_size = cookie->dmac_size; 3673 3674 /* 3675 * if we're not trimming the entire cookie, setup the current window to 3676 * account for the trim. 3677 */ 3678 if (trim_sz < cookie->dmac_size) { 3679 (*windowp)->wd_cookie_cnt++; 3680 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3681 (*windowp)->wd_trim.tr_last_cookie = cookie; 3682 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3683 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3684 (*windowp)->wd_size = dma->dp_maxxfer; 3685 3686 /* 3687 * set the adjusted cookie size now in case this is the first 3688 * window. All other windows are taken care of in get win 3689 */ 3690 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3691 } 3692 3693 /* 3694 * coffset is the current offset within the cookie, new_offset is the 3695 * current offset with the entire buffer. 3696 */ 3697 coffset = dmac_size - trim_sz; 3698 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3699 3700 /* initialize the next window */ 3701 (*windowp)++; 3702 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3703 (*windowp)->wd_cookie_cnt++; 3704 (*windowp)->wd_size = trim_sz; 3705 if (trim_sz < dmac_size) { 3706 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3707 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3708 coffset; 3709 (*windowp)->wd_trim.tr_first_size = trim_sz; 3710 } 3711 3712 return (DDI_SUCCESS); 3713 } 3714 3715 3716 /* 3717 * rootnex_dma_sync() 3718 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3719 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3720 * is set, ddi_dma_sync() returns immediately passing back success. 3721 */ 3722 /*ARGSUSED*/ 3723 static int 3724 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3725 off_t off, size_t len, uint_t cache_flags) 3726 { 3727 rootnex_sglinfo_t *sinfo; 3728 rootnex_pgmap_t *cbpage; 3729 rootnex_window_t *win; 3730 ddi_dma_impl_t *hp; 3731 rootnex_dma_t *dma; 3732 caddr_t fromaddr; 3733 caddr_t toaddr; 3734 uint_t psize; 3735 off_t offset; 3736 uint_t pidx; 3737 size_t size; 3738 off_t poff; 3739 int e; 3740 3741 3742 hp = (ddi_dma_impl_t *)handle; 3743 dma = (rootnex_dma_t *)hp->dmai_private; 3744 sinfo = &dma->dp_sglinfo; 3745 3746 /* 3747 * if we don't have any windows, we don't need to sync. A copybuf 3748 * will cause us to have at least one window. 3749 */ 3750 if (dma->dp_window == NULL) { 3751 return (DDI_SUCCESS); 3752 } 3753 3754 /* This window may not need to be sync'd */ 3755 win = &dma->dp_window[dma->dp_current_win]; 3756 if (!win->wd_dosync) { 3757 return (DDI_SUCCESS); 3758 } 3759 3760 /* handle off and len special cases */ 3761 if ((off == 0) || (rootnex_sync_ignore_params)) { 3762 offset = win->wd_offset; 3763 } else { 3764 offset = off; 3765 } 3766 if ((len == 0) || (rootnex_sync_ignore_params)) { 3767 size = win->wd_size; 3768 } else { 3769 size = len; 3770 } 3771 3772 /* check the sync args to make sure they make a little sense */ 3773 if (rootnex_sync_check_parms) { 3774 e = rootnex_valid_sync_parms(hp, win, offset, size, 3775 cache_flags); 3776 if (e != DDI_SUCCESS) { 3777 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3778 return (DDI_FAILURE); 3779 } 3780 } 3781 3782 /* 3783 * special case the first page to handle the offset into the page. The 3784 * offset to the current page for our buffer is the offset into the 3785 * first page of the buffer plus our current offset into the buffer 3786 * itself, masked of course. 3787 */ 3788 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3789 psize = MIN((MMU_PAGESIZE - poff), size); 3790 3791 /* go through all the pages that we want to sync */ 3792 while (size > 0) { 3793 /* 3794 * Calculate the page index relative to the start of the buffer. 3795 * The index to the current page for our buffer is the offset 3796 * into the first page of the buffer plus our current offset 3797 * into the buffer itself, shifted of course... 3798 */ 3799 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3800 ASSERT(pidx < sinfo->si_max_pages); 3801 3802 /* 3803 * if this page uses the copy buffer, we need to sync it, 3804 * otherwise, go on to the next page. 3805 */ 3806 cbpage = &dma->dp_pgmap[pidx]; 3807 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3808 (cbpage->pm_uses_copybuf == B_FALSE)); 3809 if (cbpage->pm_uses_copybuf) { 3810 /* cbaddr and kaddr should be page aligned */ 3811 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3812 MMU_PAGEOFFSET) == 0); 3813 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3814 MMU_PAGEOFFSET) == 0); 3815 3816 /* 3817 * if we're copying for the device, we are going to 3818 * copy from the drivers buffer and to the rootnex 3819 * allocated copy buffer. 3820 */ 3821 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3822 fromaddr = cbpage->pm_kaddr + poff; 3823 toaddr = cbpage->pm_cbaddr + poff; 3824 DTRACE_PROBE2(rootnex__sync__dev, 3825 dev_info_t *, dma->dp_dip, size_t, psize); 3826 3827 /* 3828 * if we're copying for the cpu/kernel, we are going to 3829 * copy from the rootnex allocated copy buffer to the 3830 * drivers buffer. 3831 */ 3832 } else { 3833 fromaddr = cbpage->pm_cbaddr + poff; 3834 toaddr = cbpage->pm_kaddr + poff; 3835 DTRACE_PROBE2(rootnex__sync__cpu, 3836 dev_info_t *, dma->dp_dip, size_t, psize); 3837 } 3838 3839 bcopy(fromaddr, toaddr, psize); 3840 } 3841 3842 /* 3843 * decrement size until we're done, update our offset into the 3844 * buffer, and get the next page size. 3845 */ 3846 size -= psize; 3847 offset += psize; 3848 psize = MIN(MMU_PAGESIZE, size); 3849 3850 /* page offset is zero for the rest of this loop */ 3851 poff = 0; 3852 } 3853 3854 return (DDI_SUCCESS); 3855 } 3856 3857 3858 /* 3859 * rootnex_valid_sync_parms() 3860 * checks the parameters passed to sync to verify they are correct. 3861 */ 3862 static int 3863 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 3864 off_t offset, size_t size, uint_t cache_flags) 3865 { 3866 off_t woffset; 3867 3868 3869 /* 3870 * the first part of the test to make sure the offset passed in is 3871 * within the window. 3872 */ 3873 if (offset < win->wd_offset) { 3874 return (DDI_FAILURE); 3875 } 3876 3877 /* 3878 * second and last part of the test to make sure the offset and length 3879 * passed in is within the window. 3880 */ 3881 woffset = offset - win->wd_offset; 3882 if ((woffset + size) > win->wd_size) { 3883 return (DDI_FAILURE); 3884 } 3885 3886 /* 3887 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 3888 * be set too. 3889 */ 3890 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 3891 (hp->dmai_rflags & DDI_DMA_WRITE)) { 3892 return (DDI_SUCCESS); 3893 } 3894 3895 /* 3896 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 3897 * should be set. Also DDI_DMA_READ should be set in the flags. 3898 */ 3899 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 3900 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 3901 (hp->dmai_rflags & DDI_DMA_READ)) { 3902 return (DDI_SUCCESS); 3903 } 3904 3905 return (DDI_FAILURE); 3906 } 3907 3908 3909 /* 3910 * rootnex_dma_win() 3911 * called from ddi_dma_getwin() 3912 */ 3913 /*ARGSUSED*/ 3914 static int 3915 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3916 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 3917 uint_t *ccountp) 3918 { 3919 rootnex_window_t *window; 3920 rootnex_trim_t *trim; 3921 ddi_dma_impl_t *hp; 3922 rootnex_dma_t *dma; 3923 #if !defined(__amd64) 3924 rootnex_sglinfo_t *sinfo; 3925 rootnex_pgmap_t *pmap; 3926 uint_t pidx; 3927 uint_t pcnt; 3928 off_t poff; 3929 int i; 3930 #endif 3931 3932 3933 hp = (ddi_dma_impl_t *)handle; 3934 dma = (rootnex_dma_t *)hp->dmai_private; 3935 #if !defined(__amd64) 3936 sinfo = &dma->dp_sglinfo; 3937 #endif 3938 3939 /* If we try and get a window which doesn't exist, return failure */ 3940 if (win >= hp->dmai_nwin) { 3941 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3942 return (DDI_FAILURE); 3943 } 3944 3945 /* 3946 * if we don't have any windows, and they're asking for the first 3947 * window, setup the cookie pointer to the first cookie in the bind. 3948 * setup our return values, then increment the cookie since we return 3949 * the first cookie on the stack. 3950 */ 3951 if (dma->dp_window == NULL) { 3952 if (win != 0) { 3953 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3954 return (DDI_FAILURE); 3955 } 3956 hp->dmai_cookie = dma->dp_cookies; 3957 *offp = 0; 3958 *lenp = dma->dp_dma.dmao_size; 3959 *ccountp = dma->dp_sglinfo.si_sgl_size; 3960 *cookiep = hp->dmai_cookie[0]; 3961 hp->dmai_cookie++; 3962 return (DDI_SUCCESS); 3963 } 3964 3965 /* sync the old window before moving on to the new one */ 3966 window = &dma->dp_window[dma->dp_current_win]; 3967 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 3968 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 3969 DDI_DMA_SYNC_FORCPU); 3970 } 3971 3972 #if !defined(__amd64) 3973 /* 3974 * before we move to the next window, if we need to re-map, unmap all 3975 * the pages in this window. 3976 */ 3977 if (dma->dp_cb_remaping) { 3978 /* 3979 * If we switch to this window again, we'll need to map in 3980 * on the fly next time. 3981 */ 3982 window->wd_remap_copybuf = B_TRUE; 3983 3984 /* 3985 * calculate the page index into the buffer where this window 3986 * starts, and the number of pages this window takes up. 3987 */ 3988 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 3989 MMU_PAGESHIFT; 3990 poff = (sinfo->si_buf_offset + window->wd_offset) & 3991 MMU_PAGEOFFSET; 3992 pcnt = mmu_btopr(window->wd_size + poff); 3993 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 3994 3995 /* unmap pages which are currently mapped in this window */ 3996 for (i = 0; i < pcnt; i++) { 3997 if (dma->dp_pgmap[pidx].pm_mapped) { 3998 hat_unload(kas.a_hat, 3999 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4000 HAT_UNLOAD); 4001 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4002 } 4003 pidx++; 4004 } 4005 } 4006 #endif 4007 4008 /* 4009 * Move to the new window. 4010 * NOTE: current_win must be set for sync to work right 4011 */ 4012 dma->dp_current_win = win; 4013 window = &dma->dp_window[win]; 4014 4015 /* if needed, adjust the first and/or last cookies for trim */ 4016 trim = &window->wd_trim; 4017 if (trim->tr_trim_first) { 4018 window->wd_first_cookie->_dmu._dmac_ll = trim->tr_first_paddr; 4019 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4020 #if !defined(__amd64) 4021 window->wd_first_cookie->dmac_type = 4022 (window->wd_first_cookie->dmac_type & 4023 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4024 #endif 4025 if (trim->tr_first_copybuf_win) { 4026 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4027 trim->tr_first_cbaddr; 4028 #if !defined(__amd64) 4029 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4030 trim->tr_first_kaddr; 4031 #endif 4032 } 4033 } 4034 if (trim->tr_trim_last) { 4035 trim->tr_last_cookie->_dmu._dmac_ll = trim->tr_last_paddr; 4036 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4037 if (trim->tr_last_copybuf_win) { 4038 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4039 trim->tr_last_cbaddr; 4040 #if !defined(__amd64) 4041 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4042 trim->tr_last_kaddr; 4043 #endif 4044 } 4045 } 4046 4047 /* 4048 * setup the cookie pointer to the first cookie in the window. setup 4049 * our return values, then increment the cookie since we return the 4050 * first cookie on the stack. 4051 */ 4052 hp->dmai_cookie = window->wd_first_cookie; 4053 *offp = window->wd_offset; 4054 *lenp = window->wd_size; 4055 *ccountp = window->wd_cookie_cnt; 4056 *cookiep = hp->dmai_cookie[0]; 4057 hp->dmai_cookie++; 4058 4059 #if !defined(__amd64) 4060 /* re-map copybuf if required for this window */ 4061 if (dma->dp_cb_remaping) { 4062 /* 4063 * calculate the page index into the buffer where this 4064 * window starts. 4065 */ 4066 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4067 MMU_PAGESHIFT; 4068 ASSERT(pidx < sinfo->si_max_pages); 4069 4070 /* 4071 * the first page can get unmapped if it's shared with the 4072 * previous window. Even if the rest of this window is already 4073 * mapped in, we need to still check this one. 4074 */ 4075 pmap = &dma->dp_pgmap[pidx]; 4076 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4077 if (pmap->pm_pp != NULL) { 4078 pmap->pm_mapped = B_TRUE; 4079 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4080 } else if (pmap->pm_vaddr != NULL) { 4081 pmap->pm_mapped = B_TRUE; 4082 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4083 pmap->pm_kaddr); 4084 } 4085 } 4086 pidx++; 4087 4088 /* map in the rest of the pages if required */ 4089 if (window->wd_remap_copybuf) { 4090 window->wd_remap_copybuf = B_FALSE; 4091 4092 /* figure out many pages this window takes up */ 4093 poff = (sinfo->si_buf_offset + window->wd_offset) & 4094 MMU_PAGEOFFSET; 4095 pcnt = mmu_btopr(window->wd_size + poff); 4096 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4097 4098 /* map pages which require it */ 4099 for (i = 1; i < pcnt; i++) { 4100 pmap = &dma->dp_pgmap[pidx]; 4101 if (pmap->pm_uses_copybuf) { 4102 ASSERT(pmap->pm_mapped == B_FALSE); 4103 if (pmap->pm_pp != NULL) { 4104 pmap->pm_mapped = B_TRUE; 4105 i86_pp_map(pmap->pm_pp, 4106 pmap->pm_kaddr); 4107 } else if (pmap->pm_vaddr != NULL) { 4108 pmap->pm_mapped = B_TRUE; 4109 i86_va_map(pmap->pm_vaddr, 4110 sinfo->si_asp, 4111 pmap->pm_kaddr); 4112 } 4113 } 4114 pidx++; 4115 } 4116 } 4117 } 4118 #endif 4119 4120 /* if the new window uses the copy buffer, sync it for the device */ 4121 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4122 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4123 DDI_DMA_SYNC_FORDEV); 4124 } 4125 4126 return (DDI_SUCCESS); 4127 } 4128 4129 4130 4131 /* 4132 * ************************ 4133 * obsoleted dma routines 4134 * ************************ 4135 */ 4136 4137 /* 4138 * rootnex_dma_map() 4139 * called from ddi_dma_setup() 4140 */ 4141 /* ARGSUSED */ 4142 static int 4143 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4144 ddi_dma_handle_t *handlep) 4145 { 4146 #if defined(__amd64) 4147 /* 4148 * this interface is not supported in 64-bit x86 kernel. See comment in 4149 * rootnex_dma_mctl() 4150 */ 4151 ASSERT(0); 4152 return (DDI_DMA_NORESOURCES); 4153 4154 #else /* 32-bit x86 kernel */ 4155 ddi_dma_handle_t *lhandlep; 4156 ddi_dma_handle_t lhandle; 4157 ddi_dma_cookie_t cookie; 4158 ddi_dma_attr_t dma_attr; 4159 ddi_dma_lim_t *dma_lim; 4160 uint_t ccnt; 4161 int e; 4162 4163 4164 /* 4165 * if the driver is just testing to see if it's possible to do the bind, 4166 * we'll use local state. Otherwise, use the handle pointer passed in. 4167 */ 4168 if (handlep == NULL) { 4169 lhandlep = &lhandle; 4170 } else { 4171 lhandlep = handlep; 4172 } 4173 4174 /* convert the limit structure to a dma_attr one */ 4175 dma_lim = dmareq->dmar_limits; 4176 dma_attr.dma_attr_version = DMA_ATTR_V0; 4177 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4178 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4179 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4180 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4181 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4182 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4183 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4184 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4185 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4186 dma_attr.dma_attr_align = MMU_PAGESIZE; 4187 dma_attr.dma_attr_flags = 0; 4188 4189 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4190 dmareq->dmar_arg, lhandlep); 4191 if (e != DDI_SUCCESS) { 4192 return (e); 4193 } 4194 4195 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4196 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4197 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4198 return (e); 4199 } 4200 4201 /* 4202 * if the driver is just testing to see if it's possible to do the bind, 4203 * free up the local state and return the result. 4204 */ 4205 if (handlep == NULL) { 4206 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4207 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4208 if (e == DDI_DMA_MAPPED) { 4209 return (DDI_DMA_MAPOK); 4210 } else { 4211 return (DDI_DMA_NOMAPPING); 4212 } 4213 } 4214 4215 return (e); 4216 #endif /* defined(__amd64) */ 4217 } 4218 4219 4220 /* 4221 * rootnex_dma_mctl() 4222 * 4223 */ 4224 /* ARGSUSED */ 4225 static int 4226 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4227 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4228 uint_t cache_flags) 4229 { 4230 #if defined(__amd64) 4231 /* 4232 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4233 * common implementation in genunix, so they no longer have x86 4234 * specific functionality which called into dma_ctl. 4235 * 4236 * The rest of the obsoleted interfaces were never supported in the 4237 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4238 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4239 * implementation issues. 4240 * 4241 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4242 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4243 * reflect that now too... 4244 * 4245 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4246 * not going to put this functionality into the 64-bit x86 kernel now. 4247 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4248 * that in a future release. 4249 */ 4250 ASSERT(0); 4251 return (DDI_FAILURE); 4252 4253 #else /* 32-bit x86 kernel */ 4254 ddi_dma_cookie_t lcookie; 4255 ddi_dma_cookie_t *cookie; 4256 rootnex_window_t *window; 4257 ddi_dma_impl_t *hp; 4258 rootnex_dma_t *dma; 4259 uint_t nwin; 4260 uint_t ccnt; 4261 size_t len; 4262 off_t off; 4263 int e; 4264 4265 4266 /* 4267 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4268 * hacky since were optimizing for the current interfaces and so we can 4269 * cleanup the mess in genunix. Hopefully we will remove the this 4270 * obsoleted routines someday soon. 4271 */ 4272 4273 switch (request) { 4274 4275 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4276 hp = (ddi_dma_impl_t *)handle; 4277 cookie = (ddi_dma_cookie_t *)objpp; 4278 4279 /* 4280 * convert segment to cookie. We don't distinguish between the 4281 * two :-) 4282 */ 4283 *cookie = *hp->dmai_cookie; 4284 *lenp = cookie->dmac_size; 4285 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4286 return (DDI_SUCCESS); 4287 4288 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4289 hp = (ddi_dma_impl_t *)handle; 4290 dma = (rootnex_dma_t *)hp->dmai_private; 4291 4292 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4293 return (DDI_DMA_STALE); 4294 } 4295 4296 /* handle the case where we don't have any windows */ 4297 if (dma->dp_window == NULL) { 4298 /* 4299 * if seg == NULL, and we don't have any windows, 4300 * return the first cookie in the sgl. 4301 */ 4302 if (*lenp == NULL) { 4303 dma->dp_current_cookie = 0; 4304 hp->dmai_cookie = dma->dp_cookies; 4305 *objpp = (caddr_t)handle; 4306 return (DDI_SUCCESS); 4307 4308 /* if we have more cookies, go to the next cookie */ 4309 } else { 4310 if ((dma->dp_current_cookie + 1) >= 4311 dma->dp_sglinfo.si_sgl_size) { 4312 return (DDI_DMA_DONE); 4313 } 4314 dma->dp_current_cookie++; 4315 hp->dmai_cookie++; 4316 return (DDI_SUCCESS); 4317 } 4318 } 4319 4320 /* We have one or more windows */ 4321 window = &dma->dp_window[dma->dp_current_win]; 4322 4323 /* 4324 * if seg == NULL, return the first cookie in the current 4325 * window 4326 */ 4327 if (*lenp == NULL) { 4328 dma->dp_current_cookie = 0; 4329 hp->dmai_cookie = window->wd_first_cookie; 4330 4331 /* 4332 * go to the next cookie in the window then see if we done with 4333 * this window. 4334 */ 4335 } else { 4336 if ((dma->dp_current_cookie + 1) >= 4337 window->wd_cookie_cnt) { 4338 return (DDI_DMA_DONE); 4339 } 4340 dma->dp_current_cookie++; 4341 hp->dmai_cookie++; 4342 } 4343 *objpp = (caddr_t)handle; 4344 return (DDI_SUCCESS); 4345 4346 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4347 hp = (ddi_dma_impl_t *)handle; 4348 dma = (rootnex_dma_t *)hp->dmai_private; 4349 4350 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4351 return (DDI_DMA_STALE); 4352 } 4353 4354 /* if win == NULL, return the first window in the bind */ 4355 if (*offp == NULL) { 4356 nwin = 0; 4357 4358 /* 4359 * else, go to the next window then see if we're done with all 4360 * the windows. 4361 */ 4362 } else { 4363 nwin = dma->dp_current_win + 1; 4364 if (nwin >= hp->dmai_nwin) { 4365 return (DDI_DMA_DONE); 4366 } 4367 } 4368 4369 /* switch to the next window */ 4370 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4371 &lcookie, &ccnt); 4372 ASSERT(e == DDI_SUCCESS); 4373 if (e != DDI_SUCCESS) { 4374 return (DDI_DMA_STALE); 4375 } 4376 4377 /* reset the cookie back to the first cookie in the window */ 4378 if (dma->dp_window != NULL) { 4379 window = &dma->dp_window[dma->dp_current_win]; 4380 hp->dmai_cookie = window->wd_first_cookie; 4381 } else { 4382 hp->dmai_cookie = dma->dp_cookies; 4383 } 4384 4385 *objpp = (caddr_t)handle; 4386 return (DDI_SUCCESS); 4387 4388 case DDI_DMA_FREE: /* ddi_dma_free() */ 4389 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4390 (void) rootnex_dma_freehdl(dip, rdip, handle); 4391 if (rootnex_state->r_dvma_call_list_id) { 4392 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4393 } 4394 return (DDI_SUCCESS); 4395 4396 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4397 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4398 /* should never get here, handled in genunix */ 4399 ASSERT(0); 4400 return (DDI_FAILURE); 4401 4402 case DDI_DMA_KVADDR: 4403 case DDI_DMA_GETERR: 4404 case DDI_DMA_COFF: 4405 return (DDI_FAILURE); 4406 } 4407 4408 return (DDI_FAILURE); 4409 #endif /* defined(__amd64) */ 4410 } 4411