1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2006 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * x86 root nexus driver 31 */ 32 33 #include <sys/sysmacros.h> 34 #include <sys/conf.h> 35 #include <sys/autoconf.h> 36 #include <sys/sysmacros.h> 37 #include <sys/debug.h> 38 #include <sys/psw.h> 39 #include <sys/ddidmareq.h> 40 #include <sys/promif.h> 41 #include <sys/devops.h> 42 #include <sys/kmem.h> 43 #include <sys/cmn_err.h> 44 #include <vm/seg.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_dev.h> 47 #include <sys/vmem.h> 48 #include <sys/mman.h> 49 #include <vm/hat.h> 50 #include <vm/as.h> 51 #include <vm/page.h> 52 #include <sys/avintr.h> 53 #include <sys/errno.h> 54 #include <sys/modctl.h> 55 #include <sys/ddi_impldefs.h> 56 #include <sys/sunddi.h> 57 #include <sys/sunndi.h> 58 #include <sys/mach_intr.h> 59 #include <sys/psm.h> 60 #include <sys/ontrap.h> 61 #include <sys/atomic.h> 62 #include <sys/sdt.h> 63 #include <sys/rootnex.h> 64 #include <vm/hat_i86.h> 65 66 /* 67 * enable/disable extra checking of function parameters. Useful for debugging 68 * drivers. 69 */ 70 #ifdef DEBUG 71 int rootnex_alloc_check_parms = 1; 72 int rootnex_bind_check_parms = 1; 73 int rootnex_bind_check_inuse = 1; 74 int rootnex_unbind_verify_buffer = 0; 75 int rootnex_sync_check_parms = 1; 76 #else 77 int rootnex_alloc_check_parms = 0; 78 int rootnex_bind_check_parms = 0; 79 int rootnex_bind_check_inuse = 0; 80 int rootnex_unbind_verify_buffer = 0; 81 int rootnex_sync_check_parms = 0; 82 #endif 83 84 /* Master Abort and Target Abort panic flag */ 85 int rootnex_fm_ma_ta_panic_flag = 0; 86 87 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 88 int rootnex_bind_fail = 1; 89 int rootnex_bind_warn = 1; 90 uint8_t *rootnex_warn_list; 91 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 92 #define ROOTNEX_BIND_WARNING (0x1 << 0) 93 94 /* 95 * revert back to old broken behavior of always sync'ing entire copy buffer. 96 * This is useful if be have a buggy driver which doesn't correctly pass in 97 * the offset and size into ddi_dma_sync(). 98 */ 99 int rootnex_sync_ignore_params = 0; 100 101 /* 102 * maximum size that we will allow for a copy buffer. Can be patched on the 103 * fly 104 */ 105 size_t rootnex_max_copybuf_size = 0x100000; 106 107 /* 108 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 109 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 110 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 111 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 112 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 113 * (< 8K). We will still need to allocate the copy buffer during bind though 114 * (if we need one). These can only be modified in /etc/system before rootnex 115 * attach. 116 */ 117 #if defined(__amd64) 118 int rootnex_prealloc_cookies = 65; 119 int rootnex_prealloc_windows = 4; 120 int rootnex_prealloc_copybuf = 2; 121 #else 122 int rootnex_prealloc_cookies = 33; 123 int rootnex_prealloc_windows = 4; 124 int rootnex_prealloc_copybuf = 2; 125 #endif 126 127 /* driver global state */ 128 static rootnex_state_t *rootnex_state; 129 130 /* shortcut to rootnex counters */ 131 static uint64_t *rootnex_cnt; 132 133 /* 134 * XXX - does x86 even need these or are they left over from the SPARC days? 135 */ 136 /* statically defined integer/boolean properties for the root node */ 137 static rootnex_intprop_t rootnex_intprp[] = { 138 { "PAGESIZE", PAGESIZE }, 139 { "MMU_PAGESIZE", MMU_PAGESIZE }, 140 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 141 { DDI_RELATIVE_ADDRESSING, 1 }, 142 }; 143 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 144 145 146 static struct cb_ops rootnex_cb_ops = { 147 nodev, /* open */ 148 nodev, /* close */ 149 nodev, /* strategy */ 150 nodev, /* print */ 151 nodev, /* dump */ 152 nodev, /* read */ 153 nodev, /* write */ 154 nodev, /* ioctl */ 155 nodev, /* devmap */ 156 nodev, /* mmap */ 157 nodev, /* segmap */ 158 nochpoll, /* chpoll */ 159 ddi_prop_op, /* cb_prop_op */ 160 NULL, /* struct streamtab */ 161 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 162 CB_REV, /* Rev */ 163 nodev, /* cb_aread */ 164 nodev /* cb_awrite */ 165 }; 166 167 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 168 off_t offset, off_t len, caddr_t *vaddrp); 169 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 170 struct hat *hat, struct seg *seg, caddr_t addr, 171 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 172 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 173 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 174 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 175 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 176 ddi_dma_handle_t *handlep); 177 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 178 ddi_dma_handle_t handle); 179 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 180 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 181 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 182 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 183 ddi_dma_handle_t handle); 184 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 185 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 186 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 187 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 188 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 189 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 190 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 191 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 192 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 193 ddi_ctl_enum_t ctlop, void *arg, void *result); 194 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 195 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 196 197 198 static struct bus_ops rootnex_bus_ops = { 199 BUSO_REV, 200 rootnex_map, 201 NULL, 202 NULL, 203 NULL, 204 rootnex_map_fault, 205 rootnex_dma_map, 206 rootnex_dma_allochdl, 207 rootnex_dma_freehdl, 208 rootnex_dma_bindhdl, 209 rootnex_dma_unbindhdl, 210 rootnex_dma_sync, 211 rootnex_dma_win, 212 rootnex_dma_mctl, 213 rootnex_ctlops, 214 ddi_bus_prop_op, 215 i_ddi_rootnex_get_eventcookie, 216 i_ddi_rootnex_add_eventcall, 217 i_ddi_rootnex_remove_eventcall, 218 i_ddi_rootnex_post_event, 219 0, /* bus_intr_ctl */ 220 0, /* bus_config */ 221 0, /* bus_unconfig */ 222 NULL, /* bus_fm_init */ 223 NULL, /* bus_fm_fini */ 224 NULL, /* bus_fm_access_enter */ 225 NULL, /* bus_fm_access_exit */ 226 NULL, /* bus_powr */ 227 rootnex_intr_ops /* bus_intr_op */ 228 }; 229 230 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 231 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 232 233 static struct dev_ops rootnex_ops = { 234 DEVO_REV, 235 0, 236 ddi_no_info, 237 nulldev, 238 nulldev, 239 rootnex_attach, 240 rootnex_detach, 241 nulldev, 242 &rootnex_cb_ops, 243 &rootnex_bus_ops 244 }; 245 246 static struct modldrv rootnex_modldrv = { 247 &mod_driverops, 248 "i86pc root nexus %I%", 249 &rootnex_ops 250 }; 251 252 static struct modlinkage rootnex_modlinkage = { 253 MODREV_1, 254 (void *)&rootnex_modldrv, 255 NULL 256 }; 257 258 259 /* 260 * extern hacks 261 */ 262 extern struct seg_ops segdev_ops; 263 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 264 #ifdef DDI_MAP_DEBUG 265 extern int ddi_map_debug_flag; 266 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 267 #endif 268 #define ptob64(x) (((uint64_t)(x)) << MMU_PAGESHIFT) 269 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 270 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 271 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 272 psm_intr_op_t, int *); 273 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 274 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 275 /* 276 * Use device arena to use for device control register mappings. 277 * Various kernel memory walkers (debugger, dtrace) need to know 278 * to avoid this address range to prevent undesired device activity. 279 */ 280 extern void *device_arena_alloc(size_t size, int vm_flag); 281 extern void device_arena_free(void * vaddr, size_t size); 282 283 284 /* 285 * Internal functions 286 */ 287 static int rootnex_dma_init(); 288 static void rootnex_add_props(dev_info_t *); 289 static int rootnex_ctl_reportdev(dev_info_t *dip); 290 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 291 static int rootnex_ctlops_poke(peekpoke_ctlops_t *in_args); 292 static int rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result); 293 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 294 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 295 static int rootnex_map_handle(ddi_map_req_t *mp); 296 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 297 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 298 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 299 ddi_dma_attr_t *attr); 300 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 301 rootnex_sglinfo_t *sglinfo); 302 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 303 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 304 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 305 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 306 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 307 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 308 ddi_dma_attr_t *attr, int kmflag); 309 static void rootnex_teardown_windows(rootnex_dma_t *dma); 310 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 311 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 312 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 313 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 314 size_t *copybuf_used, page_t **cur_pp); 315 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 316 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 317 ddi_dma_attr_t *attr, off_t cur_offset); 318 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 319 rootnex_dma_t *dma, rootnex_window_t **windowp, 320 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 321 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 322 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 323 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 324 off_t offset, size_t size, uint_t cache_flags); 325 static int rootnex_verify_buffer(rootnex_dma_t *dma); 326 static int rootnex_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, 327 const void *no_used); 328 329 330 /* 331 * _init() 332 * 333 */ 334 int 335 _init(void) 336 { 337 338 rootnex_state = NULL; 339 return (mod_install(&rootnex_modlinkage)); 340 } 341 342 343 /* 344 * _info() 345 * 346 */ 347 int 348 _info(struct modinfo *modinfop) 349 { 350 return (mod_info(&rootnex_modlinkage, modinfop)); 351 } 352 353 354 /* 355 * _fini() 356 * 357 */ 358 int 359 _fini(void) 360 { 361 return (EBUSY); 362 } 363 364 365 /* 366 * rootnex_attach() 367 * 368 */ 369 static int 370 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 371 { 372 int fmcap; 373 int e; 374 375 376 switch (cmd) { 377 case DDI_ATTACH: 378 break; 379 case DDI_RESUME: 380 return (DDI_SUCCESS); 381 default: 382 return (DDI_FAILURE); 383 } 384 385 /* 386 * We should only have one instance of rootnex. Save it away since we 387 * don't have an easy way to get it back later. 388 */ 389 ASSERT(rootnex_state == NULL); 390 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 391 392 rootnex_state->r_dip = dip; 393 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 394 rootnex_state->r_reserved_msg_printed = B_FALSE; 395 rootnex_cnt = &rootnex_state->r_counters[0]; 396 397 mutex_init(&rootnex_state->r_peekpoke_mutex, NULL, MUTEX_SPIN, 398 (void *)ipltospl(15)); 399 400 /* 401 * Set minimum fm capability level for i86pc platforms and then 402 * initialize error handling. Since we're the rootnex, we don't 403 * care what's returned in the fmcap field. 404 */ 405 ddi_system_fmcap = DDI_FM_ERRCB_CAPABLE; 406 fmcap = ddi_system_fmcap; 407 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 408 if (fmcap & DDI_FM_ERRCB_CAPABLE) 409 ddi_fm_handler_register(dip, rootnex_fm_callback, NULL); 410 411 /* initialize DMA related state */ 412 e = rootnex_dma_init(); 413 if (e != DDI_SUCCESS) { 414 mutex_destroy(&rootnex_state->r_peekpoke_mutex); 415 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 416 return (DDI_FAILURE); 417 } 418 419 /* Add static root node properties */ 420 rootnex_add_props(dip); 421 422 /* since we can't call ddi_report_dev() */ 423 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 424 425 /* Initialize rootnex event handle */ 426 i_ddi_rootnex_init_events(dip); 427 428 return (DDI_SUCCESS); 429 } 430 431 432 /* 433 * rootnex_detach() 434 * 435 */ 436 /*ARGSUSED*/ 437 static int 438 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 439 { 440 switch (cmd) { 441 case DDI_SUSPEND: 442 break; 443 default: 444 return (DDI_FAILURE); 445 } 446 447 return (DDI_SUCCESS); 448 } 449 450 451 /* 452 * rootnex_dma_init() 453 * 454 */ 455 /*ARGSUSED*/ 456 static int 457 rootnex_dma_init() 458 { 459 size_t bufsize; 460 461 462 /* 463 * size of our cookie/window/copybuf state needed in dma bind that we 464 * pre-alloc in dma_alloc_handle 465 */ 466 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 467 rootnex_state->r_prealloc_size = 468 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 469 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 470 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 471 472 /* 473 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 474 * allocate 16 extra bytes for struct pointer alignment 475 * (p->dmai_private & dma->dp_prealloc_buffer) 476 */ 477 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 478 rootnex_state->r_prealloc_size + 0x10; 479 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 480 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 481 if (rootnex_state->r_dmahdl_cache == NULL) { 482 return (DDI_FAILURE); 483 } 484 485 /* 486 * allocate array to track which major numbers we have printed warnings 487 * for. 488 */ 489 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 490 KM_SLEEP); 491 492 return (DDI_SUCCESS); 493 } 494 495 496 /* 497 * rootnex_add_props() 498 * 499 */ 500 static void 501 rootnex_add_props(dev_info_t *dip) 502 { 503 rootnex_intprop_t *rpp; 504 int i; 505 506 /* Add static integer/boolean properties to the root node */ 507 rpp = rootnex_intprp; 508 for (i = 0; i < NROOT_INTPROPS; i++) { 509 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 510 rpp[i].prop_name, rpp[i].prop_value); 511 } 512 } 513 514 515 516 /* 517 * ************************* 518 * ctlops related routines 519 * ************************* 520 */ 521 522 /* 523 * rootnex_ctlops() 524 * 525 */ 526 /*ARGSUSED*/ 527 static int 528 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 529 void *arg, void *result) 530 { 531 int n, *ptr; 532 struct ddi_parent_private_data *pdp; 533 534 switch (ctlop) { 535 case DDI_CTLOPS_DMAPMAPC: 536 /* 537 * Return 'partial' to indicate that dma mapping 538 * has to be done in the main MMU. 539 */ 540 return (DDI_DMA_PARTIAL); 541 542 case DDI_CTLOPS_BTOP: 543 /* 544 * Convert byte count input to physical page units. 545 * (byte counts that are not a page-size multiple 546 * are rounded down) 547 */ 548 *(ulong_t *)result = btop(*(ulong_t *)arg); 549 return (DDI_SUCCESS); 550 551 case DDI_CTLOPS_PTOB: 552 /* 553 * Convert size in physical pages to bytes 554 */ 555 *(ulong_t *)result = ptob(*(ulong_t *)arg); 556 return (DDI_SUCCESS); 557 558 case DDI_CTLOPS_BTOPR: 559 /* 560 * Convert byte count input to physical page units 561 * (byte counts that are not a page-size multiple 562 * are rounded up) 563 */ 564 *(ulong_t *)result = btopr(*(ulong_t *)arg); 565 return (DDI_SUCCESS); 566 567 case DDI_CTLOPS_POKE: 568 return (rootnex_ctlops_poke((peekpoke_ctlops_t *)arg)); 569 570 case DDI_CTLOPS_PEEK: 571 return (rootnex_ctlops_peek((peekpoke_ctlops_t *)arg, result)); 572 573 case DDI_CTLOPS_INITCHILD: 574 return (impl_ddi_sunbus_initchild(arg)); 575 576 case DDI_CTLOPS_UNINITCHILD: 577 impl_ddi_sunbus_removechild(arg); 578 return (DDI_SUCCESS); 579 580 case DDI_CTLOPS_REPORTDEV: 581 return (rootnex_ctl_reportdev(rdip)); 582 583 case DDI_CTLOPS_IOMIN: 584 /* 585 * Nothing to do here but reflect back.. 586 */ 587 return (DDI_SUCCESS); 588 589 case DDI_CTLOPS_REGSIZE: 590 case DDI_CTLOPS_NREGS: 591 break; 592 593 case DDI_CTLOPS_SIDDEV: 594 if (ndi_dev_is_prom_node(rdip)) 595 return (DDI_SUCCESS); 596 if (ndi_dev_is_persistent_node(rdip)) 597 return (DDI_SUCCESS); 598 return (DDI_FAILURE); 599 600 case DDI_CTLOPS_POWER: 601 return ((*pm_platform_power)((power_req_t *)arg)); 602 603 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 604 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 605 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 606 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 607 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 608 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 609 if (!rootnex_state->r_reserved_msg_printed) { 610 rootnex_state->r_reserved_msg_printed = B_TRUE; 611 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 612 "1 or more reserved/obsolete operations."); 613 } 614 return (DDI_FAILURE); 615 616 default: 617 return (DDI_FAILURE); 618 } 619 /* 620 * The rest are for "hardware" properties 621 */ 622 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 623 return (DDI_FAILURE); 624 625 if (ctlop == DDI_CTLOPS_NREGS) { 626 ptr = (int *)result; 627 *ptr = pdp->par_nreg; 628 } else { 629 off_t *size = (off_t *)result; 630 631 ptr = (int *)arg; 632 n = *ptr; 633 if (n >= pdp->par_nreg) { 634 return (DDI_FAILURE); 635 } 636 *size = (off_t)pdp->par_reg[n].regspec_size; 637 } 638 return (DDI_SUCCESS); 639 } 640 641 642 /* 643 * rootnex_ctl_reportdev() 644 * 645 */ 646 static int 647 rootnex_ctl_reportdev(dev_info_t *dev) 648 { 649 int i, n, len, f_len = 0; 650 char *buf; 651 652 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 653 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 654 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 655 len = strlen(buf); 656 657 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 658 659 struct regspec *rp = sparc_pd_getreg(dev, i); 660 661 if (i == 0) 662 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 663 ": "); 664 else 665 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 666 " and "); 667 len = strlen(buf); 668 669 switch (rp->regspec_bustype) { 670 671 case BTEISA: 672 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 673 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 674 break; 675 676 case BTISA: 677 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 678 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 679 break; 680 681 default: 682 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 683 "space %x offset %x", 684 rp->regspec_bustype, rp->regspec_addr); 685 break; 686 } 687 len = strlen(buf); 688 } 689 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 690 int pri; 691 692 if (i != 0) { 693 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 694 ","); 695 len = strlen(buf); 696 } 697 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 698 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 699 " sparc ipl %d", pri); 700 len = strlen(buf); 701 } 702 #ifdef DEBUG 703 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 704 cmn_err(CE_NOTE, "next message is truncated: " 705 "printed length 1024, real length %d", f_len); 706 } 707 #endif /* DEBUG */ 708 cmn_err(CE_CONT, "?%s\n", buf); 709 kmem_free(buf, REPORTDEV_BUFSIZE); 710 return (DDI_SUCCESS); 711 } 712 713 714 /* 715 * rootnex_ctlops_poke() 716 * 717 */ 718 static int 719 rootnex_ctlops_poke(peekpoke_ctlops_t *in_args) 720 { 721 int err = DDI_SUCCESS; 722 on_trap_data_t otd; 723 724 /* Cautious access not supported. */ 725 if (in_args->handle != NULL) 726 return (DDI_FAILURE); 727 728 mutex_enter(&rootnex_state->r_peekpoke_mutex); 729 730 /* Set up protected environment. */ 731 if (!on_trap(&otd, OT_DATA_ACCESS)) { 732 switch (in_args->size) { 733 case sizeof (uint8_t): 734 *(uint8_t *)in_args->dev_addr = *(uint8_t *) 735 in_args->host_addr; 736 break; 737 738 case sizeof (uint16_t): 739 *(uint16_t *)in_args->dev_addr = 740 *(uint16_t *)in_args->host_addr; 741 break; 742 743 case sizeof (uint32_t): 744 *(uint32_t *)in_args->dev_addr = 745 *(uint32_t *)in_args->host_addr; 746 break; 747 748 case sizeof (uint64_t): 749 *(uint64_t *)in_args->dev_addr = 750 *(uint64_t *)in_args->host_addr; 751 break; 752 753 default: 754 err = DDI_FAILURE; 755 break; 756 } 757 } else 758 err = DDI_FAILURE; 759 760 /* Take down protected environment. */ 761 no_trap(); 762 mutex_exit(&rootnex_state->r_peekpoke_mutex); 763 764 return (err); 765 } 766 767 768 /* 769 * rootnex_ctlops_peek() 770 * 771 */ 772 static int 773 rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result) 774 { 775 int err = DDI_SUCCESS; 776 on_trap_data_t otd; 777 778 /* Cautious access not supported. */ 779 if (in_args->handle != NULL) 780 return (DDI_FAILURE); 781 782 mutex_enter(&rootnex_state->r_peekpoke_mutex); 783 784 if (!on_trap(&otd, OT_DATA_ACCESS)) { 785 switch (in_args->size) { 786 case sizeof (uint8_t): 787 *(uint8_t *)in_args->host_addr = 788 *(uint8_t *)in_args->dev_addr; 789 break; 790 791 case sizeof (uint16_t): 792 *(uint16_t *)in_args->host_addr = 793 *(uint16_t *)in_args->dev_addr; 794 break; 795 796 case sizeof (uint32_t): 797 *(uint32_t *)in_args->host_addr = 798 *(uint32_t *)in_args->dev_addr; 799 break; 800 801 case sizeof (uint64_t): 802 *(uint64_t *)in_args->host_addr = 803 *(uint64_t *)in_args->dev_addr; 804 break; 805 806 default: 807 err = DDI_FAILURE; 808 break; 809 } 810 result = (void *)in_args->host_addr; 811 } else 812 err = DDI_FAILURE; 813 814 no_trap(); 815 mutex_exit(&rootnex_state->r_peekpoke_mutex); 816 817 return (err); 818 } 819 820 821 822 /* 823 * ****************** 824 * map related code 825 * ****************** 826 */ 827 828 /* 829 * rootnex_map() 830 * 831 */ 832 static int 833 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 834 off_t len, caddr_t *vaddrp) 835 { 836 struct regspec *rp, tmp_reg; 837 ddi_map_req_t mr = *mp; /* Get private copy of request */ 838 int error; 839 840 mp = &mr; 841 842 switch (mp->map_op) { 843 case DDI_MO_MAP_LOCKED: 844 case DDI_MO_UNMAP: 845 case DDI_MO_MAP_HANDLE: 846 break; 847 default: 848 #ifdef DDI_MAP_DEBUG 849 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 850 mp->map_op); 851 #endif /* DDI_MAP_DEBUG */ 852 return (DDI_ME_UNIMPLEMENTED); 853 } 854 855 if (mp->map_flags & DDI_MF_USER_MAPPING) { 856 #ifdef DDI_MAP_DEBUG 857 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 858 #endif /* DDI_MAP_DEBUG */ 859 return (DDI_ME_UNIMPLEMENTED); 860 } 861 862 /* 863 * First, if given an rnumber, convert it to a regspec... 864 * (Presumably, this is on behalf of a child of the root node?) 865 */ 866 867 if (mp->map_type == DDI_MT_RNUMBER) { 868 869 int rnumber = mp->map_obj.rnumber; 870 #ifdef DDI_MAP_DEBUG 871 static char *out_of_range = 872 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 873 #endif /* DDI_MAP_DEBUG */ 874 875 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 876 if (rp == NULL) { 877 #ifdef DDI_MAP_DEBUG 878 cmn_err(CE_WARN, out_of_range, rnumber, 879 ddi_get_name(rdip)); 880 #endif /* DDI_MAP_DEBUG */ 881 return (DDI_ME_RNUMBER_RANGE); 882 } 883 884 /* 885 * Convert the given ddi_map_req_t from rnumber to regspec... 886 */ 887 888 mp->map_type = DDI_MT_REGSPEC; 889 mp->map_obj.rp = rp; 890 } 891 892 /* 893 * Adjust offset and length correspnding to called values... 894 * XXX: A non-zero length means override the one in the regspec 895 * XXX: (regardless of what's in the parent's range?) 896 */ 897 898 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 899 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 900 901 #ifdef DDI_MAP_DEBUG 902 cmn_err(CE_CONT, 903 "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d>" 904 " offset %d len %d handle 0x%x\n", 905 ddi_get_name(dip), ddi_get_name(rdip), 906 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 907 offset, len, mp->map_handlep); 908 #endif /* DDI_MAP_DEBUG */ 909 910 /* 911 * I/O or memory mapping: 912 * 913 * <bustype=0, addr=x, len=x>: memory 914 * <bustype=1, addr=x, len=x>: i/o 915 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 916 */ 917 918 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 919 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 920 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 921 ddi_get_name(rdip), rp->regspec_bustype, 922 rp->regspec_addr, rp->regspec_size); 923 return (DDI_ME_INVAL); 924 } 925 926 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 927 /* 928 * compatibility i/o mapping 929 */ 930 rp->regspec_bustype += (uint_t)offset; 931 } else { 932 /* 933 * Normal memory or i/o mapping 934 */ 935 rp->regspec_addr += (uint_t)offset; 936 } 937 938 if (len != 0) 939 rp->regspec_size = (uint_t)len; 940 941 #ifdef DDI_MAP_DEBUG 942 cmn_err(CE_CONT, 943 " <%s,%s> <0x%x, 0x%x, 0x%d>" 944 " offset %d len %d handle 0x%x\n", 945 ddi_get_name(dip), ddi_get_name(rdip), 946 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 947 offset, len, mp->map_handlep); 948 #endif /* DDI_MAP_DEBUG */ 949 950 /* 951 * Apply any parent ranges at this level, if applicable. 952 * (This is where nexus specific regspec translation takes place. 953 * Use of this function is implicit agreement that translation is 954 * provided via ddi_apply_range.) 955 */ 956 957 #ifdef DDI_MAP_DEBUG 958 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 959 ddi_get_name(dip), ddi_get_name(rdip)); 960 #endif /* DDI_MAP_DEBUG */ 961 962 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 963 return (error); 964 965 switch (mp->map_op) { 966 case DDI_MO_MAP_LOCKED: 967 968 /* 969 * Set up the locked down kernel mapping to the regspec... 970 */ 971 972 return (rootnex_map_regspec(mp, vaddrp)); 973 974 case DDI_MO_UNMAP: 975 976 /* 977 * Release mapping... 978 */ 979 980 return (rootnex_unmap_regspec(mp, vaddrp)); 981 982 case DDI_MO_MAP_HANDLE: 983 984 return (rootnex_map_handle(mp)); 985 986 default: 987 return (DDI_ME_UNIMPLEMENTED); 988 } 989 } 990 991 992 /* 993 * rootnex_map_fault() 994 * 995 * fault in mappings for requestors 996 */ 997 /*ARGSUSED*/ 998 static int 999 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 1000 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 1001 uint_t lock) 1002 { 1003 1004 #ifdef DDI_MAP_DEBUG 1005 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 1006 ddi_map_debug(" Seg <%s>\n", 1007 seg->s_ops == &segdev_ops ? "segdev" : 1008 seg == &kvseg ? "segkmem" : "NONE!"); 1009 #endif /* DDI_MAP_DEBUG */ 1010 1011 /* 1012 * This is all terribly broken, but it is a start 1013 * 1014 * XXX Note that this test means that segdev_ops 1015 * must be exported from seg_dev.c. 1016 * XXX What about devices with their own segment drivers? 1017 */ 1018 if (seg->s_ops == &segdev_ops) { 1019 struct segdev_data *sdp = 1020 (struct segdev_data *)seg->s_data; 1021 1022 if (hat == NULL) { 1023 /* 1024 * This is one plausible interpretation of 1025 * a null hat i.e. use the first hat on the 1026 * address space hat list which by convention is 1027 * the hat of the system MMU. At alternative 1028 * would be to panic .. this might well be better .. 1029 */ 1030 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1031 hat = seg->s_as->a_hat; 1032 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1033 } 1034 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1035 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1036 } else if (seg == &kvseg && dp == NULL) { 1037 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1038 HAT_LOAD_LOCK); 1039 } else 1040 return (DDI_FAILURE); 1041 return (DDI_SUCCESS); 1042 } 1043 1044 1045 /* 1046 * rootnex_map_regspec() 1047 * we don't support mapping of I/O cards above 4Gb 1048 */ 1049 static int 1050 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1051 { 1052 ulong_t base; 1053 void *cvaddr; 1054 uint_t npages, pgoffset; 1055 struct regspec *rp; 1056 ddi_acc_hdl_t *hp; 1057 ddi_acc_impl_t *ap; 1058 uint_t hat_acc_flags; 1059 1060 rp = mp->map_obj.rp; 1061 hp = mp->map_handlep; 1062 1063 #ifdef DDI_MAP_DEBUG 1064 ddi_map_debug( 1065 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1066 rp->regspec_bustype, rp->regspec_addr, 1067 rp->regspec_size, mp->map_handlep); 1068 #endif /* DDI_MAP_DEBUG */ 1069 1070 /* 1071 * I/O or memory mapping 1072 * 1073 * <bustype=0, addr=x, len=x>: memory 1074 * <bustype=1, addr=x, len=x>: i/o 1075 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1076 */ 1077 1078 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1079 cmn_err(CE_WARN, "rootnex: invalid register spec" 1080 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1081 rp->regspec_addr, rp->regspec_size); 1082 return (DDI_FAILURE); 1083 } 1084 1085 if (rp->regspec_bustype != 0) { 1086 /* 1087 * I/O space - needs a handle. 1088 */ 1089 if (hp == NULL) { 1090 return (DDI_FAILURE); 1091 } 1092 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1093 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1094 impl_acc_hdl_init(hp); 1095 1096 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1097 #ifdef DDI_MAP_DEBUG 1098 ddi_map_debug("rootnex_map_regspec: mmap() \ 1099 to I/O space is not supported.\n"); 1100 #endif /* DDI_MAP_DEBUG */ 1101 return (DDI_ME_INVAL); 1102 } else { 1103 /* 1104 * 1275-compliant vs. compatibility i/o mapping 1105 */ 1106 *vaddrp = 1107 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1108 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1109 ((caddr_t)(uintptr_t)rp->regspec_addr); 1110 } 1111 1112 #ifdef DDI_MAP_DEBUG 1113 ddi_map_debug( 1114 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1115 rp->regspec_size, *vaddrp); 1116 #endif /* DDI_MAP_DEBUG */ 1117 return (DDI_SUCCESS); 1118 } 1119 1120 /* 1121 * Memory space 1122 */ 1123 1124 if (hp != NULL) { 1125 /* 1126 * hat layer ignores 1127 * hp->ah_acc.devacc_attr_endian_flags. 1128 */ 1129 switch (hp->ah_acc.devacc_attr_dataorder) { 1130 case DDI_STRICTORDER_ACC: 1131 hat_acc_flags = HAT_STRICTORDER; 1132 break; 1133 case DDI_UNORDERED_OK_ACC: 1134 hat_acc_flags = HAT_UNORDERED_OK; 1135 break; 1136 case DDI_MERGING_OK_ACC: 1137 hat_acc_flags = HAT_MERGING_OK; 1138 break; 1139 case DDI_LOADCACHING_OK_ACC: 1140 hat_acc_flags = HAT_LOADCACHING_OK; 1141 break; 1142 case DDI_STORECACHING_OK_ACC: 1143 hat_acc_flags = HAT_STORECACHING_OK; 1144 break; 1145 } 1146 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1147 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1148 impl_acc_hdl_init(hp); 1149 hp->ah_hat_flags = hat_acc_flags; 1150 } else { 1151 hat_acc_flags = HAT_STRICTORDER; 1152 } 1153 1154 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1155 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1156 1157 if (rp->regspec_size == 0) { 1158 #ifdef DDI_MAP_DEBUG 1159 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1160 #endif /* DDI_MAP_DEBUG */ 1161 return (DDI_ME_INVAL); 1162 } 1163 1164 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1165 *vaddrp = (caddr_t)mmu_btop(base); 1166 } else { 1167 npages = mmu_btopr(rp->regspec_size + pgoffset); 1168 1169 #ifdef DDI_MAP_DEBUG 1170 ddi_map_debug("rootnex_map_regspec: Mapping %d pages \ 1171 physical %x ", 1172 npages, base); 1173 #endif /* DDI_MAP_DEBUG */ 1174 1175 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1176 if (cvaddr == NULL) 1177 return (DDI_ME_NORESOURCES); 1178 1179 /* 1180 * Now map in the pages we've allocated... 1181 */ 1182 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base), 1183 mp->map_prot | hat_acc_flags, HAT_LOAD_LOCK); 1184 *vaddrp = (caddr_t)cvaddr + pgoffset; 1185 } 1186 1187 #ifdef DDI_MAP_DEBUG 1188 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1189 #endif /* DDI_MAP_DEBUG */ 1190 return (DDI_SUCCESS); 1191 } 1192 1193 1194 /* 1195 * rootnex_unmap_regspec() 1196 * 1197 */ 1198 static int 1199 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1200 { 1201 caddr_t addr = (caddr_t)*vaddrp; 1202 uint_t npages, pgoffset; 1203 struct regspec *rp; 1204 1205 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1206 return (0); 1207 1208 rp = mp->map_obj.rp; 1209 1210 if (rp->regspec_size == 0) { 1211 #ifdef DDI_MAP_DEBUG 1212 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1213 #endif /* DDI_MAP_DEBUG */ 1214 return (DDI_ME_INVAL); 1215 } 1216 1217 /* 1218 * I/O or memory mapping: 1219 * 1220 * <bustype=0, addr=x, len=x>: memory 1221 * <bustype=1, addr=x, len=x>: i/o 1222 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1223 */ 1224 if (rp->regspec_bustype != 0) { 1225 /* 1226 * This is I/O space, which requires no particular 1227 * processing on unmap since it isn't mapped in the 1228 * first place. 1229 */ 1230 return (DDI_SUCCESS); 1231 } 1232 1233 /* 1234 * Memory space 1235 */ 1236 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1237 npages = mmu_btopr(rp->regspec_size + pgoffset); 1238 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1239 device_arena_free(addr - pgoffset, ptob(npages)); 1240 1241 /* 1242 * Destroy the pointer - the mapping has logically gone 1243 */ 1244 *vaddrp = NULL; 1245 1246 return (DDI_SUCCESS); 1247 } 1248 1249 1250 /* 1251 * rootnex_map_handle() 1252 * 1253 */ 1254 static int 1255 rootnex_map_handle(ddi_map_req_t *mp) 1256 { 1257 ddi_acc_hdl_t *hp; 1258 ulong_t base; 1259 uint_t pgoffset; 1260 struct regspec *rp; 1261 1262 rp = mp->map_obj.rp; 1263 1264 #ifdef DDI_MAP_DEBUG 1265 ddi_map_debug( 1266 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1267 rp->regspec_bustype, rp->regspec_addr, 1268 rp->regspec_size, mp->map_handlep); 1269 #endif /* DDI_MAP_DEBUG */ 1270 1271 /* 1272 * I/O or memory mapping: 1273 * 1274 * <bustype=0, addr=x, len=x>: memory 1275 * <bustype=1, addr=x, len=x>: i/o 1276 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1277 */ 1278 if (rp->regspec_bustype != 0) { 1279 /* 1280 * This refers to I/O space, and we don't support "mapping" 1281 * I/O space to a user. 1282 */ 1283 return (DDI_FAILURE); 1284 } 1285 1286 /* 1287 * Set up the hat_flags for the mapping. 1288 */ 1289 hp = mp->map_handlep; 1290 1291 switch (hp->ah_acc.devacc_attr_endian_flags) { 1292 case DDI_NEVERSWAP_ACC: 1293 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1294 break; 1295 case DDI_STRUCTURE_LE_ACC: 1296 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1297 break; 1298 case DDI_STRUCTURE_BE_ACC: 1299 return (DDI_FAILURE); 1300 default: 1301 return (DDI_REGS_ACC_CONFLICT); 1302 } 1303 1304 switch (hp->ah_acc.devacc_attr_dataorder) { 1305 case DDI_STRICTORDER_ACC: 1306 break; 1307 case DDI_UNORDERED_OK_ACC: 1308 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1309 break; 1310 case DDI_MERGING_OK_ACC: 1311 hp->ah_hat_flags |= HAT_MERGING_OK; 1312 break; 1313 case DDI_LOADCACHING_OK_ACC: 1314 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1315 break; 1316 case DDI_STORECACHING_OK_ACC: 1317 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1318 break; 1319 default: 1320 return (DDI_FAILURE); 1321 } 1322 1323 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1324 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1325 1326 if (rp->regspec_size == 0) 1327 return (DDI_ME_INVAL); 1328 1329 hp->ah_pfn = mmu_btop(base); 1330 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1331 1332 return (DDI_SUCCESS); 1333 } 1334 1335 1336 1337 /* 1338 * ************************ 1339 * interrupt related code 1340 * ************************ 1341 */ 1342 1343 /* 1344 * rootnex_intr_ops() 1345 * bus_intr_op() function for interrupt support 1346 */ 1347 /* ARGSUSED */ 1348 static int 1349 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1350 ddi_intr_handle_impl_t *hdlp, void *result) 1351 { 1352 struct intrspec *ispec; 1353 struct ddi_parent_private_data *pdp; 1354 1355 DDI_INTR_NEXDBG((CE_CONT, 1356 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1357 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1358 1359 /* Process the interrupt operation */ 1360 switch (intr_op) { 1361 case DDI_INTROP_GETCAP: 1362 /* First check with pcplusmp */ 1363 if (psm_intr_ops == NULL) 1364 return (DDI_FAILURE); 1365 1366 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1367 *(int *)result = 0; 1368 return (DDI_FAILURE); 1369 } 1370 break; 1371 case DDI_INTROP_SETCAP: 1372 if (psm_intr_ops == NULL) 1373 return (DDI_FAILURE); 1374 1375 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1376 return (DDI_FAILURE); 1377 break; 1378 case DDI_INTROP_ALLOC: 1379 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1380 return (DDI_FAILURE); 1381 hdlp->ih_pri = ispec->intrspec_pri; 1382 *(int *)result = hdlp->ih_scratch1; 1383 break; 1384 case DDI_INTROP_FREE: 1385 pdp = ddi_get_parent_data(rdip); 1386 /* 1387 * Special case for 'pcic' driver' only. 1388 * If an intrspec was created for it, clean it up here 1389 * See detailed comments on this in the function 1390 * rootnex_get_ispec(). 1391 */ 1392 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1393 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1394 pdp->par_nintr); 1395 /* 1396 * Set it to zero; so that 1397 * DDI framework doesn't free it again 1398 */ 1399 pdp->par_intr = NULL; 1400 pdp->par_nintr = 0; 1401 } 1402 break; 1403 case DDI_INTROP_GETPRI: 1404 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1405 return (DDI_FAILURE); 1406 *(int *)result = ispec->intrspec_pri; 1407 break; 1408 case DDI_INTROP_SETPRI: 1409 /* Validate the interrupt priority passed to us */ 1410 if (*(int *)result > LOCK_LEVEL) 1411 return (DDI_FAILURE); 1412 1413 /* Ensure that PSM is all initialized and ispec is ok */ 1414 if ((psm_intr_ops == NULL) || 1415 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1416 return (DDI_FAILURE); 1417 1418 /* Change the priority */ 1419 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1420 PSM_FAILURE) 1421 return (DDI_FAILURE); 1422 1423 /* update the ispec with the new priority */ 1424 ispec->intrspec_pri = *(int *)result; 1425 break; 1426 case DDI_INTROP_ADDISR: 1427 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1428 return (DDI_FAILURE); 1429 ispec->intrspec_func = hdlp->ih_cb_func; 1430 break; 1431 case DDI_INTROP_REMISR: 1432 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1433 return (DDI_FAILURE); 1434 ispec->intrspec_func = (uint_t (*)()) 0; 1435 break; 1436 case DDI_INTROP_ENABLE: 1437 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1438 return (DDI_FAILURE); 1439 1440 /* Call psmi to translate irq with the dip */ 1441 if (psm_intr_ops == NULL) 1442 return (DDI_FAILURE); 1443 1444 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1445 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1446 (int *)&hdlp->ih_vector); 1447 1448 /* Add the interrupt handler */ 1449 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1450 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1451 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1452 return (DDI_FAILURE); 1453 break; 1454 case DDI_INTROP_DISABLE: 1455 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1456 return (DDI_FAILURE); 1457 1458 /* Call psm_ops() to translate irq with the dip */ 1459 if (psm_intr_ops == NULL) 1460 return (DDI_FAILURE); 1461 1462 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1463 (void) (*psm_intr_ops)(rdip, hdlp, 1464 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1465 1466 /* Remove the interrupt handler */ 1467 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1468 hdlp->ih_cb_func, hdlp->ih_vector); 1469 break; 1470 case DDI_INTROP_SETMASK: 1471 if (psm_intr_ops == NULL) 1472 return (DDI_FAILURE); 1473 1474 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1475 return (DDI_FAILURE); 1476 break; 1477 case DDI_INTROP_CLRMASK: 1478 if (psm_intr_ops == NULL) 1479 return (DDI_FAILURE); 1480 1481 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1482 return (DDI_FAILURE); 1483 break; 1484 case DDI_INTROP_GETPENDING: 1485 if (psm_intr_ops == NULL) 1486 return (DDI_FAILURE); 1487 1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1489 result)) { 1490 *(int *)result = 0; 1491 return (DDI_FAILURE); 1492 } 1493 break; 1494 case DDI_INTROP_NINTRS: 1495 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 1496 return (DDI_FAILURE); 1497 *(int *)result = pdp->par_nintr; 1498 if (pdp->par_nintr == 0) { 1499 /* 1500 * Special case for 'pcic' driver' only. This driver 1501 * driver is a child of 'isa' and 'rootnex' drivers. 1502 * 1503 * See detailed comments on this in the function 1504 * rootnex_get_ispec(). 1505 * 1506 * Children of 'pcic' send 'NINITR' request all the 1507 * way to rootnex driver. But, the 'pdp->par_nintr' 1508 * field may not initialized. So, we fake it here 1509 * to return 1 (a la what PCMCIA nexus does). 1510 */ 1511 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1512 *(int *)result = 1; 1513 } 1514 break; 1515 case DDI_INTROP_SUPPORTED_TYPES: 1516 *(int *)result = 0; 1517 *(int *)result |= DDI_INTR_TYPE_FIXED; /* Always ... */ 1518 break; 1519 case DDI_INTROP_NAVAIL: 1520 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1521 return (DDI_FAILURE); 1522 1523 if (psm_intr_ops == NULL) { 1524 *(int *)result = 1; 1525 break; 1526 } 1527 1528 /* Priority in the handle not initialized yet */ 1529 hdlp->ih_pri = ispec->intrspec_pri; 1530 (void) (*psm_intr_ops)(rdip, hdlp, 1531 PSM_INTR_OP_NAVAIL_VECTORS, result); 1532 break; 1533 default: 1534 return (DDI_FAILURE); 1535 } 1536 1537 return (DDI_SUCCESS); 1538 } 1539 1540 1541 /* 1542 * rootnex_get_ispec() 1543 * convert an interrupt number to an interrupt specification. 1544 * The interrupt number determines which interrupt spec will be 1545 * returned if more than one exists. 1546 * 1547 * Look into the parent private data area of the 'rdip' to find out 1548 * the interrupt specification. First check to make sure there is 1549 * one that matchs "inumber" and then return a pointer to it. 1550 * 1551 * Return NULL if one could not be found. 1552 * 1553 * NOTE: This is needed for rootnex_intr_ops() 1554 */ 1555 static struct intrspec * 1556 rootnex_get_ispec(dev_info_t *rdip, int inum) 1557 { 1558 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1559 1560 /* 1561 * Special case handling for drivers that provide their own 1562 * intrspec structures instead of relying on the DDI framework. 1563 * 1564 * A broken hardware driver in ON could potentially provide its 1565 * own intrspec structure, instead of relying on the hardware. 1566 * If these drivers are children of 'rootnex' then we need to 1567 * continue to provide backward compatibility to them here. 1568 * 1569 * Following check is a special case for 'pcic' driver which 1570 * was found to have broken hardwre andby provides its own intrspec. 1571 * 1572 * Verbatim comments from this driver are shown here: 1573 * "Don't use the ddi_add_intr since we don't have a 1574 * default intrspec in all cases." 1575 * 1576 * Since an 'ispec' may not be always created for it, 1577 * check for that and create one if so. 1578 * 1579 * NOTE: Currently 'pcic' is the only driver found to do this. 1580 */ 1581 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1582 pdp->par_nintr = 1; 1583 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1584 pdp->par_nintr, KM_SLEEP); 1585 } 1586 1587 /* Validate the interrupt number */ 1588 if (inum >= pdp->par_nintr) 1589 return (NULL); 1590 1591 /* Get the interrupt structure pointer and return that */ 1592 return ((struct intrspec *)&pdp->par_intr[inum]); 1593 } 1594 1595 1596 /* 1597 * ****************** 1598 * dma related code 1599 * ****************** 1600 */ 1601 1602 /* 1603 * rootnex_dma_allochdl() 1604 * called from ddi_dma_alloc_handle(). 1605 */ 1606 /*ARGSUSED*/ 1607 static int 1608 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1609 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1610 { 1611 uint64_t maxsegmentsize_ll; 1612 uint_t maxsegmentsize; 1613 ddi_dma_impl_t *hp; 1614 rootnex_dma_t *dma; 1615 uint64_t count_max; 1616 uint64_t seg; 1617 int kmflag; 1618 int e; 1619 1620 1621 /* convert our sleep flags */ 1622 if (waitfp == DDI_DMA_SLEEP) { 1623 kmflag = KM_SLEEP; 1624 } else { 1625 kmflag = KM_NOSLEEP; 1626 } 1627 1628 /* 1629 * We try to do only one memory allocation here. We'll do a little 1630 * pointer manipulation later. If the bind ends up taking more than 1631 * our prealloc's space, we'll have to allocate more memory in the 1632 * bind operation. Not great, but much better than before and the 1633 * best we can do with the current bind interfaces. 1634 */ 1635 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1636 if (hp == NULL) { 1637 if (waitfp != DDI_DMA_DONTWAIT) { 1638 ddi_set_callback(waitfp, arg, 1639 &rootnex_state->r_dvma_call_list_id); 1640 } 1641 return (DDI_DMA_NORESOURCES); 1642 } 1643 1644 /* Do our pointer manipulation now, align the structures */ 1645 hp->dmai_private = (void *)(((uintptr_t)hp + 1646 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1647 dma = (rootnex_dma_t *)hp->dmai_private; 1648 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1649 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1650 1651 /* setup the handle */ 1652 rootnex_clean_dmahdl(hp); 1653 dma->dp_dip = rdip; 1654 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1655 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1656 hp->dmai_minxfer = attr->dma_attr_minxfer; 1657 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1658 hp->dmai_rdip = rdip; 1659 hp->dmai_attr = *attr; 1660 1661 /* we don't need to worry about the SPL since we do a tryenter */ 1662 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1663 1664 /* 1665 * Figure out our maximum segment size. If the segment size is greater 1666 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1667 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1668 * dma_attr_count_max are size-1 type values. 1669 * 1670 * Maximum segment size is the largest physically contiguous chunk of 1671 * memory that we can return from a bind (i.e. the maximum size of a 1672 * single cookie). 1673 */ 1674 1675 /* handle the rollover cases */ 1676 seg = attr->dma_attr_seg + 1; 1677 if (seg < attr->dma_attr_seg) { 1678 seg = attr->dma_attr_seg; 1679 } 1680 count_max = attr->dma_attr_count_max + 1; 1681 if (count_max < attr->dma_attr_count_max) { 1682 count_max = attr->dma_attr_count_max; 1683 } 1684 1685 /* 1686 * granularity may or may not be a power of two. If it isn't, we can't 1687 * use a simple mask. 1688 */ 1689 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1690 dma->dp_granularity_power_2 = B_FALSE; 1691 } else { 1692 dma->dp_granularity_power_2 = B_TRUE; 1693 } 1694 1695 /* 1696 * maxxfer should be a whole multiple of granularity. If we're going to 1697 * break up a window because we're greater than maxxfer, we might as 1698 * well make sure it's maxxfer is a whole multiple so we don't have to 1699 * worry about triming the window later on for this case. 1700 */ 1701 if (attr->dma_attr_granular > 1) { 1702 if (dma->dp_granularity_power_2) { 1703 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1704 (attr->dma_attr_maxxfer & 1705 (attr->dma_attr_granular - 1)); 1706 } else { 1707 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1708 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1709 } 1710 } else { 1711 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1712 } 1713 1714 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1715 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1716 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1717 maxsegmentsize = 0xFFFFFFFF; 1718 } else { 1719 maxsegmentsize = maxsegmentsize_ll; 1720 } 1721 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1722 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1723 1724 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1725 if (rootnex_alloc_check_parms) { 1726 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1727 if (e != DDI_SUCCESS) { 1728 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1729 (void) rootnex_dma_freehdl(dip, rdip, 1730 (ddi_dma_handle_t)hp); 1731 return (e); 1732 } 1733 } 1734 1735 *handlep = (ddi_dma_handle_t)hp; 1736 1737 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1738 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1739 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1740 1741 return (DDI_SUCCESS); 1742 } 1743 1744 1745 /* 1746 * rootnex_dma_freehdl() 1747 * called from ddi_dma_free_handle(). 1748 */ 1749 /*ARGSUSED*/ 1750 static int 1751 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1752 { 1753 ddi_dma_impl_t *hp; 1754 rootnex_dma_t *dma; 1755 1756 1757 hp = (ddi_dma_impl_t *)handle; 1758 dma = (rootnex_dma_t *)hp->dmai_private; 1759 1760 /* unbind should have been called first */ 1761 ASSERT(!dma->dp_inuse); 1762 1763 mutex_destroy(&dma->dp_mutex); 1764 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1765 1766 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1767 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1768 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1769 1770 if (rootnex_state->r_dvma_call_list_id) 1771 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1772 1773 return (DDI_SUCCESS); 1774 } 1775 1776 1777 /* 1778 * rootnex_dma_bindhdl() 1779 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1780 */ 1781 /*ARGSUSED*/ 1782 static int 1783 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1784 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1785 { 1786 rootnex_sglinfo_t *sinfo; 1787 ddi_dma_attr_t *attr; 1788 ddi_dma_impl_t *hp; 1789 rootnex_dma_t *dma; 1790 int kmflag; 1791 int e; 1792 1793 1794 hp = (ddi_dma_impl_t *)handle; 1795 dma = (rootnex_dma_t *)hp->dmai_private; 1796 sinfo = &dma->dp_sglinfo; 1797 attr = &hp->dmai_attr; 1798 1799 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1800 1801 /* 1802 * This is useful for debugging a driver. Not as useful in a production 1803 * system. The only time this will fail is if you have a driver bug. 1804 */ 1805 if (rootnex_bind_check_inuse) { 1806 /* 1807 * No one else should ever have this lock unless someone else 1808 * is trying to use this handle. So contention on the lock 1809 * is the same as inuse being set. 1810 */ 1811 e = mutex_tryenter(&dma->dp_mutex); 1812 if (e == 0) { 1813 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1814 return (DDI_DMA_INUSE); 1815 } 1816 if (dma->dp_inuse) { 1817 mutex_exit(&dma->dp_mutex); 1818 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1819 return (DDI_DMA_INUSE); 1820 } 1821 dma->dp_inuse = B_TRUE; 1822 mutex_exit(&dma->dp_mutex); 1823 } 1824 1825 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1826 if (rootnex_bind_check_parms) { 1827 e = rootnex_valid_bind_parms(dmareq, attr); 1828 if (e != DDI_SUCCESS) { 1829 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1830 rootnex_clean_dmahdl(hp); 1831 return (e); 1832 } 1833 } 1834 1835 /* save away the original bind info */ 1836 dma->dp_dma = dmareq->dmar_object; 1837 1838 /* 1839 * Figure out a rough estimate of what maximum number of pages this 1840 * buffer could use (a high estimate of course). 1841 */ 1842 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1843 1844 /* 1845 * We'll use the pre-allocated cookies for any bind that will *always* 1846 * fit (more important to be consistent, we don't want to create 1847 * additional degenerate cases). 1848 */ 1849 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1850 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1851 dma->dp_need_to_free_cookie = B_FALSE; 1852 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1853 uint_t, sinfo->si_max_pages); 1854 1855 /* 1856 * For anything larger than that, we'll go ahead and allocate the 1857 * maximum number of pages we expect to see. Hopefuly, we won't be 1858 * seeing this path in the fast path for high performance devices very 1859 * frequently. 1860 * 1861 * a ddi bind interface that allowed the driver to provide storage to 1862 * the bind interface would speed this case up. 1863 */ 1864 } else { 1865 /* convert the sleep flags */ 1866 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1867 kmflag = KM_SLEEP; 1868 } else { 1869 kmflag = KM_NOSLEEP; 1870 } 1871 1872 /* 1873 * Save away how much memory we allocated. If we're doing a 1874 * nosleep, the alloc could fail... 1875 */ 1876 dma->dp_cookie_size = sinfo->si_max_pages * 1877 sizeof (ddi_dma_cookie_t); 1878 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1879 if (dma->dp_cookies == NULL) { 1880 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1881 rootnex_clean_dmahdl(hp); 1882 return (DDI_DMA_NORESOURCES); 1883 } 1884 dma->dp_need_to_free_cookie = B_TRUE; 1885 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1886 sinfo->si_max_pages); 1887 } 1888 hp->dmai_cookie = dma->dp_cookies; 1889 1890 /* 1891 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1892 * looking at the contraints in the dma structure. It will then put some 1893 * additional state about the sgl in the dma struct (i.e. is the sgl 1894 * clean, or do we need to do some munging; how many pages need to be 1895 * copied, etc.) 1896 */ 1897 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1898 &dma->dp_sglinfo); 1899 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1900 1901 /* if we don't need a copy buffer, we don't need to sync */ 1902 if (sinfo->si_copybuf_req == 0) { 1903 hp->dmai_rflags |= DMP_NOSYNC; 1904 } 1905 1906 /* 1907 * if we don't need the copybuf and we don't need to do a partial, we 1908 * hit the fast path. All the high performance devices should be trying 1909 * to hit this path. To hit this path, a device should be able to reach 1910 * all of memory, shouldn't try to bind more than it can transfer, and 1911 * the buffer shouldn't require more cookies than the driver/device can 1912 * handle [sgllen]). 1913 */ 1914 if ((sinfo->si_copybuf_req == 0) && 1915 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1916 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1917 /* 1918 * copy out the first cookie and ccountp, set the cookie 1919 * pointer to the second cookie. The first cookie is passed 1920 * back on the stack. Additional cookies are accessed via 1921 * ddi_dma_nextcookie() 1922 */ 1923 *cookiep = dma->dp_cookies[0]; 1924 *ccountp = sinfo->si_sgl_size; 1925 hp->dmai_cookie++; 1926 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1927 hp->dmai_nwin = 1; 1928 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1929 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1930 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1931 dma->dp_dma.dmao_size); 1932 return (DDI_DMA_MAPPED); 1933 } 1934 1935 /* 1936 * go to the slow path, we may need to alloc more memory, create 1937 * multiple windows, and munge up a sgl to make the device happy. 1938 */ 1939 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 1940 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 1941 if (dma->dp_need_to_free_cookie) { 1942 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1943 } 1944 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1945 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 1946 return (e); 1947 } 1948 1949 /* if the first window uses the copy buffer, sync it for the device */ 1950 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 1951 (hp->dmai_rflags & DDI_DMA_WRITE)) { 1952 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1953 DDI_DMA_SYNC_FORDEV); 1954 } 1955 1956 /* 1957 * copy out the first cookie and ccountp, set the cookie pointer to the 1958 * second cookie. Make sure the partial flag is set/cleared correctly. 1959 * If we have a partial map (i.e. multiple windows), the number of 1960 * cookies we return is the number of cookies in the first window. 1961 */ 1962 if (e == DDI_DMA_MAPPED) { 1963 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1964 *ccountp = sinfo->si_sgl_size; 1965 } else { 1966 hp->dmai_rflags |= DDI_DMA_PARTIAL; 1967 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 1968 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 1969 } 1970 *cookiep = dma->dp_cookies[0]; 1971 hp->dmai_cookie++; 1972 1973 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1974 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 1975 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1976 dma->dp_dma.dmao_size); 1977 return (e); 1978 } 1979 1980 1981 /* 1982 * rootnex_dma_unbindhdl() 1983 * called from ddi_dma_unbind_handle() 1984 */ 1985 /*ARGSUSED*/ 1986 static int 1987 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 1988 ddi_dma_handle_t handle) 1989 { 1990 ddi_dma_impl_t *hp; 1991 rootnex_dma_t *dma; 1992 int e; 1993 1994 1995 hp = (ddi_dma_impl_t *)handle; 1996 dma = (rootnex_dma_t *)hp->dmai_private; 1997 1998 /* make sure the buffer wasn't free'd before calling unbind */ 1999 if (rootnex_unbind_verify_buffer) { 2000 e = rootnex_verify_buffer(dma); 2001 if (e != DDI_SUCCESS) { 2002 ASSERT(0); 2003 return (DDI_FAILURE); 2004 } 2005 } 2006 2007 /* sync the current window before unbinding the buffer */ 2008 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2009 (hp->dmai_rflags & DDI_DMA_READ)) { 2010 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2011 DDI_DMA_SYNC_FORCPU); 2012 } 2013 2014 /* 2015 * cleanup and copy buffer or window state. if we didn't use the copy 2016 * buffer or windows, there won't be much to do :-) 2017 */ 2018 rootnex_teardown_copybuf(dma); 2019 rootnex_teardown_windows(dma); 2020 2021 /* 2022 * If we had to allocate space to for the worse case sgl (it didn't 2023 * fit into our pre-allocate buffer), free that up now 2024 */ 2025 if (dma->dp_need_to_free_cookie) { 2026 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2027 } 2028 2029 /* 2030 * clean up the handle so it's ready for the next bind (i.e. if the 2031 * handle is reused). 2032 */ 2033 rootnex_clean_dmahdl(hp); 2034 2035 if (rootnex_state->r_dvma_call_list_id) 2036 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2037 2038 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2039 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2040 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2041 2042 return (DDI_SUCCESS); 2043 } 2044 2045 2046 /* 2047 * rootnex_verify_buffer() 2048 * verify buffer wasn't free'd 2049 */ 2050 static int 2051 rootnex_verify_buffer(rootnex_dma_t *dma) 2052 { 2053 peekpoke_ctlops_t peek; 2054 page_t **pplist; 2055 caddr_t vaddr; 2056 uint_t pcnt; 2057 uint_t poff; 2058 page_t *pp; 2059 uint8_t b; 2060 int i; 2061 int e; 2062 2063 2064 /* Figure out how many pages this buffer occupies */ 2065 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2066 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2067 } else { 2068 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2069 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2070 } 2071 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2072 2073 switch (dma->dp_dma.dmao_type) { 2074 case DMA_OTYP_PAGES: 2075 /* 2076 * for a linked list of pp's walk through them to make sure 2077 * they're locked and not free. 2078 */ 2079 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2080 for (i = 0; i < pcnt; i++) { 2081 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2082 return (DDI_FAILURE); 2083 } 2084 pp = pp->p_next; 2085 } 2086 break; 2087 2088 case DMA_OTYP_VADDR: 2089 case DMA_OTYP_BUFVADDR: 2090 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2091 /* 2092 * for an array of pp's walk through them to make sure they're 2093 * not free. It's possible that they may not be locked. 2094 */ 2095 if (pplist) { 2096 for (i = 0; i < pcnt; i++) { 2097 if (PP_ISFREE(pplist[i])) { 2098 return (DDI_FAILURE); 2099 } 2100 } 2101 2102 /* For a virtual address, try to peek at each page */ 2103 } else { 2104 if (dma->dp_sglinfo.si_asp == &kas) { 2105 bzero(&peek, sizeof (peekpoke_ctlops_t)); 2106 peek.host_addr = (uintptr_t)&b; 2107 peek.size = sizeof (uint8_t); 2108 peek.dev_addr = (uintptr_t)vaddr; 2109 for (i = 0; i < pcnt; i++) { 2110 e = rootnex_ctlops_peek(&peek, &b); 2111 if (e != DDI_SUCCESS) { 2112 return (DDI_FAILURE); 2113 } 2114 peek.dev_addr += MMU_PAGESIZE; 2115 } 2116 } 2117 } 2118 break; 2119 2120 default: 2121 ASSERT(0); 2122 break; 2123 } 2124 2125 return (DDI_SUCCESS); 2126 } 2127 2128 2129 /* 2130 * rootnex_clean_dmahdl() 2131 * Clean the dma handle. This should be called on a handle alloc and an 2132 * unbind handle. Set the handle state to the default settings. 2133 */ 2134 static void 2135 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2136 { 2137 rootnex_dma_t *dma; 2138 2139 2140 dma = (rootnex_dma_t *)hp->dmai_private; 2141 2142 hp->dmai_nwin = 0; 2143 dma->dp_current_cookie = 0; 2144 dma->dp_copybuf_size = 0; 2145 dma->dp_window = NULL; 2146 dma->dp_cbaddr = NULL; 2147 dma->dp_inuse = B_FALSE; 2148 dma->dp_need_to_free_cookie = B_FALSE; 2149 dma->dp_need_to_free_window = B_FALSE; 2150 dma->dp_partial_required = B_FALSE; 2151 dma->dp_trim_required = B_FALSE; 2152 dma->dp_sglinfo.si_copybuf_req = 0; 2153 #if !defined(__amd64) 2154 dma->dp_cb_remaping = B_FALSE; 2155 dma->dp_kva = NULL; 2156 #endif 2157 2158 /* FMA related initialization */ 2159 hp->dmai_fault = 0; 2160 hp->dmai_fault_check = NULL; 2161 hp->dmai_fault_notify = NULL; 2162 hp->dmai_error.err_ena = 0; 2163 hp->dmai_error.err_status = DDI_FM_OK; 2164 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2165 hp->dmai_error.err_ontrap = NULL; 2166 hp->dmai_error.err_fep = NULL; 2167 } 2168 2169 2170 /* 2171 * rootnex_valid_alloc_parms() 2172 * Called in ddi_dma_alloc_handle path to validate its parameters. 2173 */ 2174 static int 2175 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2176 { 2177 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2178 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2179 (attr->dma_attr_granular > MMU_PAGESIZE) || 2180 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2181 return (DDI_DMA_BADATTR); 2182 } 2183 2184 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2185 return (DDI_DMA_BADATTR); 2186 } 2187 2188 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2189 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2190 attr->dma_attr_sgllen <= 0) { 2191 return (DDI_DMA_BADATTR); 2192 } 2193 2194 /* We should be able to DMA into every byte offset in a page */ 2195 if (maxsegmentsize < MMU_PAGESIZE) { 2196 return (DDI_DMA_BADATTR); 2197 } 2198 2199 return (DDI_SUCCESS); 2200 } 2201 2202 2203 /* 2204 * rootnex_valid_bind_parms() 2205 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2206 */ 2207 /* ARGSUSED */ 2208 static int 2209 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2210 { 2211 #if !defined(__amd64) 2212 /* 2213 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2214 * we can track the offset for the obsoleted interfaces. 2215 */ 2216 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2217 return (DDI_DMA_TOOBIG); 2218 } 2219 #endif 2220 2221 return (DDI_SUCCESS); 2222 } 2223 2224 2225 /* 2226 * rootnex_get_sgl() 2227 * Called in bind fastpath to get the sgl. Most of this will be replaced 2228 * with a call to the vm layer when vm2.0 comes around... 2229 */ 2230 static void 2231 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2232 rootnex_sglinfo_t *sglinfo) 2233 { 2234 ddi_dma_atyp_t buftype; 2235 uint64_t last_page; 2236 uint64_t offset; 2237 uint64_t addrhi; 2238 uint64_t addrlo; 2239 uint64_t maxseg; 2240 page_t **pplist; 2241 uint64_t paddr; 2242 uint32_t psize; 2243 uint32_t size; 2244 caddr_t vaddr; 2245 uint_t pcnt; 2246 page_t *pp; 2247 uint_t cnt; 2248 2249 2250 /* shortcuts */ 2251 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2252 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2253 maxseg = sglinfo->si_max_cookie_size; 2254 buftype = dmar_object->dmao_type; 2255 addrhi = sglinfo->si_max_addr; 2256 addrlo = sglinfo->si_min_addr; 2257 size = dmar_object->dmao_size; 2258 2259 pcnt = 0; 2260 cnt = 0; 2261 2262 /* 2263 * if we were passed down a linked list of pages, i.e. pointer to 2264 * page_t, use this to get our physical address and buf offset. 2265 */ 2266 if (buftype == DMA_OTYP_PAGES) { 2267 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2268 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2269 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2270 MMU_PAGEOFFSET; 2271 paddr = ptob64(pp->p_pagenum) + offset; 2272 psize = MIN(size, (MMU_PAGESIZE - offset)); 2273 pp = pp->p_next; 2274 sglinfo->si_asp = NULL; 2275 2276 /* 2277 * We weren't passed down a linked list of pages, but if we were passed 2278 * down an array of pages, use this to get our physical address and buf 2279 * offset. 2280 */ 2281 } else if (pplist != NULL) { 2282 ASSERT((buftype == DMA_OTYP_VADDR) || 2283 (buftype == DMA_OTYP_BUFVADDR)); 2284 2285 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2286 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2287 if (sglinfo->si_asp == NULL) { 2288 sglinfo->si_asp = &kas; 2289 } 2290 2291 ASSERT(!PP_ISFREE(pplist[pcnt])); 2292 paddr = ptob64(pplist[pcnt]->p_pagenum); 2293 paddr += offset; 2294 psize = MIN(size, (MMU_PAGESIZE - offset)); 2295 pcnt++; 2296 2297 /* 2298 * All we have is a virtual address, we'll need to call into the VM 2299 * to get the physical address. 2300 */ 2301 } else { 2302 ASSERT((buftype == DMA_OTYP_VADDR) || 2303 (buftype == DMA_OTYP_BUFVADDR)); 2304 2305 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2306 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2307 if (sglinfo->si_asp == NULL) { 2308 sglinfo->si_asp = &kas; 2309 } 2310 2311 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2312 paddr += offset; 2313 psize = MIN(size, (MMU_PAGESIZE - offset)); 2314 vaddr += psize; 2315 } 2316 2317 /* 2318 * Setup the first cookie with the physical address of the page and the 2319 * size of the page (which takes into account the initial offset into 2320 * the page. 2321 */ 2322 sgl[cnt].dmac_laddress = paddr; 2323 sgl[cnt].dmac_size = psize; 2324 sgl[cnt].dmac_type = 0; 2325 2326 /* 2327 * Save away the buffer offset into the page. We'll need this later in 2328 * the copy buffer code to help figure out the page index within the 2329 * buffer and the offset into the current page. 2330 */ 2331 sglinfo->si_buf_offset = offset; 2332 2333 /* 2334 * If the DMA engine can't reach the physical address, increase how 2335 * much copy buffer we need. We always increase by pagesize so we don't 2336 * have to worry about converting offsets. Set a flag in the cookies 2337 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2338 * last cookie, go to the next cookie (since we separate each page which 2339 * uses the copy buffer in case the copy buffer is not physically 2340 * contiguous. 2341 */ 2342 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2343 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2344 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2345 if ((cnt + 1) < sglinfo->si_max_pages) { 2346 cnt++; 2347 sgl[cnt].dmac_laddress = 0; 2348 sgl[cnt].dmac_size = 0; 2349 sgl[cnt].dmac_type = 0; 2350 } 2351 } 2352 2353 /* 2354 * save this page's physical address so we can figure out if the next 2355 * page is physically contiguous. Keep decrementing size until we are 2356 * done with the buffer. 2357 */ 2358 last_page = paddr & MMU_PAGEMASK; 2359 size -= psize; 2360 2361 while (size > 0) { 2362 /* Get the size for this page (i.e. partial or full page) */ 2363 psize = MIN(size, MMU_PAGESIZE); 2364 2365 if (buftype == DMA_OTYP_PAGES) { 2366 /* get the paddr from the page_t */ 2367 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2368 paddr = ptob64(pp->p_pagenum); 2369 pp = pp->p_next; 2370 } else if (pplist != NULL) { 2371 /* index into the array of page_t's to get the paddr */ 2372 ASSERT(!PP_ISFREE(pplist[pcnt])); 2373 paddr = ptob64(pplist[pcnt]->p_pagenum); 2374 pcnt++; 2375 } else { 2376 /* call into the VM to get the paddr */ 2377 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, 2378 vaddr)); 2379 vaddr += psize; 2380 } 2381 2382 /* check to see if this page needs the copy buffer */ 2383 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2384 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2385 2386 /* 2387 * if there is something in the current cookie, go to 2388 * the next one. We only want one page in a cookie which 2389 * uses the copybuf since the copybuf doesn't have to 2390 * be physically contiguous. 2391 */ 2392 if (sgl[cnt].dmac_size != 0) { 2393 cnt++; 2394 } 2395 sgl[cnt].dmac_laddress = paddr; 2396 sgl[cnt].dmac_size = psize; 2397 #if defined(__amd64) 2398 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2399 #else 2400 /* 2401 * save the buf offset for 32-bit kernel. used in the 2402 * obsoleted interfaces. 2403 */ 2404 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2405 (dmar_object->dmao_size - size); 2406 #endif 2407 /* if this isn't the last cookie, go to the next one */ 2408 if ((cnt + 1) < sglinfo->si_max_pages) { 2409 cnt++; 2410 sgl[cnt].dmac_laddress = 0; 2411 sgl[cnt].dmac_size = 0; 2412 sgl[cnt].dmac_type = 0; 2413 } 2414 2415 /* 2416 * this page didn't need the copy buffer, if it's not physically 2417 * contiguous, or it would put us over a segment boundary, or it 2418 * puts us over the max cookie size, or the current sgl doesn't 2419 * have anything in it. 2420 */ 2421 } else if (((last_page + MMU_PAGESIZE) != paddr) || 2422 !(paddr & sglinfo->si_segmask) || 2423 ((sgl[cnt].dmac_size + psize) > maxseg) || 2424 (sgl[cnt].dmac_size == 0)) { 2425 /* 2426 * if we're not already in a new cookie, go to the next 2427 * cookie. 2428 */ 2429 if (sgl[cnt].dmac_size != 0) { 2430 cnt++; 2431 } 2432 2433 /* save the cookie information */ 2434 sgl[cnt].dmac_laddress = paddr; 2435 sgl[cnt].dmac_size = psize; 2436 #if defined(__amd64) 2437 sgl[cnt].dmac_type = 0; 2438 #else 2439 /* 2440 * save the buf offset for 32-bit kernel. used in the 2441 * obsoleted interfaces. 2442 */ 2443 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2444 #endif 2445 2446 /* 2447 * this page didn't need the copy buffer, it is physically 2448 * contiguous with the last page, and it's <= the max cookie 2449 * size. 2450 */ 2451 } else { 2452 sgl[cnt].dmac_size += psize; 2453 2454 /* 2455 * if this exactly == the maximum cookie size, and 2456 * it isn't the last cookie, go to the next cookie. 2457 */ 2458 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2459 ((cnt + 1) < sglinfo->si_max_pages)) { 2460 cnt++; 2461 sgl[cnt].dmac_laddress = 0; 2462 sgl[cnt].dmac_size = 0; 2463 sgl[cnt].dmac_type = 0; 2464 } 2465 } 2466 2467 /* 2468 * save this page's physical address so we can figure out if the 2469 * next page is physically contiguous. Keep decrementing size 2470 * until we are done with the buffer. 2471 */ 2472 last_page = paddr; 2473 size -= psize; 2474 } 2475 2476 /* we're done, save away how many cookies the sgl has */ 2477 if (sgl[cnt].dmac_size == 0) { 2478 ASSERT(cnt < sglinfo->si_max_pages); 2479 sglinfo->si_sgl_size = cnt; 2480 } else { 2481 sglinfo->si_sgl_size = cnt + 1; 2482 } 2483 } 2484 2485 2486 /* 2487 * rootnex_bind_slowpath() 2488 * Call in the bind path if the calling driver can't use the sgl without 2489 * modifying it. We either need to use the copy buffer and/or we will end up 2490 * with a partial bind. 2491 */ 2492 static int 2493 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2494 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2495 { 2496 rootnex_sglinfo_t *sinfo; 2497 rootnex_window_t *window; 2498 ddi_dma_cookie_t *cookie; 2499 size_t copybuf_used; 2500 size_t dmac_size; 2501 boolean_t partial; 2502 off_t cur_offset; 2503 page_t *cur_pp; 2504 major_t mnum; 2505 int e; 2506 int i; 2507 2508 2509 sinfo = &dma->dp_sglinfo; 2510 copybuf_used = 0; 2511 partial = B_FALSE; 2512 2513 /* 2514 * If we're using the copybuf, set the copybuf state in dma struct. 2515 * Needs to be first since it sets the copy buffer size. 2516 */ 2517 if (sinfo->si_copybuf_req != 0) { 2518 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2519 if (e != DDI_SUCCESS) { 2520 return (e); 2521 } 2522 } else { 2523 dma->dp_copybuf_size = 0; 2524 } 2525 2526 /* 2527 * Figure out if we need to do a partial mapping. If so, figure out 2528 * if we need to trim the buffers when we munge the sgl. 2529 */ 2530 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2531 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2532 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2533 dma->dp_partial_required = B_TRUE; 2534 if (attr->dma_attr_granular != 1) { 2535 dma->dp_trim_required = B_TRUE; 2536 } 2537 } else { 2538 dma->dp_partial_required = B_FALSE; 2539 dma->dp_trim_required = B_FALSE; 2540 } 2541 2542 /* If we need to do a partial bind, make sure the driver supports it */ 2543 if (dma->dp_partial_required && 2544 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2545 2546 mnum = ddi_driver_major(dma->dp_dip); 2547 /* 2548 * patchable which allows us to print one warning per major 2549 * number. 2550 */ 2551 if ((rootnex_bind_warn) && 2552 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2553 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2554 cmn_err(CE_WARN, "!%s: coding error detected, the " 2555 "driver is using ddi_dma_attr(9S) incorrectly. " 2556 "There is a small risk of data corruption in " 2557 "particular with large I/Os. The driver should be " 2558 "replaced with a corrected version for proper " 2559 "system operation. To disable this warning, add " 2560 "'set rootnex:rootnex_bind_warn=0' to " 2561 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2562 } 2563 return (DDI_DMA_TOOBIG); 2564 } 2565 2566 /* 2567 * we might need multiple windows, setup state to handle them. In this 2568 * code path, we will have at least one window. 2569 */ 2570 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2571 if (e != DDI_SUCCESS) { 2572 rootnex_teardown_copybuf(dma); 2573 return (e); 2574 } 2575 2576 window = &dma->dp_window[0]; 2577 cookie = &dma->dp_cookies[0]; 2578 cur_offset = 0; 2579 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2580 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2581 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2582 } 2583 2584 /* loop though all the cookies we got back from get_sgl() */ 2585 for (i = 0; i < sinfo->si_sgl_size; i++) { 2586 /* 2587 * If we're using the copy buffer, check this cookie and setup 2588 * its associated copy buffer state. If this cookie uses the 2589 * copy buffer, make sure we sync this window during dma_sync. 2590 */ 2591 if (dma->dp_copybuf_size > 0) { 2592 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2593 cur_offset, ©buf_used, &cur_pp); 2594 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2595 window->wd_dosync = B_TRUE; 2596 } 2597 } 2598 2599 /* 2600 * save away the cookie size, since it could be modified in 2601 * the windowing code. 2602 */ 2603 dmac_size = cookie->dmac_size; 2604 2605 /* if we went over max copybuf size */ 2606 if (dma->dp_copybuf_size && 2607 (copybuf_used > dma->dp_copybuf_size)) { 2608 partial = B_TRUE; 2609 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2610 cookie, cur_offset, ©buf_used); 2611 if (e != DDI_SUCCESS) { 2612 rootnex_teardown_copybuf(dma); 2613 rootnex_teardown_windows(dma); 2614 return (e); 2615 } 2616 2617 /* 2618 * if the coookie uses the copy buffer, make sure the 2619 * new window we just moved to is set to sync. 2620 */ 2621 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2622 window->wd_dosync = B_TRUE; 2623 } 2624 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2625 dma->dp_dip); 2626 2627 /* if the cookie cnt == max sgllen, move to the next window */ 2628 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2629 partial = B_TRUE; 2630 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2631 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2632 cookie, attr, cur_offset); 2633 if (e != DDI_SUCCESS) { 2634 rootnex_teardown_copybuf(dma); 2635 rootnex_teardown_windows(dma); 2636 return (e); 2637 } 2638 2639 /* 2640 * if the coookie uses the copy buffer, make sure the 2641 * new window we just moved to is set to sync. 2642 */ 2643 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2644 window->wd_dosync = B_TRUE; 2645 } 2646 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2647 dma->dp_dip); 2648 2649 /* else if we will be over maxxfer */ 2650 } else if ((window->wd_size + dmac_size) > 2651 dma->dp_maxxfer) { 2652 partial = B_TRUE; 2653 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2654 cookie); 2655 if (e != DDI_SUCCESS) { 2656 rootnex_teardown_copybuf(dma); 2657 rootnex_teardown_windows(dma); 2658 return (e); 2659 } 2660 2661 /* 2662 * if the coookie uses the copy buffer, make sure the 2663 * new window we just moved to is set to sync. 2664 */ 2665 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2666 window->wd_dosync = B_TRUE; 2667 } 2668 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2669 dma->dp_dip); 2670 2671 /* else this cookie fits in the current window */ 2672 } else { 2673 window->wd_cookie_cnt++; 2674 window->wd_size += dmac_size; 2675 } 2676 2677 /* track our offset into the buffer, go to the next cookie */ 2678 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2679 ASSERT(cookie->dmac_size <= dmac_size); 2680 cur_offset += dmac_size; 2681 cookie++; 2682 } 2683 2684 /* if we ended up with a zero sized window in the end, clean it up */ 2685 if (window->wd_size == 0) { 2686 hp->dmai_nwin--; 2687 window--; 2688 } 2689 2690 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2691 2692 if (!partial) { 2693 return (DDI_DMA_MAPPED); 2694 } 2695 2696 ASSERT(dma->dp_partial_required); 2697 return (DDI_DMA_PARTIAL_MAP); 2698 } 2699 2700 2701 /* 2702 * rootnex_setup_copybuf() 2703 * Called in bind slowpath. Figures out if we're going to use the copy 2704 * buffer, and if we do, sets up the basic state to handle it. 2705 */ 2706 static int 2707 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2708 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2709 { 2710 rootnex_sglinfo_t *sinfo; 2711 ddi_dma_attr_t lattr; 2712 size_t max_copybuf; 2713 int cansleep; 2714 int e; 2715 #if !defined(__amd64) 2716 int vmflag; 2717 #endif 2718 2719 2720 sinfo = &dma->dp_sglinfo; 2721 2722 /* 2723 * read this first so it's consistent through the routine so we can 2724 * patch it on the fly. 2725 */ 2726 max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK; 2727 2728 /* We need to call into the rootnex on ddi_dma_sync() */ 2729 hp->dmai_rflags &= ~DMP_NOSYNC; 2730 2731 /* make sure the copybuf size <= the max size */ 2732 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2733 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2734 2735 #if !defined(__amd64) 2736 /* 2737 * if we don't have kva space to copy to/from, allocate the KVA space 2738 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2739 * the 64-bit kernel. 2740 */ 2741 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2742 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2743 2744 /* convert the sleep flags */ 2745 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2746 vmflag = VM_SLEEP; 2747 } else { 2748 vmflag = VM_NOSLEEP; 2749 } 2750 2751 /* allocate Kernel VA space that we can bcopy to/from */ 2752 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2753 vmflag); 2754 if (dma->dp_kva == NULL) { 2755 return (DDI_DMA_NORESOURCES); 2756 } 2757 } 2758 #endif 2759 2760 /* convert the sleep flags */ 2761 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2762 cansleep = 1; 2763 } else { 2764 cansleep = 0; 2765 } 2766 2767 /* 2768 * Allocated the actual copy buffer. This needs to fit within the DMA 2769 * engines limits, so we can't use kmem_alloc... 2770 */ 2771 lattr = *attr; 2772 lattr.dma_attr_align = MMU_PAGESIZE; 2773 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2774 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2775 if (e != DDI_SUCCESS) { 2776 #if !defined(__amd64) 2777 if (dma->dp_kva != NULL) { 2778 vmem_free(heap_arena, dma->dp_kva, 2779 dma->dp_copybuf_size); 2780 } 2781 #endif 2782 return (DDI_DMA_NORESOURCES); 2783 } 2784 2785 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2786 size_t, dma->dp_copybuf_size); 2787 2788 return (DDI_SUCCESS); 2789 } 2790 2791 2792 /* 2793 * rootnex_setup_windows() 2794 * Called in bind slowpath to setup the window state. We always have windows 2795 * in the slowpath. Even if the window count = 1. 2796 */ 2797 static int 2798 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2799 ddi_dma_attr_t *attr, int kmflag) 2800 { 2801 rootnex_window_t *windowp; 2802 rootnex_sglinfo_t *sinfo; 2803 size_t copy_state_size; 2804 size_t win_state_size; 2805 size_t state_available; 2806 size_t space_needed; 2807 uint_t copybuf_win; 2808 uint_t maxxfer_win; 2809 size_t space_used; 2810 uint_t sglwin; 2811 2812 2813 sinfo = &dma->dp_sglinfo; 2814 2815 dma->dp_current_win = 0; 2816 hp->dmai_nwin = 0; 2817 2818 /* If we don't need to do a partial, we only have one window */ 2819 if (!dma->dp_partial_required) { 2820 dma->dp_max_win = 1; 2821 2822 /* 2823 * we need multiple windows, need to figure out the worse case number 2824 * of windows. 2825 */ 2826 } else { 2827 /* 2828 * if we need windows because we need more copy buffer that 2829 * we allow, the worse case number of windows we could need 2830 * here would be (copybuf space required / copybuf space that 2831 * we have) plus one for remainder, and plus 2 to handle the 2832 * extra pages on the trim for the first and last pages of the 2833 * buffer (a page is the minimum window size so under the right 2834 * attr settings, you could have a window for each page). 2835 * The last page will only be hit here if the size is not a 2836 * multiple of the granularity (which theoretically shouldn't 2837 * be the case but never has been enforced, so we could have 2838 * broken things without it). 2839 */ 2840 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2841 ASSERT(dma->dp_copybuf_size > 0); 2842 copybuf_win = (sinfo->si_copybuf_req / 2843 dma->dp_copybuf_size) + 1 + 2; 2844 } else { 2845 copybuf_win = 0; 2846 } 2847 2848 /* 2849 * if we need windows because we have more cookies than the H/W 2850 * can handle, the number of windows we would need here would 2851 * be (cookie count / cookies count H/W supports) plus one for 2852 * remainder, and plus 2 to handle the extra pages on the trim 2853 * (see above comment about trim) 2854 */ 2855 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2856 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2857 + 1) + 2; 2858 } else { 2859 sglwin = 0; 2860 } 2861 2862 /* 2863 * if we need windows because we're binding more memory than the 2864 * H/W can transfer at once, the number of windows we would need 2865 * here would be (xfer count / max xfer H/W supports) plus one 2866 * for remainder, and plus 2 to handle the extra pages on the 2867 * trim (see above comment about trim) 2868 */ 2869 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 2870 maxxfer_win = (dma->dp_dma.dmao_size / 2871 dma->dp_maxxfer) + 1 + 2; 2872 } else { 2873 maxxfer_win = 0; 2874 } 2875 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 2876 ASSERT(dma->dp_max_win > 0); 2877 } 2878 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 2879 2880 /* 2881 * Get space for window and potential copy buffer state. Before we 2882 * go and allocate memory, see if we can get away with using what's 2883 * left in the pre-allocted state or the dynamically allocated sgl. 2884 */ 2885 space_used = (uintptr_t)(sinfo->si_sgl_size * 2886 sizeof (ddi_dma_cookie_t)); 2887 2888 /* if we dynamically allocated space for the cookies */ 2889 if (dma->dp_need_to_free_cookie) { 2890 /* if we have more space in the pre-allocted buffer, use it */ 2891 ASSERT(space_used <= dma->dp_cookie_size); 2892 if ((dma->dp_cookie_size - space_used) <= 2893 rootnex_state->r_prealloc_size) { 2894 state_available = rootnex_state->r_prealloc_size; 2895 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 2896 2897 /* 2898 * else, we have more free space in the dynamically allocated 2899 * buffer, i.e. the buffer wasn't worse case fragmented so we 2900 * didn't need a lot of cookies. 2901 */ 2902 } else { 2903 state_available = dma->dp_cookie_size - space_used; 2904 windowp = (rootnex_window_t *) 2905 &dma->dp_cookies[sinfo->si_sgl_size]; 2906 } 2907 2908 /* we used the pre-alloced buffer */ 2909 } else { 2910 ASSERT(space_used <= rootnex_state->r_prealloc_size); 2911 state_available = rootnex_state->r_prealloc_size - space_used; 2912 windowp = (rootnex_window_t *) 2913 &dma->dp_cookies[sinfo->si_sgl_size]; 2914 } 2915 2916 /* 2917 * figure out how much state we need to track the copy buffer. Add an 2918 * addition 8 bytes for pointer alignemnt later. 2919 */ 2920 if (dma->dp_copybuf_size > 0) { 2921 copy_state_size = sinfo->si_max_pages * 2922 sizeof (rootnex_pgmap_t); 2923 } else { 2924 copy_state_size = 0; 2925 } 2926 /* add an additional 8 bytes for pointer alignment */ 2927 space_needed = win_state_size + copy_state_size + 0x8; 2928 2929 /* if we have enough space already, use it */ 2930 if (state_available >= space_needed) { 2931 dma->dp_window = windowp; 2932 dma->dp_need_to_free_window = B_FALSE; 2933 2934 /* not enough space, need to allocate more. */ 2935 } else { 2936 dma->dp_window = kmem_alloc(space_needed, kmflag); 2937 if (dma->dp_window == NULL) { 2938 return (DDI_DMA_NORESOURCES); 2939 } 2940 dma->dp_need_to_free_window = B_TRUE; 2941 dma->dp_window_size = space_needed; 2942 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 2943 dma->dp_dip, size_t, space_needed); 2944 } 2945 2946 /* 2947 * we allocate copy buffer state and window state at the same time. 2948 * setup our copy buffer state pointers. Make sure it's aligned. 2949 */ 2950 if (dma->dp_copybuf_size > 0) { 2951 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 2952 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 2953 2954 #if !defined(__amd64) 2955 /* 2956 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 2957 * false/NULL. Should be quicker to bzero vs loop and set. 2958 */ 2959 bzero(dma->dp_pgmap, copy_state_size); 2960 #endif 2961 } else { 2962 dma->dp_pgmap = NULL; 2963 } 2964 2965 return (DDI_SUCCESS); 2966 } 2967 2968 2969 /* 2970 * rootnex_teardown_copybuf() 2971 * cleans up after rootnex_setup_copybuf() 2972 */ 2973 static void 2974 rootnex_teardown_copybuf(rootnex_dma_t *dma) 2975 { 2976 #if !defined(__amd64) 2977 int i; 2978 2979 /* 2980 * if we allocated kernel heap VMEM space, go through all the pages and 2981 * map out any of the ones that we're mapped into the kernel heap VMEM 2982 * arena. Then free the VMEM space. 2983 */ 2984 if (dma->dp_kva != NULL) { 2985 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 2986 if (dma->dp_pgmap[i].pm_mapped) { 2987 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 2988 MMU_PAGESIZE, HAT_UNLOAD); 2989 dma->dp_pgmap[i].pm_mapped = B_FALSE; 2990 } 2991 } 2992 2993 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 2994 } 2995 2996 #endif 2997 2998 /* if we allocated a copy buffer, free it */ 2999 if (dma->dp_cbaddr != NULL) { 3000 i_ddi_mem_free(dma->dp_cbaddr, 0); 3001 } 3002 } 3003 3004 3005 /* 3006 * rootnex_teardown_windows() 3007 * cleans up after rootnex_setup_windows() 3008 */ 3009 static void 3010 rootnex_teardown_windows(rootnex_dma_t *dma) 3011 { 3012 /* 3013 * if we had to allocate window state on the last bind (because we 3014 * didn't have enough pre-allocated space in the handle), free it. 3015 */ 3016 if (dma->dp_need_to_free_window) { 3017 kmem_free(dma->dp_window, dma->dp_window_size); 3018 } 3019 } 3020 3021 3022 /* 3023 * rootnex_init_win() 3024 * Called in bind slow path during creation of a new window. Initializes 3025 * window state to default values. 3026 */ 3027 /*ARGSUSED*/ 3028 static void 3029 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3030 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3031 { 3032 hp->dmai_nwin++; 3033 window->wd_dosync = B_FALSE; 3034 window->wd_offset = cur_offset; 3035 window->wd_size = 0; 3036 window->wd_first_cookie = cookie; 3037 window->wd_cookie_cnt = 0; 3038 window->wd_trim.tr_trim_first = B_FALSE; 3039 window->wd_trim.tr_trim_last = B_FALSE; 3040 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3041 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3042 #if !defined(__amd64) 3043 window->wd_remap_copybuf = dma->dp_cb_remaping; 3044 #endif 3045 } 3046 3047 3048 /* 3049 * rootnex_setup_cookie() 3050 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3051 * the sgl uses the copy buffer, we need to go through each cookie, figure 3052 * out if it uses the copy buffer, and if it does, save away everything we'll 3053 * need during sync. 3054 */ 3055 static void 3056 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3057 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3058 page_t **cur_pp) 3059 { 3060 boolean_t copybuf_sz_power_2; 3061 rootnex_sglinfo_t *sinfo; 3062 uint_t pidx; 3063 uint_t pcnt; 3064 off_t poff; 3065 #if defined(__amd64) 3066 pfn_t pfn; 3067 #else 3068 page_t **pplist; 3069 #endif 3070 3071 sinfo = &dma->dp_sglinfo; 3072 3073 /* 3074 * Calculate the page index relative to the start of the buffer. The 3075 * index to the current page for our buffer is the offset into the 3076 * first page of the buffer plus our current offset into the buffer 3077 * itself, shifted of course... 3078 */ 3079 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3080 ASSERT(pidx < sinfo->si_max_pages); 3081 3082 /* if this cookie uses the copy buffer */ 3083 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3084 /* 3085 * NOTE: we know that since this cookie uses the copy buffer, it 3086 * is <= MMU_PAGESIZE. 3087 */ 3088 3089 /* 3090 * get the offset into the page. For the 64-bit kernel, get the 3091 * pfn which we'll use with seg kpm. 3092 */ 3093 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3094 #if defined(__amd64) 3095 pfn = cookie->_dmu._dmac_ll >> MMU_PAGESHIFT; 3096 #endif 3097 3098 /* figure out if the copybuf size is a power of 2 */ 3099 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3100 copybuf_sz_power_2 = B_FALSE; 3101 } else { 3102 copybuf_sz_power_2 = B_TRUE; 3103 } 3104 3105 /* This page uses the copy buffer */ 3106 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3107 3108 /* 3109 * save the copy buffer KVA that we'll use with this page. 3110 * if we still fit within the copybuf, it's a simple add. 3111 * otherwise, we need to wrap over using & or % accordingly. 3112 */ 3113 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3114 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3115 *copybuf_used; 3116 } else { 3117 if (copybuf_sz_power_2) { 3118 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3119 (uintptr_t)dma->dp_cbaddr + 3120 (*copybuf_used & 3121 (dma->dp_copybuf_size - 1))); 3122 } else { 3123 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3124 (uintptr_t)dma->dp_cbaddr + 3125 (*copybuf_used % dma->dp_copybuf_size)); 3126 } 3127 } 3128 3129 /* 3130 * over write the cookie physical address with the address of 3131 * the physical address of the copy buffer page that we will 3132 * use. 3133 */ 3134 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3135 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3136 3137 /* if we have a kernel VA, it's easy, just save that address */ 3138 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3139 (sinfo->si_asp == &kas)) { 3140 /* 3141 * save away the page aligned virtual address of the 3142 * driver buffer. Offsets are handled in the sync code. 3143 */ 3144 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3145 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3146 & MMU_PAGEMASK); 3147 #if !defined(__amd64) 3148 /* 3149 * we didn't need to, and will never need to map this 3150 * page. 3151 */ 3152 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3153 #endif 3154 3155 /* we don't have a kernel VA. We need one for the bcopy. */ 3156 } else { 3157 #if defined(__amd64) 3158 /* 3159 * for the 64-bit kernel, it's easy. We use seg kpm to 3160 * get a Kernel VA for the corresponding pfn. 3161 */ 3162 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3163 #else 3164 /* 3165 * for the 32-bit kernel, this is a pain. First we'll 3166 * save away the page_t or user VA for this page. This 3167 * is needed in rootnex_dma_win() when we switch to a 3168 * new window which requires us to re-map the copy 3169 * buffer. 3170 */ 3171 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3172 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3173 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3174 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3175 } else if (pplist != NULL) { 3176 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3177 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3178 } else { 3179 dma->dp_pgmap[pidx].pm_pp = NULL; 3180 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3181 (((uintptr_t) 3182 dmar_object->dmao_obj.virt_obj.v_addr + 3183 cur_offset) & MMU_PAGEMASK); 3184 } 3185 3186 /* 3187 * save away the page aligned virtual address which was 3188 * allocated from the kernel heap arena (taking into 3189 * account if we need more copy buffer than we alloced 3190 * and use multiple windows to handle this, i.e. &,%). 3191 * NOTE: there isn't and physical memory backing up this 3192 * virtual address space currently. 3193 */ 3194 if ((*copybuf_used + MMU_PAGESIZE) <= 3195 dma->dp_copybuf_size) { 3196 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3197 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3198 MMU_PAGEMASK); 3199 } else { 3200 if (copybuf_sz_power_2) { 3201 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3202 (((uintptr_t)dma->dp_kva + 3203 (*copybuf_used & 3204 (dma->dp_copybuf_size - 1))) & 3205 MMU_PAGEMASK); 3206 } else { 3207 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3208 (((uintptr_t)dma->dp_kva + 3209 (*copybuf_used % 3210 dma->dp_copybuf_size)) & 3211 MMU_PAGEMASK); 3212 } 3213 } 3214 3215 /* 3216 * if we haven't used up the available copy buffer yet, 3217 * map the kva to the physical page. 3218 */ 3219 if (!dma->dp_cb_remaping && ((*copybuf_used + 3220 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3221 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3222 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3223 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3224 dma->dp_pgmap[pidx].pm_kaddr); 3225 } else { 3226 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3227 sinfo->si_asp, 3228 dma->dp_pgmap[pidx].pm_kaddr); 3229 } 3230 3231 /* 3232 * we've used up the available copy buffer, this page 3233 * will have to be mapped during rootnex_dma_win() when 3234 * we switch to a new window which requires a re-map 3235 * the copy buffer. (32-bit kernel only) 3236 */ 3237 } else { 3238 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3239 } 3240 #endif 3241 /* go to the next page_t */ 3242 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3243 *cur_pp = (*cur_pp)->p_next; 3244 } 3245 } 3246 3247 /* add to the copy buffer count */ 3248 *copybuf_used += MMU_PAGESIZE; 3249 3250 /* 3251 * This cookie doesn't use the copy buffer. Walk through the pages this 3252 * cookie occupies to reflect this. 3253 */ 3254 } else { 3255 /* 3256 * figure out how many pages the cookie occupies. We need to 3257 * use the original page offset of the buffer and the cookies 3258 * offset in the buffer to do this. 3259 */ 3260 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3261 pcnt = mmu_btopr(cookie->dmac_size + poff); 3262 3263 while (pcnt > 0) { 3264 #if !defined(__amd64) 3265 /* 3266 * the 32-bit kernel doesn't have seg kpm, so we need 3267 * to map in the driver buffer (if it didn't come down 3268 * with a kernel VA) on the fly. Since this page doesn't 3269 * use the copy buffer, it's not, or will it ever, have 3270 * to be mapped in. 3271 */ 3272 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3273 #endif 3274 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3275 3276 /* 3277 * we need to update pidx and cur_pp or we'll loose 3278 * track of where we are. 3279 */ 3280 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3281 *cur_pp = (*cur_pp)->p_next; 3282 } 3283 pidx++; 3284 pcnt--; 3285 } 3286 } 3287 } 3288 3289 3290 /* 3291 * rootnex_sgllen_window_boundary() 3292 * Called in the bind slow path when the next cookie causes us to exceed (in 3293 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3294 * length supported by the DMA H/W. 3295 */ 3296 static int 3297 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3298 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3299 off_t cur_offset) 3300 { 3301 off_t new_offset; 3302 size_t trim_sz; 3303 off_t coffset; 3304 3305 3306 /* 3307 * if we know we'll never have to trim, it's pretty easy. Just move to 3308 * the next window and init it. We're done. 3309 */ 3310 if (!dma->dp_trim_required) { 3311 (*windowp)++; 3312 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3313 (*windowp)->wd_cookie_cnt++; 3314 (*windowp)->wd_size = cookie->dmac_size; 3315 return (DDI_SUCCESS); 3316 } 3317 3318 /* figure out how much we need to trim from the window */ 3319 ASSERT(attr->dma_attr_granular != 0); 3320 if (dma->dp_granularity_power_2) { 3321 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3322 } else { 3323 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3324 } 3325 3326 /* The window's a whole multiple of granularity. We're done */ 3327 if (trim_sz == 0) { 3328 (*windowp)++; 3329 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3330 (*windowp)->wd_cookie_cnt++; 3331 (*windowp)->wd_size = cookie->dmac_size; 3332 return (DDI_SUCCESS); 3333 } 3334 3335 /* 3336 * The window's not a whole multiple of granularity, since we know this 3337 * is due to the sgllen, we need to go back to the last cookie and trim 3338 * that one, add the left over part of the old cookie into the new 3339 * window, and then add in the new cookie into the new window. 3340 */ 3341 3342 /* 3343 * make sure the driver isn't making us do something bad... Trimming and 3344 * sgllen == 1 don't go together. 3345 */ 3346 if (attr->dma_attr_sgllen == 1) { 3347 return (DDI_DMA_NOMAPPING); 3348 } 3349 3350 /* 3351 * first, setup the current window to account for the trim. Need to go 3352 * back to the last cookie for this. 3353 */ 3354 cookie--; 3355 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3356 (*windowp)->wd_trim.tr_last_cookie = cookie; 3357 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3358 ASSERT(cookie->dmac_size > trim_sz); 3359 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3360 (*windowp)->wd_size -= trim_sz; 3361 3362 /* save the buffer offsets for the next window */ 3363 coffset = cookie->dmac_size - trim_sz; 3364 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3365 3366 /* 3367 * set this now in case this is the first window. all other cases are 3368 * set in dma_win() 3369 */ 3370 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3371 3372 /* 3373 * initialize the next window using what's left over in the previous 3374 * cookie. 3375 */ 3376 (*windowp)++; 3377 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3378 (*windowp)->wd_cookie_cnt++; 3379 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3380 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3381 (*windowp)->wd_trim.tr_first_size = trim_sz; 3382 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3383 (*windowp)->wd_dosync = B_TRUE; 3384 } 3385 3386 /* 3387 * now go back to the current cookie and add it to the new window. set 3388 * the new window size to the what was left over from the previous 3389 * cookie and what's in the current cookie. 3390 */ 3391 cookie++; 3392 (*windowp)->wd_cookie_cnt++; 3393 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3394 3395 /* 3396 * trim plus the next cookie could put us over maxxfer (a cookie can be 3397 * a max size of maxxfer). Handle that case. 3398 */ 3399 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3400 /* 3401 * maxxfer is already a whole multiple of granularity, and this 3402 * trim will be <= the previous trim (since a cookie can't be 3403 * larger than maxxfer). Make things simple here. 3404 */ 3405 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3406 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3407 (*windowp)->wd_trim.tr_last_cookie = cookie; 3408 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3409 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3410 (*windowp)->wd_size -= trim_sz; 3411 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3412 3413 /* save the buffer offsets for the next window */ 3414 coffset = cookie->dmac_size - trim_sz; 3415 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3416 3417 /* setup the next window */ 3418 (*windowp)++; 3419 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3420 (*windowp)->wd_cookie_cnt++; 3421 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3422 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3423 coffset; 3424 (*windowp)->wd_trim.tr_first_size = trim_sz; 3425 } 3426 3427 return (DDI_SUCCESS); 3428 } 3429 3430 3431 /* 3432 * rootnex_copybuf_window_boundary() 3433 * Called in bind slowpath when we get to a window boundary because we used 3434 * up all the copy buffer that we have. 3435 */ 3436 static int 3437 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3438 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3439 size_t *copybuf_used) 3440 { 3441 rootnex_sglinfo_t *sinfo; 3442 off_t new_offset; 3443 size_t trim_sz; 3444 off_t coffset; 3445 uint_t pidx; 3446 off_t poff; 3447 3448 3449 sinfo = &dma->dp_sglinfo; 3450 3451 /* 3452 * the copy buffer should be a whole multiple of page size. We know that 3453 * this cookie is <= MMU_PAGESIZE. 3454 */ 3455 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3456 3457 /* 3458 * from now on, all new windows in this bind need to be re-mapped during 3459 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3460 * space... 3461 */ 3462 #if !defined(__amd64) 3463 dma->dp_cb_remaping = B_TRUE; 3464 #endif 3465 3466 /* reset copybuf used */ 3467 *copybuf_used = 0; 3468 3469 /* 3470 * if we don't have to trim (since granularity is set to 1), go to the 3471 * next window and add the current cookie to it. We know the current 3472 * cookie uses the copy buffer since we're in this code path. 3473 */ 3474 if (!dma->dp_trim_required) { 3475 (*windowp)++; 3476 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3477 3478 /* Add this cookie to the new window */ 3479 (*windowp)->wd_cookie_cnt++; 3480 (*windowp)->wd_size += cookie->dmac_size; 3481 *copybuf_used += MMU_PAGESIZE; 3482 return (DDI_SUCCESS); 3483 } 3484 3485 /* 3486 * *** may need to trim, figure it out. 3487 */ 3488 3489 /* figure out how much we need to trim from the window */ 3490 if (dma->dp_granularity_power_2) { 3491 trim_sz = (*windowp)->wd_size & 3492 (hp->dmai_attr.dma_attr_granular - 1); 3493 } else { 3494 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3495 } 3496 3497 /* 3498 * if the window's a whole multiple of granularity, go to the next 3499 * window, init it, then add in the current cookie. We know the current 3500 * cookie uses the copy buffer since we're in this code path. 3501 */ 3502 if (trim_sz == 0) { 3503 (*windowp)++; 3504 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3505 3506 /* Add this cookie to the new window */ 3507 (*windowp)->wd_cookie_cnt++; 3508 (*windowp)->wd_size += cookie->dmac_size; 3509 *copybuf_used += MMU_PAGESIZE; 3510 return (DDI_SUCCESS); 3511 } 3512 3513 /* 3514 * *** We figured it out, we definitly need to trim 3515 */ 3516 3517 /* 3518 * make sure the driver isn't making us do something bad... 3519 * Trimming and sgllen == 1 don't go together. 3520 */ 3521 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3522 return (DDI_DMA_NOMAPPING); 3523 } 3524 3525 /* 3526 * first, setup the current window to account for the trim. Need to go 3527 * back to the last cookie for this. Some of the last cookie will be in 3528 * the current window, and some of the last cookie will be in the new 3529 * window. All of the current cookie will be in the new window. 3530 */ 3531 cookie--; 3532 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3533 (*windowp)->wd_trim.tr_last_cookie = cookie; 3534 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3535 ASSERT(cookie->dmac_size > trim_sz); 3536 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3537 (*windowp)->wd_size -= trim_sz; 3538 3539 /* 3540 * we're trimming the last cookie (not the current cookie). So that 3541 * last cookie may have or may not have been using the copy buffer ( 3542 * we know the cookie passed in uses the copy buffer since we're in 3543 * this code path). 3544 * 3545 * If the last cookie doesn't use the copy buffer, nothing special to 3546 * do. However, if it does uses the copy buffer, it will be both the 3547 * last page in the current window and the first page in the next 3548 * window. Since we are reusing the copy buffer (and KVA space on the 3549 * 32-bit kernel), this page will use the end of the copy buffer in the 3550 * current window, and the start of the copy buffer in the next window. 3551 * Track that info... The cookie physical address was already set to 3552 * the copy buffer physical address in setup_cookie.. 3553 */ 3554 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3555 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3556 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3557 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3558 (*windowp)->wd_trim.tr_last_pidx = pidx; 3559 (*windowp)->wd_trim.tr_last_cbaddr = 3560 dma->dp_pgmap[pidx].pm_cbaddr; 3561 #if !defined(__amd64) 3562 (*windowp)->wd_trim.tr_last_kaddr = 3563 dma->dp_pgmap[pidx].pm_kaddr; 3564 #endif 3565 } 3566 3567 /* save the buffer offsets for the next window */ 3568 coffset = cookie->dmac_size - trim_sz; 3569 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3570 3571 /* 3572 * set this now in case this is the first window. all other cases are 3573 * set in dma_win() 3574 */ 3575 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3576 3577 /* 3578 * initialize the next window using what's left over in the previous 3579 * cookie. 3580 */ 3581 (*windowp)++; 3582 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3583 (*windowp)->wd_cookie_cnt++; 3584 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3585 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3586 (*windowp)->wd_trim.tr_first_size = trim_sz; 3587 3588 /* 3589 * again, we're tracking if the last cookie uses the copy buffer. 3590 * read the comment above for more info on why we need to track 3591 * additional state. 3592 * 3593 * For the first cookie in the new window, we need reset the physical 3594 * address to DMA into to the start of the copy buffer plus any 3595 * initial page offset which may be present. 3596 */ 3597 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3598 (*windowp)->wd_dosync = B_TRUE; 3599 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3600 (*windowp)->wd_trim.tr_first_pidx = pidx; 3601 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3602 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3603 (*windowp)->wd_trim.tr_first_paddr = ptob64(hat_getpfnum( 3604 kas.a_hat, dma->dp_cbaddr)) + poff; 3605 #if !defined(__amd64) 3606 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3607 #endif 3608 /* account for the cookie copybuf usage in the new window */ 3609 *copybuf_used += MMU_PAGESIZE; 3610 3611 /* 3612 * every piece of code has to have a hack, and here is this 3613 * ones :-) 3614 * 3615 * There is a complex interaction between setup_cookie and the 3616 * copybuf window boundary. The complexity had to be in either 3617 * the maxxfer window, or the copybuf window, and I chose the 3618 * copybuf code. 3619 * 3620 * So in this code path, we have taken the last cookie, 3621 * virtually broken it in half due to the trim, and it happens 3622 * to use the copybuf which further complicates life. At the 3623 * same time, we have already setup the current cookie, which 3624 * is now wrong. More background info: the current cookie uses 3625 * the copybuf, so it is only a page long max. So we need to 3626 * fix the current cookies copy buffer address, physical 3627 * address, and kva for the 32-bit kernel. We due this by 3628 * bumping them by page size (of course, we can't due this on 3629 * the physical address since the copy buffer may not be 3630 * physically contiguous). 3631 */ 3632 cookie++; 3633 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3634 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3635 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3636 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3637 #if !defined(__amd64) 3638 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3639 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3640 #endif 3641 } else { 3642 /* go back to the current cookie */ 3643 cookie++; 3644 } 3645 3646 /* 3647 * add the current cookie to the new window. set the new window size to 3648 * the what was left over from the previous cookie and what's in the 3649 * current cookie. 3650 */ 3651 (*windowp)->wd_cookie_cnt++; 3652 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3653 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3654 3655 /* 3656 * we know that the cookie passed in always uses the copy buffer. We 3657 * wouldn't be here if it didn't. 3658 */ 3659 *copybuf_used += MMU_PAGESIZE; 3660 3661 return (DDI_SUCCESS); 3662 } 3663 3664 3665 /* 3666 * rootnex_maxxfer_window_boundary() 3667 * Called in bind slowpath when we get to a window boundary because we will 3668 * go over maxxfer. 3669 */ 3670 static int 3671 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3672 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3673 { 3674 size_t dmac_size; 3675 off_t new_offset; 3676 size_t trim_sz; 3677 off_t coffset; 3678 3679 3680 /* 3681 * calculate how much we have to trim off of the current cookie to equal 3682 * maxxfer. We don't have to account for granularity here since our 3683 * maxxfer already takes that into account. 3684 */ 3685 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3686 ASSERT(trim_sz <= cookie->dmac_size); 3687 ASSERT(trim_sz <= dma->dp_maxxfer); 3688 3689 /* save cookie size since we need it later and we might change it */ 3690 dmac_size = cookie->dmac_size; 3691 3692 /* 3693 * if we're not trimming the entire cookie, setup the current window to 3694 * account for the trim. 3695 */ 3696 if (trim_sz < cookie->dmac_size) { 3697 (*windowp)->wd_cookie_cnt++; 3698 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3699 (*windowp)->wd_trim.tr_last_cookie = cookie; 3700 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3701 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3702 (*windowp)->wd_size = dma->dp_maxxfer; 3703 3704 /* 3705 * set the adjusted cookie size now in case this is the first 3706 * window. All other windows are taken care of in get win 3707 */ 3708 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3709 } 3710 3711 /* 3712 * coffset is the current offset within the cookie, new_offset is the 3713 * current offset with the entire buffer. 3714 */ 3715 coffset = dmac_size - trim_sz; 3716 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3717 3718 /* initialize the next window */ 3719 (*windowp)++; 3720 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3721 (*windowp)->wd_cookie_cnt++; 3722 (*windowp)->wd_size = trim_sz; 3723 if (trim_sz < dmac_size) { 3724 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3725 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3726 coffset; 3727 (*windowp)->wd_trim.tr_first_size = trim_sz; 3728 } 3729 3730 return (DDI_SUCCESS); 3731 } 3732 3733 3734 /* 3735 * rootnex_dma_sync() 3736 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3737 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3738 * is set, ddi_dma_sync() returns immediately passing back success. 3739 */ 3740 /*ARGSUSED*/ 3741 static int 3742 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3743 off_t off, size_t len, uint_t cache_flags) 3744 { 3745 rootnex_sglinfo_t *sinfo; 3746 rootnex_pgmap_t *cbpage; 3747 rootnex_window_t *win; 3748 ddi_dma_impl_t *hp; 3749 rootnex_dma_t *dma; 3750 caddr_t fromaddr; 3751 caddr_t toaddr; 3752 uint_t psize; 3753 off_t offset; 3754 uint_t pidx; 3755 size_t size; 3756 off_t poff; 3757 int e; 3758 3759 3760 hp = (ddi_dma_impl_t *)handle; 3761 dma = (rootnex_dma_t *)hp->dmai_private; 3762 sinfo = &dma->dp_sglinfo; 3763 3764 /* 3765 * if we don't have any windows, we don't need to sync. A copybuf 3766 * will cause us to have at least one window. 3767 */ 3768 if (dma->dp_window == NULL) { 3769 return (DDI_SUCCESS); 3770 } 3771 3772 /* This window may not need to be sync'd */ 3773 win = &dma->dp_window[dma->dp_current_win]; 3774 if (!win->wd_dosync) { 3775 return (DDI_SUCCESS); 3776 } 3777 3778 /* handle off and len special cases */ 3779 if ((off == 0) || (rootnex_sync_ignore_params)) { 3780 offset = win->wd_offset; 3781 } else { 3782 offset = off; 3783 } 3784 if ((len == 0) || (rootnex_sync_ignore_params)) { 3785 size = win->wd_size; 3786 } else { 3787 size = len; 3788 } 3789 3790 /* check the sync args to make sure they make a little sense */ 3791 if (rootnex_sync_check_parms) { 3792 e = rootnex_valid_sync_parms(hp, win, offset, size, 3793 cache_flags); 3794 if (e != DDI_SUCCESS) { 3795 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3796 return (DDI_FAILURE); 3797 } 3798 } 3799 3800 /* 3801 * special case the first page to handle the offset into the page. The 3802 * offset to the current page for our buffer is the offset into the 3803 * first page of the buffer plus our current offset into the buffer 3804 * itself, masked of course. 3805 */ 3806 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3807 psize = MIN((MMU_PAGESIZE - poff), size); 3808 3809 /* go through all the pages that we want to sync */ 3810 while (size > 0) { 3811 /* 3812 * Calculate the page index relative to the start of the buffer. 3813 * The index to the current page for our buffer is the offset 3814 * into the first page of the buffer plus our current offset 3815 * into the buffer itself, shifted of course... 3816 */ 3817 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3818 ASSERT(pidx < sinfo->si_max_pages); 3819 3820 /* 3821 * if this page uses the copy buffer, we need to sync it, 3822 * otherwise, go on to the next page. 3823 */ 3824 cbpage = &dma->dp_pgmap[pidx]; 3825 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3826 (cbpage->pm_uses_copybuf == B_FALSE)); 3827 if (cbpage->pm_uses_copybuf) { 3828 /* cbaddr and kaddr should be page aligned */ 3829 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3830 MMU_PAGEOFFSET) == 0); 3831 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3832 MMU_PAGEOFFSET) == 0); 3833 3834 /* 3835 * if we're copying for the device, we are going to 3836 * copy from the drivers buffer and to the rootnex 3837 * allocated copy buffer. 3838 */ 3839 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3840 fromaddr = cbpage->pm_kaddr + poff; 3841 toaddr = cbpage->pm_cbaddr + poff; 3842 DTRACE_PROBE2(rootnex__sync__dev, 3843 dev_info_t *, dma->dp_dip, size_t, psize); 3844 3845 /* 3846 * if we're copying for the cpu/kernel, we are going to 3847 * copy from the rootnex allocated copy buffer to the 3848 * drivers buffer. 3849 */ 3850 } else { 3851 fromaddr = cbpage->pm_cbaddr + poff; 3852 toaddr = cbpage->pm_kaddr + poff; 3853 DTRACE_PROBE2(rootnex__sync__cpu, 3854 dev_info_t *, dma->dp_dip, size_t, psize); 3855 } 3856 3857 bcopy(fromaddr, toaddr, psize); 3858 } 3859 3860 /* 3861 * decrement size until we're done, update our offset into the 3862 * buffer, and get the next page size. 3863 */ 3864 size -= psize; 3865 offset += psize; 3866 psize = MIN(MMU_PAGESIZE, size); 3867 3868 /* page offset is zero for the rest of this loop */ 3869 poff = 0; 3870 } 3871 3872 return (DDI_SUCCESS); 3873 } 3874 3875 3876 /* 3877 * rootnex_valid_sync_parms() 3878 * checks the parameters passed to sync to verify they are correct. 3879 */ 3880 static int 3881 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 3882 off_t offset, size_t size, uint_t cache_flags) 3883 { 3884 off_t woffset; 3885 3886 3887 /* 3888 * the first part of the test to make sure the offset passed in is 3889 * within the window. 3890 */ 3891 if (offset < win->wd_offset) { 3892 return (DDI_FAILURE); 3893 } 3894 3895 /* 3896 * second and last part of the test to make sure the offset and length 3897 * passed in is within the window. 3898 */ 3899 woffset = offset - win->wd_offset; 3900 if ((woffset + size) > win->wd_size) { 3901 return (DDI_FAILURE); 3902 } 3903 3904 /* 3905 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 3906 * be set too. 3907 */ 3908 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 3909 (hp->dmai_rflags & DDI_DMA_WRITE)) { 3910 return (DDI_SUCCESS); 3911 } 3912 3913 /* 3914 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 3915 * should be set. Also DDI_DMA_READ should be set in the flags. 3916 */ 3917 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 3918 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 3919 (hp->dmai_rflags & DDI_DMA_READ)) { 3920 return (DDI_SUCCESS); 3921 } 3922 3923 return (DDI_FAILURE); 3924 } 3925 3926 3927 /* 3928 * rootnex_dma_win() 3929 * called from ddi_dma_getwin() 3930 */ 3931 /*ARGSUSED*/ 3932 static int 3933 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3934 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 3935 uint_t *ccountp) 3936 { 3937 rootnex_window_t *window; 3938 rootnex_trim_t *trim; 3939 ddi_dma_impl_t *hp; 3940 rootnex_dma_t *dma; 3941 #if !defined(__amd64) 3942 rootnex_sglinfo_t *sinfo; 3943 rootnex_pgmap_t *pmap; 3944 uint_t pidx; 3945 uint_t pcnt; 3946 off_t poff; 3947 int i; 3948 #endif 3949 3950 3951 hp = (ddi_dma_impl_t *)handle; 3952 dma = (rootnex_dma_t *)hp->dmai_private; 3953 #if !defined(__amd64) 3954 sinfo = &dma->dp_sglinfo; 3955 #endif 3956 3957 /* If we try and get a window which doesn't exist, return failure */ 3958 if (win >= hp->dmai_nwin) { 3959 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3960 return (DDI_FAILURE); 3961 } 3962 3963 /* 3964 * if we don't have any windows, and they're asking for the first 3965 * window, setup the cookie pointer to the first cookie in the bind. 3966 * setup our return values, then increment the cookie since we return 3967 * the first cookie on the stack. 3968 */ 3969 if (dma->dp_window == NULL) { 3970 if (win != 0) { 3971 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3972 return (DDI_FAILURE); 3973 } 3974 hp->dmai_cookie = dma->dp_cookies; 3975 *offp = 0; 3976 *lenp = dma->dp_dma.dmao_size; 3977 *ccountp = dma->dp_sglinfo.si_sgl_size; 3978 *cookiep = hp->dmai_cookie[0]; 3979 hp->dmai_cookie++; 3980 return (DDI_SUCCESS); 3981 } 3982 3983 /* sync the old window before moving on to the new one */ 3984 window = &dma->dp_window[dma->dp_current_win]; 3985 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 3986 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 3987 DDI_DMA_SYNC_FORCPU); 3988 } 3989 3990 #if !defined(__amd64) 3991 /* 3992 * before we move to the next window, if we need to re-map, unmap all 3993 * the pages in this window. 3994 */ 3995 if (dma->dp_cb_remaping) { 3996 /* 3997 * If we switch to this window again, we'll need to map in 3998 * on the fly next time. 3999 */ 4000 window->wd_remap_copybuf = B_TRUE; 4001 4002 /* 4003 * calculate the page index into the buffer where this window 4004 * starts, and the number of pages this window takes up. 4005 */ 4006 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4007 MMU_PAGESHIFT; 4008 poff = (sinfo->si_buf_offset + window->wd_offset) & 4009 MMU_PAGEOFFSET; 4010 pcnt = mmu_btopr(window->wd_size + poff); 4011 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4012 4013 /* unmap pages which are currently mapped in this window */ 4014 for (i = 0; i < pcnt; i++) { 4015 if (dma->dp_pgmap[pidx].pm_mapped) { 4016 hat_unload(kas.a_hat, 4017 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4018 HAT_UNLOAD); 4019 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4020 } 4021 pidx++; 4022 } 4023 } 4024 #endif 4025 4026 /* 4027 * Move to the new window. 4028 * NOTE: current_win must be set for sync to work right 4029 */ 4030 dma->dp_current_win = win; 4031 window = &dma->dp_window[win]; 4032 4033 /* if needed, adjust the first and/or last cookies for trim */ 4034 trim = &window->wd_trim; 4035 if (trim->tr_trim_first) { 4036 window->wd_first_cookie->_dmu._dmac_ll = trim->tr_first_paddr; 4037 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4038 #if !defined(__amd64) 4039 window->wd_first_cookie->dmac_type = 4040 (window->wd_first_cookie->dmac_type & 4041 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4042 #endif 4043 if (trim->tr_first_copybuf_win) { 4044 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4045 trim->tr_first_cbaddr; 4046 #if !defined(__amd64) 4047 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4048 trim->tr_first_kaddr; 4049 #endif 4050 } 4051 } 4052 if (trim->tr_trim_last) { 4053 trim->tr_last_cookie->_dmu._dmac_ll = trim->tr_last_paddr; 4054 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4055 if (trim->tr_last_copybuf_win) { 4056 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4057 trim->tr_last_cbaddr; 4058 #if !defined(__amd64) 4059 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4060 trim->tr_last_kaddr; 4061 #endif 4062 } 4063 } 4064 4065 /* 4066 * setup the cookie pointer to the first cookie in the window. setup 4067 * our return values, then increment the cookie since we return the 4068 * first cookie on the stack. 4069 */ 4070 hp->dmai_cookie = window->wd_first_cookie; 4071 *offp = window->wd_offset; 4072 *lenp = window->wd_size; 4073 *ccountp = window->wd_cookie_cnt; 4074 *cookiep = hp->dmai_cookie[0]; 4075 hp->dmai_cookie++; 4076 4077 #if !defined(__amd64) 4078 /* re-map copybuf if required for this window */ 4079 if (dma->dp_cb_remaping) { 4080 /* 4081 * calculate the page index into the buffer where this 4082 * window starts. 4083 */ 4084 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4085 MMU_PAGESHIFT; 4086 ASSERT(pidx < sinfo->si_max_pages); 4087 4088 /* 4089 * the first page can get unmapped if it's shared with the 4090 * previous window. Even if the rest of this window is already 4091 * mapped in, we need to still check this one. 4092 */ 4093 pmap = &dma->dp_pgmap[pidx]; 4094 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4095 if (pmap->pm_pp != NULL) { 4096 pmap->pm_mapped = B_TRUE; 4097 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4098 } else if (pmap->pm_vaddr != NULL) { 4099 pmap->pm_mapped = B_TRUE; 4100 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4101 pmap->pm_kaddr); 4102 } 4103 } 4104 pidx++; 4105 4106 /* map in the rest of the pages if required */ 4107 if (window->wd_remap_copybuf) { 4108 window->wd_remap_copybuf = B_FALSE; 4109 4110 /* figure out many pages this window takes up */ 4111 poff = (sinfo->si_buf_offset + window->wd_offset) & 4112 MMU_PAGEOFFSET; 4113 pcnt = mmu_btopr(window->wd_size + poff); 4114 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4115 4116 /* map pages which require it */ 4117 for (i = 1; i < pcnt; i++) { 4118 pmap = &dma->dp_pgmap[pidx]; 4119 if (pmap->pm_uses_copybuf) { 4120 ASSERT(pmap->pm_mapped == B_FALSE); 4121 if (pmap->pm_pp != NULL) { 4122 pmap->pm_mapped = B_TRUE; 4123 i86_pp_map(pmap->pm_pp, 4124 pmap->pm_kaddr); 4125 } else if (pmap->pm_vaddr != NULL) { 4126 pmap->pm_mapped = B_TRUE; 4127 i86_va_map(pmap->pm_vaddr, 4128 sinfo->si_asp, 4129 pmap->pm_kaddr); 4130 } 4131 } 4132 pidx++; 4133 } 4134 } 4135 } 4136 #endif 4137 4138 /* if the new window uses the copy buffer, sync it for the device */ 4139 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4140 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4141 DDI_DMA_SYNC_FORDEV); 4142 } 4143 4144 return (DDI_SUCCESS); 4145 } 4146 4147 4148 4149 /* 4150 * ************************ 4151 * obsoleted dma routines 4152 * ************************ 4153 */ 4154 4155 /* 4156 * rootnex_dma_map() 4157 * called from ddi_dma_setup() 4158 */ 4159 /* ARGSUSED */ 4160 static int 4161 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4162 ddi_dma_handle_t *handlep) 4163 { 4164 #if defined(__amd64) 4165 /* 4166 * this interface is not supported in 64-bit x86 kernel. See comment in 4167 * rootnex_dma_mctl() 4168 */ 4169 return (DDI_DMA_NORESOURCES); 4170 4171 #else /* 32-bit x86 kernel */ 4172 ddi_dma_handle_t *lhandlep; 4173 ddi_dma_handle_t lhandle; 4174 ddi_dma_cookie_t cookie; 4175 ddi_dma_attr_t dma_attr; 4176 ddi_dma_lim_t *dma_lim; 4177 uint_t ccnt; 4178 int e; 4179 4180 4181 /* 4182 * if the driver is just testing to see if it's possible to do the bind, 4183 * we'll use local state. Otherwise, use the handle pointer passed in. 4184 */ 4185 if (handlep == NULL) { 4186 lhandlep = &lhandle; 4187 } else { 4188 lhandlep = handlep; 4189 } 4190 4191 /* convert the limit structure to a dma_attr one */ 4192 dma_lim = dmareq->dmar_limits; 4193 dma_attr.dma_attr_version = DMA_ATTR_V0; 4194 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4195 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4196 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4197 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4198 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4199 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4200 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4201 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4202 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4203 dma_attr.dma_attr_align = MMU_PAGESIZE; 4204 dma_attr.dma_attr_flags = 0; 4205 4206 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4207 dmareq->dmar_arg, lhandlep); 4208 if (e != DDI_SUCCESS) { 4209 return (e); 4210 } 4211 4212 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4213 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4214 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4215 return (e); 4216 } 4217 4218 /* 4219 * if the driver is just testing to see if it's possible to do the bind, 4220 * free up the local state and return the result. 4221 */ 4222 if (handlep == NULL) { 4223 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4224 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4225 if (e == DDI_DMA_MAPPED) { 4226 return (DDI_DMA_MAPOK); 4227 } else { 4228 return (DDI_DMA_NOMAPPING); 4229 } 4230 } 4231 4232 return (e); 4233 #endif /* defined(__amd64) */ 4234 } 4235 4236 4237 /* 4238 * rootnex_dma_mctl() 4239 * 4240 */ 4241 /* ARGSUSED */ 4242 static int 4243 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4244 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4245 uint_t cache_flags) 4246 { 4247 #if defined(__amd64) 4248 /* 4249 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4250 * common implementation in genunix, so they no longer have x86 4251 * specific functionality which called into dma_ctl. 4252 * 4253 * The rest of the obsoleted interfaces were never supported in the 4254 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4255 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4256 * implementation issues. 4257 * 4258 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4259 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4260 * reflect that now too... 4261 * 4262 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4263 * not going to put this functionality into the 64-bit x86 kernel now. 4264 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4265 * that in a future release. 4266 */ 4267 return (DDI_FAILURE); 4268 4269 #else /* 32-bit x86 kernel */ 4270 ddi_dma_cookie_t lcookie; 4271 ddi_dma_cookie_t *cookie; 4272 rootnex_window_t *window; 4273 ddi_dma_impl_t *hp; 4274 rootnex_dma_t *dma; 4275 uint_t nwin; 4276 uint_t ccnt; 4277 size_t len; 4278 off_t off; 4279 int e; 4280 4281 4282 /* 4283 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4284 * hacky since were optimizing for the current interfaces and so we can 4285 * cleanup the mess in genunix. Hopefully we will remove the this 4286 * obsoleted routines someday soon. 4287 */ 4288 4289 switch (request) { 4290 4291 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4292 hp = (ddi_dma_impl_t *)handle; 4293 cookie = (ddi_dma_cookie_t *)objpp; 4294 4295 /* 4296 * convert segment to cookie. We don't distinguish between the 4297 * two :-) 4298 */ 4299 *cookie = *hp->dmai_cookie; 4300 *lenp = cookie->dmac_size; 4301 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4302 return (DDI_SUCCESS); 4303 4304 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4305 hp = (ddi_dma_impl_t *)handle; 4306 dma = (rootnex_dma_t *)hp->dmai_private; 4307 4308 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4309 return (DDI_DMA_STALE); 4310 } 4311 4312 /* handle the case where we don't have any windows */ 4313 if (dma->dp_window == NULL) { 4314 /* 4315 * if seg == NULL, and we don't have any windows, 4316 * return the first cookie in the sgl. 4317 */ 4318 if (*lenp == NULL) { 4319 dma->dp_current_cookie = 0; 4320 hp->dmai_cookie = dma->dp_cookies; 4321 *objpp = (caddr_t)handle; 4322 return (DDI_SUCCESS); 4323 4324 /* if we have more cookies, go to the next cookie */ 4325 } else { 4326 if ((dma->dp_current_cookie + 1) >= 4327 dma->dp_sglinfo.si_sgl_size) { 4328 return (DDI_DMA_DONE); 4329 } 4330 dma->dp_current_cookie++; 4331 hp->dmai_cookie++; 4332 return (DDI_SUCCESS); 4333 } 4334 } 4335 4336 /* We have one or more windows */ 4337 window = &dma->dp_window[dma->dp_current_win]; 4338 4339 /* 4340 * if seg == NULL, return the first cookie in the current 4341 * window 4342 */ 4343 if (*lenp == NULL) { 4344 dma->dp_current_cookie = 0; 4345 hp->dmai_cookie = window->wd_first_cookie; 4346 4347 /* 4348 * go to the next cookie in the window then see if we done with 4349 * this window. 4350 */ 4351 } else { 4352 if ((dma->dp_current_cookie + 1) >= 4353 window->wd_cookie_cnt) { 4354 return (DDI_DMA_DONE); 4355 } 4356 dma->dp_current_cookie++; 4357 hp->dmai_cookie++; 4358 } 4359 *objpp = (caddr_t)handle; 4360 return (DDI_SUCCESS); 4361 4362 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4363 hp = (ddi_dma_impl_t *)handle; 4364 dma = (rootnex_dma_t *)hp->dmai_private; 4365 4366 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4367 return (DDI_DMA_STALE); 4368 } 4369 4370 /* if win == NULL, return the first window in the bind */ 4371 if (*offp == NULL) { 4372 nwin = 0; 4373 4374 /* 4375 * else, go to the next window then see if we're done with all 4376 * the windows. 4377 */ 4378 } else { 4379 nwin = dma->dp_current_win + 1; 4380 if (nwin >= hp->dmai_nwin) { 4381 return (DDI_DMA_DONE); 4382 } 4383 } 4384 4385 /* switch to the next window */ 4386 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4387 &lcookie, &ccnt); 4388 ASSERT(e == DDI_SUCCESS); 4389 if (e != DDI_SUCCESS) { 4390 return (DDI_DMA_STALE); 4391 } 4392 4393 /* reset the cookie back to the first cookie in the window */ 4394 if (dma->dp_window != NULL) { 4395 window = &dma->dp_window[dma->dp_current_win]; 4396 hp->dmai_cookie = window->wd_first_cookie; 4397 } else { 4398 hp->dmai_cookie = dma->dp_cookies; 4399 } 4400 4401 *objpp = (caddr_t)handle; 4402 return (DDI_SUCCESS); 4403 4404 case DDI_DMA_FREE: /* ddi_dma_free() */ 4405 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4406 (void) rootnex_dma_freehdl(dip, rdip, handle); 4407 if (rootnex_state->r_dvma_call_list_id) { 4408 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4409 } 4410 return (DDI_SUCCESS); 4411 4412 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4413 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4414 /* should never get here, handled in genunix */ 4415 ASSERT(0); 4416 return (DDI_FAILURE); 4417 4418 case DDI_DMA_KVADDR: 4419 case DDI_DMA_GETERR: 4420 case DDI_DMA_COFF: 4421 return (DDI_FAILURE); 4422 } 4423 4424 return (DDI_FAILURE); 4425 #endif /* defined(__amd64) */ 4426 } 4427 4428 /*ARGSUSED*/ 4429 static int 4430 rootnex_fm_callback(dev_info_t *dip, ddi_fm_error_t *derr, const void *no_used) 4431 { 4432 return (rootnex_fm_ma_ta_panic_flag ? DDI_FM_FATAL : DDI_FM_NONFATAL); 4433 } 4434