1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License, Version 1.0 only 6 * (the "License"). You may not use this file except in compliance 7 * with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 #pragma ident "%Z%%M% %I% %E% SMI" 28 29 /* 30 * x86 root nexus driver 31 */ 32 33 #include <sys/sysmacros.h> 34 #include <sys/conf.h> 35 #include <sys/autoconf.h> 36 #include <sys/sysmacros.h> 37 #include <sys/debug.h> 38 #include <sys/psw.h> 39 #include <sys/ddidmareq.h> 40 #include <sys/promif.h> 41 #include <sys/devops.h> 42 #include <sys/kmem.h> 43 #include <sys/cmn_err.h> 44 #include <vm/seg.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_dev.h> 47 #include <sys/vmem.h> 48 #include <sys/mman.h> 49 #include <vm/hat.h> 50 #include <vm/as.h> 51 #include <vm/page.h> 52 #include <sys/avintr.h> 53 #include <sys/errno.h> 54 #include <sys/modctl.h> 55 #include <sys/ddi_impldefs.h> 56 #include <sys/sunddi.h> 57 #include <sys/sunndi.h> 58 #include <sys/psm.h> 59 #include <sys/ontrap.h> 60 #include <sys/atomic.h> 61 #include <sys/sdt.h> 62 #include <sys/rootnex.h> 63 #include <vm/hat_i86.h> 64 65 66 /* 67 * enable/disable extra checking of function parameters. Useful for debugging 68 * drivers. 69 */ 70 #ifdef DEBUG 71 int rootnex_alloc_check_parms = 1; 72 int rootnex_bind_check_parms = 1; 73 int rootnex_bind_check_inuse = 1; 74 int rootnex_unbind_verify_buffer = 0; 75 int rootnex_sync_check_parms = 1; 76 #else 77 int rootnex_alloc_check_parms = 0; 78 int rootnex_bind_check_parms = 0; 79 int rootnex_bind_check_inuse = 0; 80 int rootnex_unbind_verify_buffer = 0; 81 int rootnex_sync_check_parms = 0; 82 #endif 83 84 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 85 int rootnex_bind_fail = 1; 86 int rootnex_bind_warn = 1; 87 uint8_t *rootnex_warn_list; 88 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 89 #define ROOTNEX_BIND_WARNING (0x1 << 0) 90 91 /* 92 * revert back to old broken behavior of always sync'ing entire copy buffer. 93 * This is useful if be have a buggy driver which doesn't correctly pass in 94 * the offset and size into ddi_dma_sync(). 95 */ 96 int rootnex_sync_ignore_params = 0; 97 98 /* 99 * maximum size that we will allow for a copy buffer. Can be patched on the 100 * fly 101 */ 102 size_t rootnex_max_copybuf_size = 0x100000; 103 104 /* 105 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 106 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 107 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 108 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 109 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 110 * (< 8K). We will still need to allocate the copy buffer during bind though 111 * (if we need one). These can only be modified in /etc/system before rootnex 112 * attach. 113 */ 114 #if defined(__amd64) 115 int rootnex_prealloc_cookies = 65; 116 int rootnex_prealloc_windows = 4; 117 int rootnex_prealloc_copybuf = 2; 118 #else 119 int rootnex_prealloc_cookies = 33; 120 int rootnex_prealloc_windows = 4; 121 int rootnex_prealloc_copybuf = 2; 122 #endif 123 124 /* driver global state */ 125 static rootnex_state_t *rootnex_state; 126 127 /* shortcut to rootnex counters */ 128 static uint64_t *rootnex_cnt; 129 130 /* 131 * XXX - does x86 even need these or are they left over from the SPARC days? 132 */ 133 /* statically defined integer/boolean properties for the root node */ 134 static rootnex_intprop_t rootnex_intprp[] = { 135 { "PAGESIZE", PAGESIZE }, 136 { "MMU_PAGESIZE", MMU_PAGESIZE }, 137 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 138 { DDI_RELATIVE_ADDRESSING, 1 }, 139 }; 140 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 141 142 143 static struct cb_ops rootnex_cb_ops = { 144 nodev, /* open */ 145 nodev, /* close */ 146 nodev, /* strategy */ 147 nodev, /* print */ 148 nodev, /* dump */ 149 nodev, /* read */ 150 nodev, /* write */ 151 nodev, /* ioctl */ 152 nodev, /* devmap */ 153 nodev, /* mmap */ 154 nodev, /* segmap */ 155 nochpoll, /* chpoll */ 156 ddi_prop_op, /* cb_prop_op */ 157 NULL, /* struct streamtab */ 158 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 159 CB_REV, /* Rev */ 160 nodev, /* cb_aread */ 161 nodev /* cb_awrite */ 162 }; 163 164 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 165 off_t offset, off_t len, caddr_t *vaddrp); 166 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 167 struct hat *hat, struct seg *seg, caddr_t addr, 168 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 169 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 170 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 171 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 172 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 173 ddi_dma_handle_t *handlep); 174 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 175 ddi_dma_handle_t handle); 176 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 177 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 178 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 179 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 180 ddi_dma_handle_t handle); 181 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 182 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 183 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 184 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 185 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 186 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 187 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 188 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 189 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 190 ddi_ctl_enum_t ctlop, void *arg, void *result); 191 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 192 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 193 194 195 static struct bus_ops rootnex_bus_ops = { 196 BUSO_REV, 197 rootnex_map, 198 NULL, 199 NULL, 200 NULL, 201 rootnex_map_fault, 202 rootnex_dma_map, 203 rootnex_dma_allochdl, 204 rootnex_dma_freehdl, 205 rootnex_dma_bindhdl, 206 rootnex_dma_unbindhdl, 207 rootnex_dma_sync, 208 rootnex_dma_win, 209 rootnex_dma_mctl, 210 rootnex_ctlops, 211 ddi_bus_prop_op, 212 i_ddi_rootnex_get_eventcookie, 213 i_ddi_rootnex_add_eventcall, 214 i_ddi_rootnex_remove_eventcall, 215 i_ddi_rootnex_post_event, 216 0, /* bus_intr_ctl */ 217 0, /* bus_config */ 218 0, /* bus_unconfig */ 219 NULL, /* bus_fm_init */ 220 NULL, /* bus_fm_fini */ 221 NULL, /* bus_fm_access_enter */ 222 NULL, /* bus_fm_access_exit */ 223 NULL, /* bus_powr */ 224 rootnex_intr_ops /* bus_intr_op */ 225 }; 226 227 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 228 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 229 230 static struct dev_ops rootnex_ops = { 231 DEVO_REV, 232 0, 233 ddi_no_info, 234 nulldev, 235 nulldev, 236 rootnex_attach, 237 rootnex_detach, 238 nulldev, 239 &rootnex_cb_ops, 240 &rootnex_bus_ops 241 }; 242 243 static struct modldrv rootnex_modldrv = { 244 &mod_driverops, 245 "i86pc root nexus %I%", 246 &rootnex_ops 247 }; 248 249 static struct modlinkage rootnex_modlinkage = { 250 MODREV_1, 251 (void *)&rootnex_modldrv, 252 NULL 253 }; 254 255 256 /* 257 * extern hacks 258 */ 259 extern struct seg_ops segdev_ops; 260 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 261 #ifdef DDI_MAP_DEBUG 262 extern int ddi_map_debug_flag; 263 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 264 #endif 265 #define ptob64(x) (((uint64_t)(x)) << MMU_PAGESHIFT) 266 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 267 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 268 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 269 psm_intr_op_t, int *); 270 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 271 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 272 /* 273 * Use device arena to use for device control register mappings. 274 * Various kernel memory walkers (debugger, dtrace) need to know 275 * to avoid this address range to prevent undesired device activity. 276 */ 277 extern void *device_arena_alloc(size_t size, int vm_flag); 278 extern void device_arena_free(void * vaddr, size_t size); 279 280 281 /* 282 * Internal functions 283 */ 284 static int rootnex_dma_init(); 285 static void rootnex_add_props(dev_info_t *); 286 static int rootnex_ctl_reportdev(dev_info_t *dip); 287 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 288 static int rootnex_xlate_intrs(dev_info_t *dip, dev_info_t *rdip, int *in, 289 struct ddi_parent_private_data *pdptr); 290 static int rootnex_ctlops_poke(peekpoke_ctlops_t *in_args); 291 static int rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result); 292 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 293 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 294 static int rootnex_map_handle(ddi_map_req_t *mp); 295 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 296 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 297 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 298 ddi_dma_attr_t *attr); 299 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 300 rootnex_sglinfo_t *sglinfo); 301 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 302 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 303 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 304 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 305 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 306 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 307 ddi_dma_attr_t *attr, int kmflag); 308 static void rootnex_teardown_windows(rootnex_dma_t *dma); 309 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 310 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 311 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 312 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 313 size_t *copybuf_used, page_t **cur_pp); 314 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 315 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 316 ddi_dma_attr_t *attr, off_t cur_offset); 317 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 318 rootnex_dma_t *dma, rootnex_window_t **windowp, 319 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 320 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 321 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 322 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 323 off_t offset, size_t size, uint_t cache_flags); 324 static int rootnex_verify_buffer(rootnex_dma_t *dma); 325 326 327 /* 328 * _init() 329 * 330 */ 331 int 332 _init(void) 333 { 334 335 rootnex_state = NULL; 336 return (mod_install(&rootnex_modlinkage)); 337 } 338 339 340 /* 341 * _info() 342 * 343 */ 344 int 345 _info(struct modinfo *modinfop) 346 { 347 return (mod_info(&rootnex_modlinkage, modinfop)); 348 } 349 350 351 /* 352 * _fini() 353 * 354 */ 355 int 356 _fini(void) 357 { 358 return (EBUSY); 359 } 360 361 362 /* 363 * rootnex_attach() 364 * 365 */ 366 static int 367 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 368 { 369 int e; 370 371 372 switch (cmd) { 373 case DDI_ATTACH: 374 break; 375 case DDI_RESUME: 376 return (DDI_SUCCESS); 377 default: 378 return (DDI_FAILURE); 379 } 380 381 /* 382 * We should only have one instance of rootnex. Save it away since we 383 * don't have an easy way to get it back later. 384 */ 385 ASSERT(rootnex_state == NULL); 386 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 387 388 rootnex_state->r_dip = dip; 389 rootnex_state->r_reserved_msg_printed = B_FALSE; 390 rootnex_cnt = &rootnex_state->r_counters[0]; 391 392 mutex_init(&rootnex_state->r_peekpoke_mutex, NULL, MUTEX_SPIN, 393 (void *)ipltospl(15)); 394 395 /* initialize DMA related state */ 396 e = rootnex_dma_init(); 397 if (e != DDI_SUCCESS) { 398 mutex_destroy(&rootnex_state->r_peekpoke_mutex); 399 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 400 return (DDI_FAILURE); 401 } 402 403 /* Add static root node properties */ 404 rootnex_add_props(dip); 405 406 /* since we can't call ddi_report_dev() */ 407 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 408 409 /* Initialize rootnex event handle */ 410 i_ddi_rootnex_init_events(dip); 411 412 return (DDI_SUCCESS); 413 } 414 415 416 /* 417 * rootnex_detach() 418 * 419 */ 420 /*ARGSUSED*/ 421 static int 422 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 423 { 424 switch (cmd) { 425 case DDI_SUSPEND: 426 break; 427 default: 428 return (DDI_FAILURE); 429 } 430 431 return (DDI_SUCCESS); 432 } 433 434 435 /* 436 * rootnex_dma_init() 437 * 438 */ 439 /*ARGSUSED*/ 440 static int 441 rootnex_dma_init() 442 { 443 size_t bufsize; 444 445 446 /* 447 * size of our cookie/window/copybuf state needed in dma bind that we 448 * pre-alloc in dma_alloc_handle 449 */ 450 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 451 rootnex_state->r_prealloc_size = 452 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 453 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 454 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 455 456 /* 457 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 458 * allocate 16 extra bytes for struct pointer alignment 459 * (p->dmai_private & dma->dp_prealloc_buffer) 460 */ 461 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 462 rootnex_state->r_prealloc_size + 0x10; 463 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 464 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 465 if (rootnex_state->r_dmahdl_cache == NULL) { 466 return (DDI_FAILURE); 467 } 468 469 /* 470 * allocate array to track which major numbers we have printed warnings 471 * for. 472 */ 473 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 474 KM_SLEEP); 475 476 return (DDI_SUCCESS); 477 } 478 479 480 /* 481 * rootnex_add_props() 482 * 483 */ 484 static void 485 rootnex_add_props(dev_info_t *dip) 486 { 487 rootnex_intprop_t *rpp; 488 int i; 489 490 /* Add static integer/boolean properties to the root node */ 491 rpp = rootnex_intprp; 492 for (i = 0; i < NROOT_INTPROPS; i++) { 493 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 494 rpp[i].prop_name, rpp[i].prop_value); 495 } 496 } 497 498 499 500 /* 501 * ************************* 502 * ctlops related routines 503 * ************************* 504 */ 505 506 /* 507 * rootnex_ctlops() 508 * 509 */ 510 static int 511 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 512 void *arg, void *result) 513 { 514 int n, *ptr; 515 struct ddi_parent_private_data *pdp; 516 517 518 switch (ctlop) { 519 case DDI_CTLOPS_DMAPMAPC: 520 /* 521 * Return 'partial' to indicate that dma mapping 522 * has to be done in the main MMU. 523 */ 524 return (DDI_DMA_PARTIAL); 525 526 case DDI_CTLOPS_BTOP: 527 /* 528 * Convert byte count input to physical page units. 529 * (byte counts that are not a page-size multiple 530 * are rounded down) 531 */ 532 *(ulong_t *)result = btop(*(ulong_t *)arg); 533 return (DDI_SUCCESS); 534 535 case DDI_CTLOPS_PTOB: 536 /* 537 * Convert size in physical pages to bytes 538 */ 539 *(ulong_t *)result = ptob(*(ulong_t *)arg); 540 return (DDI_SUCCESS); 541 542 case DDI_CTLOPS_BTOPR: 543 /* 544 * Convert byte count input to physical page units 545 * (byte counts that are not a page-size multiple 546 * are rounded up) 547 */ 548 *(ulong_t *)result = btopr(*(ulong_t *)arg); 549 return (DDI_SUCCESS); 550 551 case DDI_CTLOPS_POKE: 552 return (rootnex_ctlops_poke((peekpoke_ctlops_t *)arg)); 553 554 case DDI_CTLOPS_PEEK: 555 return (rootnex_ctlops_peek((peekpoke_ctlops_t *)arg, result)); 556 557 case DDI_CTLOPS_INITCHILD: 558 return (impl_ddi_sunbus_initchild(arg)); 559 560 case DDI_CTLOPS_UNINITCHILD: 561 impl_ddi_sunbus_removechild(arg); 562 return (DDI_SUCCESS); 563 564 case DDI_CTLOPS_REPORTDEV: 565 return (rootnex_ctl_reportdev(rdip)); 566 567 case DDI_CTLOPS_IOMIN: 568 /* 569 * Nothing to do here but reflect back.. 570 */ 571 return (DDI_SUCCESS); 572 573 case DDI_CTLOPS_REGSIZE: 574 case DDI_CTLOPS_NREGS: 575 case DDI_CTLOPS_NINTRS: 576 break; 577 578 case DDI_CTLOPS_SIDDEV: 579 if (ndi_dev_is_prom_node(rdip)) 580 return (DDI_SUCCESS); 581 if (ndi_dev_is_persistent_node(rdip)) 582 return (DDI_SUCCESS); 583 return (DDI_FAILURE); 584 585 case DDI_CTLOPS_INTR_HILEVEL: 586 /* 587 * Indicate whether the interrupt specified is to be handled 588 * above lock level. In other words, above the level that 589 * cv_signal and default type mutexes can be used. 590 */ 591 *(int *)result = 592 (INT_IPL(((struct intrspec *)arg)->intrspec_pri) 593 > LOCK_LEVEL); 594 return (DDI_SUCCESS); 595 596 case DDI_CTLOPS_XLATE_INTRS: 597 return (rootnex_xlate_intrs(dip, rdip, arg, result)); 598 599 case DDI_CTLOPS_POWER: 600 return ((*pm_platform_power)((power_req_t *)arg)); 601 602 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 603 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 604 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 605 if (!rootnex_state->r_reserved_msg_printed) { 606 rootnex_state->r_reserved_msg_printed = B_TRUE; 607 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 608 "1 or more reserved/obsolete operations."); 609 } 610 return (DDI_FAILURE); 611 612 default: 613 return (DDI_FAILURE); 614 } 615 /* 616 * The rest are for "hardware" properties 617 */ 618 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 619 return (DDI_FAILURE); 620 621 if (ctlop == DDI_CTLOPS_NREGS) { 622 ptr = (int *)result; 623 *ptr = pdp->par_nreg; 624 } else if (ctlop == DDI_CTLOPS_NINTRS) { 625 ptr = (int *)result; 626 *ptr = pdp->par_nintr; 627 } else { 628 off_t *size = (off_t *)result; 629 630 ptr = (int *)arg; 631 n = *ptr; 632 if (n >= pdp->par_nreg) { 633 return (DDI_FAILURE); 634 } 635 *size = (off_t)pdp->par_reg[n].regspec_size; 636 } 637 return (DDI_SUCCESS); 638 } 639 640 641 /* 642 * rootnex_ctl_reportdev() 643 * 644 */ 645 static int 646 rootnex_ctl_reportdev(dev_info_t *dev) 647 { 648 int i, n, len, f_len = 0; 649 char *buf; 650 651 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 652 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 653 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 654 len = strlen(buf); 655 656 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 657 658 struct regspec *rp = sparc_pd_getreg(dev, i); 659 660 if (i == 0) 661 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 662 ": "); 663 else 664 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 665 " and "); 666 len = strlen(buf); 667 668 switch (rp->regspec_bustype) { 669 670 case BTEISA: 671 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 672 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 673 break; 674 675 case BTISA: 676 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 677 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 678 break; 679 680 default: 681 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 682 "space %x offset %x", 683 rp->regspec_bustype, rp->regspec_addr); 684 break; 685 } 686 len = strlen(buf); 687 } 688 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 689 int pri; 690 691 if (i != 0) { 692 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 693 ","); 694 len = strlen(buf); 695 } 696 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 697 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 698 " sparc ipl %d", pri); 699 len = strlen(buf); 700 } 701 #ifdef DEBUG 702 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 703 cmn_err(CE_NOTE, "next message is truncated: " 704 "printed length 1024, real length %d", f_len); 705 } 706 #endif /* DEBUG */ 707 cmn_err(CE_CONT, "?%s\n", buf); 708 kmem_free(buf, REPORTDEV_BUFSIZE); 709 return (DDI_SUCCESS); 710 } 711 712 713 /* 714 * rootnex_ctlops_poke() 715 * 716 */ 717 static int 718 rootnex_ctlops_poke(peekpoke_ctlops_t *in_args) 719 { 720 int err = DDI_SUCCESS; 721 on_trap_data_t otd; 722 723 /* Cautious access not supported. */ 724 if (in_args->handle != NULL) 725 return (DDI_FAILURE); 726 727 mutex_enter(&rootnex_state->r_peekpoke_mutex); 728 729 /* Set up protected environment. */ 730 if (!on_trap(&otd, OT_DATA_ACCESS)) { 731 switch (in_args->size) { 732 case sizeof (uint8_t): 733 *(uint8_t *)in_args->dev_addr = *(uint8_t *) 734 in_args->host_addr; 735 break; 736 737 case sizeof (uint16_t): 738 *(uint16_t *)in_args->dev_addr = 739 *(uint16_t *)in_args->host_addr; 740 break; 741 742 case sizeof (uint32_t): 743 *(uint32_t *)in_args->dev_addr = 744 *(uint32_t *)in_args->host_addr; 745 break; 746 747 case sizeof (uint64_t): 748 *(uint64_t *)in_args->dev_addr = 749 *(uint64_t *)in_args->host_addr; 750 break; 751 752 default: 753 err = DDI_FAILURE; 754 break; 755 } 756 } else 757 err = DDI_FAILURE; 758 759 /* Take down protected environment. */ 760 no_trap(); 761 mutex_exit(&rootnex_state->r_peekpoke_mutex); 762 763 return (err); 764 } 765 766 767 /* 768 * rootnex_ctlops_peek() 769 * 770 */ 771 static int 772 rootnex_ctlops_peek(peekpoke_ctlops_t *in_args, void *result) 773 { 774 int err = DDI_SUCCESS; 775 on_trap_data_t otd; 776 777 /* Cautious access not supported. */ 778 if (in_args->handle != NULL) 779 return (DDI_FAILURE); 780 781 mutex_enter(&rootnex_state->r_peekpoke_mutex); 782 783 if (!on_trap(&otd, OT_DATA_ACCESS)) { 784 switch (in_args->size) { 785 case sizeof (uint8_t): 786 *(uint8_t *)in_args->host_addr = 787 *(uint8_t *)in_args->dev_addr; 788 break; 789 790 case sizeof (uint16_t): 791 *(uint16_t *)in_args->host_addr = 792 *(uint16_t *)in_args->dev_addr; 793 break; 794 795 case sizeof (uint32_t): 796 *(uint32_t *)in_args->host_addr = 797 *(uint32_t *)in_args->dev_addr; 798 break; 799 800 case sizeof (uint64_t): 801 *(uint64_t *)in_args->host_addr = 802 *(uint64_t *)in_args->dev_addr; 803 break; 804 805 default: 806 err = DDI_FAILURE; 807 break; 808 } 809 result = (void *)in_args->host_addr; 810 } else 811 err = DDI_FAILURE; 812 813 no_trap(); 814 mutex_exit(&rootnex_state->r_peekpoke_mutex); 815 816 return (err); 817 } 818 819 820 821 /* 822 * ****************** 823 * map related code 824 * ****************** 825 */ 826 827 /* 828 * rootnex_map() 829 * 830 */ 831 static int 832 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 833 off_t len, caddr_t *vaddrp) 834 { 835 struct regspec *rp, tmp_reg; 836 ddi_map_req_t mr = *mp; /* Get private copy of request */ 837 int error; 838 839 mp = &mr; 840 841 switch (mp->map_op) { 842 case DDI_MO_MAP_LOCKED: 843 case DDI_MO_UNMAP: 844 case DDI_MO_MAP_HANDLE: 845 break; 846 default: 847 #ifdef DDI_MAP_DEBUG 848 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 849 mp->map_op); 850 #endif /* DDI_MAP_DEBUG */ 851 return (DDI_ME_UNIMPLEMENTED); 852 } 853 854 if (mp->map_flags & DDI_MF_USER_MAPPING) { 855 #ifdef DDI_MAP_DEBUG 856 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 857 #endif /* DDI_MAP_DEBUG */ 858 return (DDI_ME_UNIMPLEMENTED); 859 } 860 861 /* 862 * First, if given an rnumber, convert it to a regspec... 863 * (Presumably, this is on behalf of a child of the root node?) 864 */ 865 866 if (mp->map_type == DDI_MT_RNUMBER) { 867 868 int rnumber = mp->map_obj.rnumber; 869 #ifdef DDI_MAP_DEBUG 870 static char *out_of_range = 871 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 872 #endif /* DDI_MAP_DEBUG */ 873 874 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 875 if (rp == NULL) { 876 #ifdef DDI_MAP_DEBUG 877 cmn_err(CE_WARN, out_of_range, rnumber, 878 ddi_get_name(rdip)); 879 #endif /* DDI_MAP_DEBUG */ 880 return (DDI_ME_RNUMBER_RANGE); 881 } 882 883 /* 884 * Convert the given ddi_map_req_t from rnumber to regspec... 885 */ 886 887 mp->map_type = DDI_MT_REGSPEC; 888 mp->map_obj.rp = rp; 889 } 890 891 /* 892 * Adjust offset and length correspnding to called values... 893 * XXX: A non-zero length means override the one in the regspec 894 * XXX: (regardless of what's in the parent's range?) 895 */ 896 897 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 898 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 899 900 #ifdef DDI_MAP_DEBUG 901 cmn_err(CE_CONT, 902 "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d>" 903 " offset %d len %d handle 0x%x\n", 904 ddi_get_name(dip), ddi_get_name(rdip), 905 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 906 offset, len, mp->map_handlep); 907 #endif /* DDI_MAP_DEBUG */ 908 909 /* 910 * I/O or memory mapping: 911 * 912 * <bustype=0, addr=x, len=x>: memory 913 * <bustype=1, addr=x, len=x>: i/o 914 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 915 */ 916 917 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 918 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 919 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 920 ddi_get_name(rdip), rp->regspec_bustype, 921 rp->regspec_addr, rp->regspec_size); 922 return (DDI_ME_INVAL); 923 } 924 925 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 926 /* 927 * compatibility i/o mapping 928 */ 929 rp->regspec_bustype += (uint_t)offset; 930 } else { 931 /* 932 * Normal memory or i/o mapping 933 */ 934 rp->regspec_addr += (uint_t)offset; 935 } 936 937 if (len != 0) 938 rp->regspec_size = (uint_t)len; 939 940 #ifdef DDI_MAP_DEBUG 941 cmn_err(CE_CONT, 942 " <%s,%s> <0x%x, 0x%x, 0x%d>" 943 " offset %d len %d handle 0x%x\n", 944 ddi_get_name(dip), ddi_get_name(rdip), 945 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 946 offset, len, mp->map_handlep); 947 #endif /* DDI_MAP_DEBUG */ 948 949 /* 950 * Apply any parent ranges at this level, if applicable. 951 * (This is where nexus specific regspec translation takes place. 952 * Use of this function is implicit agreement that translation is 953 * provided via ddi_apply_range.) 954 */ 955 956 #ifdef DDI_MAP_DEBUG 957 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 958 ddi_get_name(dip), ddi_get_name(rdip)); 959 #endif /* DDI_MAP_DEBUG */ 960 961 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 962 return (error); 963 964 switch (mp->map_op) { 965 case DDI_MO_MAP_LOCKED: 966 967 /* 968 * Set up the locked down kernel mapping to the regspec... 969 */ 970 971 return (rootnex_map_regspec(mp, vaddrp)); 972 973 case DDI_MO_UNMAP: 974 975 /* 976 * Release mapping... 977 */ 978 979 return (rootnex_unmap_regspec(mp, vaddrp)); 980 981 case DDI_MO_MAP_HANDLE: 982 983 return (rootnex_map_handle(mp)); 984 985 default: 986 return (DDI_ME_UNIMPLEMENTED); 987 } 988 } 989 990 991 /* 992 * rootnex_map_fault() 993 * 994 * fault in mappings for requestors 995 */ 996 /*ARGSUSED*/ 997 static int 998 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 999 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 1000 uint_t lock) 1001 { 1002 1003 #ifdef DDI_MAP_DEBUG 1004 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 1005 ddi_map_debug(" Seg <%s>\n", 1006 seg->s_ops == &segdev_ops ? "segdev" : 1007 seg == &kvseg ? "segkmem" : "NONE!"); 1008 #endif /* DDI_MAP_DEBUG */ 1009 1010 /* 1011 * This is all terribly broken, but it is a start 1012 * 1013 * XXX Note that this test means that segdev_ops 1014 * must be exported from seg_dev.c. 1015 * XXX What about devices with their own segment drivers? 1016 */ 1017 if (seg->s_ops == &segdev_ops) { 1018 struct segdev_data *sdp = 1019 (struct segdev_data *)seg->s_data; 1020 1021 if (hat == NULL) { 1022 /* 1023 * This is one plausible interpretation of 1024 * a null hat i.e. use the first hat on the 1025 * address space hat list which by convention is 1026 * the hat of the system MMU. At alternative 1027 * would be to panic .. this might well be better .. 1028 */ 1029 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1030 hat = seg->s_as->a_hat; 1031 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1032 } 1033 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1034 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1035 } else if (seg == &kvseg && dp == NULL) { 1036 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1037 HAT_LOAD_LOCK); 1038 } else 1039 return (DDI_FAILURE); 1040 return (DDI_SUCCESS); 1041 } 1042 1043 1044 /* 1045 * rootnex_map_regspec() 1046 * we don't support mapping of I/O cards above 4Gb 1047 */ 1048 static int 1049 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1050 { 1051 ulong_t base; 1052 void *cvaddr; 1053 uint_t npages, pgoffset; 1054 struct regspec *rp; 1055 ddi_acc_hdl_t *hp; 1056 ddi_acc_impl_t *ap; 1057 uint_t hat_acc_flags; 1058 1059 rp = mp->map_obj.rp; 1060 hp = mp->map_handlep; 1061 1062 #ifdef DDI_MAP_DEBUG 1063 ddi_map_debug( 1064 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1065 rp->regspec_bustype, rp->regspec_addr, 1066 rp->regspec_size, mp->map_handlep); 1067 #endif /* DDI_MAP_DEBUG */ 1068 1069 /* 1070 * I/O or memory mapping 1071 * 1072 * <bustype=0, addr=x, len=x>: memory 1073 * <bustype=1, addr=x, len=x>: i/o 1074 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1075 */ 1076 1077 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1078 cmn_err(CE_WARN, "rootnex: invalid register spec" 1079 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1080 rp->regspec_addr, rp->regspec_size); 1081 return (DDI_FAILURE); 1082 } 1083 1084 if (rp->regspec_bustype != 0) { 1085 /* 1086 * I/O space - needs a handle. 1087 */ 1088 if (hp == NULL) { 1089 return (DDI_FAILURE); 1090 } 1091 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1092 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1093 impl_acc_hdl_init(hp); 1094 1095 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1096 #ifdef DDI_MAP_DEBUG 1097 ddi_map_debug("rootnex_map_regspec: mmap() \ 1098 to I/O space is not supported.\n"); 1099 #endif /* DDI_MAP_DEBUG */ 1100 return (DDI_ME_INVAL); 1101 } else { 1102 /* 1103 * 1275-compliant vs. compatibility i/o mapping 1104 */ 1105 *vaddrp = 1106 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1107 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1108 ((caddr_t)(uintptr_t)rp->regspec_addr); 1109 } 1110 1111 #ifdef DDI_MAP_DEBUG 1112 ddi_map_debug( 1113 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1114 rp->regspec_size, *vaddrp); 1115 #endif /* DDI_MAP_DEBUG */ 1116 return (DDI_SUCCESS); 1117 } 1118 1119 /* 1120 * Memory space 1121 */ 1122 1123 if (hp != NULL) { 1124 /* 1125 * hat layer ignores 1126 * hp->ah_acc.devacc_attr_endian_flags. 1127 */ 1128 switch (hp->ah_acc.devacc_attr_dataorder) { 1129 case DDI_STRICTORDER_ACC: 1130 hat_acc_flags = HAT_STRICTORDER; 1131 break; 1132 case DDI_UNORDERED_OK_ACC: 1133 hat_acc_flags = HAT_UNORDERED_OK; 1134 break; 1135 case DDI_MERGING_OK_ACC: 1136 hat_acc_flags = HAT_MERGING_OK; 1137 break; 1138 case DDI_LOADCACHING_OK_ACC: 1139 hat_acc_flags = HAT_LOADCACHING_OK; 1140 break; 1141 case DDI_STORECACHING_OK_ACC: 1142 hat_acc_flags = HAT_STORECACHING_OK; 1143 break; 1144 } 1145 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1146 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1147 impl_acc_hdl_init(hp); 1148 hp->ah_hat_flags = hat_acc_flags; 1149 } else { 1150 hat_acc_flags = HAT_STRICTORDER; 1151 } 1152 1153 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1154 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1155 1156 if (rp->regspec_size == 0) { 1157 #ifdef DDI_MAP_DEBUG 1158 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1159 #endif /* DDI_MAP_DEBUG */ 1160 return (DDI_ME_INVAL); 1161 } 1162 1163 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1164 *vaddrp = (caddr_t)mmu_btop(base); 1165 } else { 1166 npages = mmu_btopr(rp->regspec_size + pgoffset); 1167 1168 #ifdef DDI_MAP_DEBUG 1169 ddi_map_debug("rootnex_map_regspec: Mapping %d pages \ 1170 physical %x ", 1171 npages, base); 1172 #endif /* DDI_MAP_DEBUG */ 1173 1174 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1175 if (cvaddr == NULL) 1176 return (DDI_ME_NORESOURCES); 1177 1178 /* 1179 * Now map in the pages we've allocated... 1180 */ 1181 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), mmu_btop(base), 1182 mp->map_prot | hat_acc_flags, HAT_LOAD_LOCK); 1183 *vaddrp = (caddr_t)cvaddr + pgoffset; 1184 } 1185 1186 #ifdef DDI_MAP_DEBUG 1187 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1188 #endif /* DDI_MAP_DEBUG */ 1189 return (DDI_SUCCESS); 1190 } 1191 1192 1193 /* 1194 * rootnex_unmap_regspec() 1195 * 1196 */ 1197 static int 1198 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1199 { 1200 caddr_t addr = (caddr_t)*vaddrp; 1201 uint_t npages, pgoffset; 1202 struct regspec *rp; 1203 1204 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1205 return (0); 1206 1207 rp = mp->map_obj.rp; 1208 1209 if (rp->regspec_size == 0) { 1210 #ifdef DDI_MAP_DEBUG 1211 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1212 #endif /* DDI_MAP_DEBUG */ 1213 return (DDI_ME_INVAL); 1214 } 1215 1216 /* 1217 * I/O or memory mapping: 1218 * 1219 * <bustype=0, addr=x, len=x>: memory 1220 * <bustype=1, addr=x, len=x>: i/o 1221 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1222 */ 1223 if (rp->regspec_bustype != 0) { 1224 /* 1225 * This is I/O space, which requires no particular 1226 * processing on unmap since it isn't mapped in the 1227 * first place. 1228 */ 1229 return (DDI_SUCCESS); 1230 } 1231 1232 /* 1233 * Memory space 1234 */ 1235 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1236 npages = mmu_btopr(rp->regspec_size + pgoffset); 1237 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1238 device_arena_free(addr - pgoffset, ptob(npages)); 1239 1240 /* 1241 * Destroy the pointer - the mapping has logically gone 1242 */ 1243 *vaddrp = NULL; 1244 1245 return (DDI_SUCCESS); 1246 } 1247 1248 1249 /* 1250 * rootnex_map_handle() 1251 * 1252 */ 1253 static int 1254 rootnex_map_handle(ddi_map_req_t *mp) 1255 { 1256 ddi_acc_hdl_t *hp; 1257 ulong_t base; 1258 uint_t pgoffset; 1259 struct regspec *rp; 1260 1261 rp = mp->map_obj.rp; 1262 1263 #ifdef DDI_MAP_DEBUG 1264 ddi_map_debug( 1265 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1266 rp->regspec_bustype, rp->regspec_addr, 1267 rp->regspec_size, mp->map_handlep); 1268 #endif /* DDI_MAP_DEBUG */ 1269 1270 /* 1271 * I/O or memory mapping: 1272 * 1273 * <bustype=0, addr=x, len=x>: memory 1274 * <bustype=1, addr=x, len=x>: i/o 1275 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1276 */ 1277 if (rp->regspec_bustype != 0) { 1278 /* 1279 * This refers to I/O space, and we don't support "mapping" 1280 * I/O space to a user. 1281 */ 1282 return (DDI_FAILURE); 1283 } 1284 1285 /* 1286 * Set up the hat_flags for the mapping. 1287 */ 1288 hp = mp->map_handlep; 1289 1290 switch (hp->ah_acc.devacc_attr_endian_flags) { 1291 case DDI_NEVERSWAP_ACC: 1292 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1293 break; 1294 case DDI_STRUCTURE_LE_ACC: 1295 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1296 break; 1297 case DDI_STRUCTURE_BE_ACC: 1298 return (DDI_FAILURE); 1299 default: 1300 return (DDI_REGS_ACC_CONFLICT); 1301 } 1302 1303 switch (hp->ah_acc.devacc_attr_dataorder) { 1304 case DDI_STRICTORDER_ACC: 1305 break; 1306 case DDI_UNORDERED_OK_ACC: 1307 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1308 break; 1309 case DDI_MERGING_OK_ACC: 1310 hp->ah_hat_flags |= HAT_MERGING_OK; 1311 break; 1312 case DDI_LOADCACHING_OK_ACC: 1313 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1314 break; 1315 case DDI_STORECACHING_OK_ACC: 1316 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1317 break; 1318 default: 1319 return (DDI_FAILURE); 1320 } 1321 1322 base = (ulong_t)rp->regspec_addr & (~MMU_PAGEOFFSET); /* base addr */ 1323 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; /* offset */ 1324 1325 if (rp->regspec_size == 0) 1326 return (DDI_ME_INVAL); 1327 1328 hp->ah_pfn = mmu_btop(base); 1329 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1330 1331 return (DDI_SUCCESS); 1332 } 1333 1334 1335 1336 /* 1337 * ************************ 1338 * interrupt related code 1339 * ************************ 1340 */ 1341 1342 /* 1343 * rootnex_intr_ops() 1344 * bus_intr_op() function for interrupt support 1345 */ 1346 /* ARGSUSED */ 1347 static int 1348 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1349 ddi_intr_handle_impl_t *hdlp, void *result) 1350 { 1351 struct intrspec *ispec; 1352 struct ddi_parent_private_data *pdp; 1353 1354 DDI_INTR_NEXDBG((CE_CONT, 1355 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1356 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1357 1358 /* Process the interrupt operation */ 1359 switch (intr_op) { 1360 case DDI_INTROP_GETCAP: 1361 /* First check with pcplusmp */ 1362 if (psm_intr_ops == NULL) 1363 return (DDI_FAILURE); 1364 1365 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1366 *(int *)result = 0; 1367 return (DDI_FAILURE); 1368 } 1369 break; 1370 case DDI_INTROP_SETCAP: 1371 if (psm_intr_ops == NULL) 1372 return (DDI_FAILURE); 1373 1374 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1375 return (DDI_FAILURE); 1376 break; 1377 case DDI_INTROP_ALLOC: 1378 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1379 return (DDI_FAILURE); 1380 hdlp->ih_pri = ispec->intrspec_pri; 1381 *(int *)result = hdlp->ih_scratch1; 1382 break; 1383 case DDI_INTROP_FREE: 1384 pdp = ddi_get_parent_data(rdip); 1385 /* 1386 * Special case for 'pcic' driver' only. 1387 * If an intrspec was created for it, clean it up here 1388 * See detailed comments on this in the function 1389 * rootnex_get_ispec(). 1390 */ 1391 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1392 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1393 pdp->par_nintr); 1394 /* 1395 * Set it to zero; so that 1396 * DDI framework doesn't free it again 1397 */ 1398 pdp->par_intr = NULL; 1399 pdp->par_nintr = 0; 1400 } 1401 break; 1402 case DDI_INTROP_GETPRI: 1403 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1404 return (DDI_FAILURE); 1405 *(int *)result = ispec->intrspec_pri; 1406 break; 1407 case DDI_INTROP_SETPRI: 1408 /* Validate the interrupt priority passed to us */ 1409 if (*(int *)result > LOCK_LEVEL) 1410 return (DDI_FAILURE); 1411 1412 /* Ensure that PSM is all initialized and ispec is ok */ 1413 if ((psm_intr_ops == NULL) || 1414 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1415 return (DDI_FAILURE); 1416 1417 /* Change the priority */ 1418 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1419 PSM_FAILURE) 1420 return (DDI_FAILURE); 1421 1422 /* update the ispec with the new priority */ 1423 ispec->intrspec_pri = *(int *)result; 1424 break; 1425 case DDI_INTROP_ADDISR: 1426 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1427 return (DDI_FAILURE); 1428 ispec->intrspec_func = hdlp->ih_cb_func; 1429 break; 1430 case DDI_INTROP_REMISR: 1431 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1432 return (DDI_FAILURE); 1433 ispec->intrspec_func = (uint_t (*)()) 0; 1434 break; 1435 case DDI_INTROP_ENABLE: 1436 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1437 return (DDI_FAILURE); 1438 1439 /* Call psmi to translate irq with the dip */ 1440 if (psm_intr_ops == NULL) 1441 return (DDI_FAILURE); 1442 1443 hdlp->ih_private = (void *)ispec; 1444 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1445 (int *)&hdlp->ih_vector); 1446 1447 /* Add the interrupt handler */ 1448 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1449 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1450 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, rdip)) 1451 return (DDI_FAILURE); 1452 break; 1453 case DDI_INTROP_DISABLE: 1454 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1455 return (DDI_FAILURE); 1456 1457 /* Call psm_ops() to translate irq with the dip */ 1458 if (psm_intr_ops == NULL) 1459 return (DDI_FAILURE); 1460 1461 hdlp->ih_private = (void *)ispec; 1462 (void) (*psm_intr_ops)(rdip, hdlp, 1463 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1464 1465 /* Remove the interrupt handler */ 1466 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1467 hdlp->ih_cb_func, hdlp->ih_vector); 1468 break; 1469 case DDI_INTROP_SETMASK: 1470 if (psm_intr_ops == NULL) 1471 return (DDI_FAILURE); 1472 1473 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1474 return (DDI_FAILURE); 1475 break; 1476 case DDI_INTROP_CLRMASK: 1477 if (psm_intr_ops == NULL) 1478 return (DDI_FAILURE); 1479 1480 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1481 return (DDI_FAILURE); 1482 break; 1483 case DDI_INTROP_GETPENDING: 1484 if (psm_intr_ops == NULL) 1485 return (DDI_FAILURE); 1486 1487 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1488 result)) { 1489 *(int *)result = 0; 1490 return (DDI_FAILURE); 1491 } 1492 break; 1493 case DDI_INTROP_NINTRS: 1494 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 1495 return (DDI_FAILURE); 1496 *(int *)result = pdp->par_nintr; 1497 if (pdp->par_nintr == 0) { 1498 /* 1499 * Special case for 'pcic' driver' only. This driver 1500 * driver is a child of 'isa' and 'rootnex' drivers. 1501 * 1502 * See detailed comments on this in the function 1503 * rootnex_get_ispec(). 1504 * 1505 * Children of 'pcic' send 'NINITR' request all the 1506 * way to rootnex driver. But, the 'pdp->par_nintr' 1507 * field may not initialized. So, we fake it here 1508 * to return 1 (a la what PCMCIA nexus does). 1509 */ 1510 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1511 *(int *)result = 1; 1512 } 1513 break; 1514 case DDI_INTROP_SUPPORTED_TYPES: 1515 *(int *)result = 0; 1516 *(int *)result |= DDI_INTR_TYPE_FIXED; /* Always ... */ 1517 break; 1518 case DDI_INTROP_NAVAIL: 1519 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1520 return (DDI_FAILURE); 1521 1522 if (psm_intr_ops == NULL) { 1523 *(int *)result = 1; 1524 break; 1525 } 1526 1527 /* Priority in the handle not initialized yet */ 1528 hdlp->ih_pri = ispec->intrspec_pri; 1529 (void) (*psm_intr_ops)(rdip, hdlp, 1530 PSM_INTR_OP_NAVAIL_VECTORS, result); 1531 break; 1532 default: 1533 return (DDI_FAILURE); 1534 } 1535 1536 return (DDI_SUCCESS); 1537 } 1538 1539 1540 /* 1541 * rootnex_get_ispec() 1542 * convert an interrupt number to an interrupt specification. 1543 * The interrupt number determines which interrupt spec will be 1544 * returned if more than one exists. 1545 * 1546 * Look into the parent private data area of the 'rdip' to find out 1547 * the interrupt specification. First check to make sure there is 1548 * one that matchs "inumber" and then return a pointer to it. 1549 * 1550 * Return NULL if one could not be found. 1551 * 1552 * NOTE: This is needed for rootnex_intr_ops() 1553 */ 1554 static struct intrspec * 1555 rootnex_get_ispec(dev_info_t *rdip, int inum) 1556 { 1557 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1558 1559 /* 1560 * Special case handling for drivers that provide their own 1561 * intrspec structures instead of relying on the DDI framework. 1562 * 1563 * A broken hardware driver in ON could potentially provide its 1564 * own intrspec structure, instead of relying on the hardware. 1565 * If these drivers are children of 'rootnex' then we need to 1566 * continue to provide backward compatibility to them here. 1567 * 1568 * Following check is a special case for 'pcic' driver which 1569 * was found to have broken hardwre andby provides its own intrspec. 1570 * 1571 * Verbatim comments from this driver are shown here: 1572 * "Don't use the ddi_add_intr since we don't have a 1573 * default intrspec in all cases." 1574 * 1575 * Since an 'ispec' may not be always created for it, 1576 * check for that and create one if so. 1577 * 1578 * NOTE: Currently 'pcic' is the only driver found to do this. 1579 */ 1580 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1581 pdp->par_nintr = 1; 1582 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1583 pdp->par_nintr, KM_SLEEP); 1584 } 1585 1586 /* Validate the interrupt number */ 1587 if (inum >= pdp->par_nintr) 1588 return (NULL); 1589 1590 /* Get the interrupt structure pointer and return that */ 1591 return ((struct intrspec *)&pdp->par_intr[inum]); 1592 } 1593 1594 1595 /* 1596 * rootnex_xlate_intrs() 1597 * For the x86 rootnexus, we're prepared to claim that the interrupt string 1598 * is in the form of a list of <ipl,vec> specifications. 1599 */ 1600 static int 1601 rootnex_xlate_intrs(dev_info_t *dip, dev_info_t *rdip, int *in, 1602 struct ddi_parent_private_data *pdptr) 1603 { 1604 size_t size; 1605 int n; 1606 struct intrspec *new; 1607 caddr_t got_prop; 1608 int *inpri; 1609 int got_len; 1610 1611 static char bad_intr_fmt[] = 1612 "rootnex: bad interrupt spec from %s%d - ipl %d, irq %d\n"; 1613 1614 #ifdef lint 1615 dip = dip; 1616 #endif 1617 /* 1618 * determine if the driver is expecting the new style "interrupts" 1619 * property which just contains the IRQ, or the old style which 1620 * contains pairs of <IPL,IRQ>. if it is the new style, we always 1621 * assign IPL 5 unless an "interrupt-priorities" property exists. 1622 * in that case, the "interrupt-priorities" property contains the 1623 * IPL values that match, one for one, the IRQ values in the 1624 * "interrupts" property. 1625 */ 1626 inpri = NULL; 1627 if ((ddi_getprop(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS, 1628 "ignore-hardware-nodes", -1) != -1) || 1629 ignore_hardware_nodes) { 1630 /* the old style "interrupts" property... */ 1631 1632 /* 1633 * The list consists of <ipl,vec> elements 1634 */ 1635 if ((n = (*in++ >> 1)) < 1) 1636 return (DDI_FAILURE); 1637 1638 pdptr->par_nintr = n; 1639 size = n * sizeof (struct intrspec); 1640 new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP); 1641 1642 while (n--) { 1643 int level = *in++; 1644 int vec = *in++; 1645 1646 if (level < 1 || level > MAXIPL || 1647 vec < VEC_MIN || vec > VEC_MAX) { 1648 cmn_err(CE_CONT, bad_intr_fmt, 1649 DEVI(rdip)->devi_name, 1650 DEVI(rdip)->devi_instance, level, vec); 1651 goto broken; 1652 } 1653 new->intrspec_pri = level; 1654 if (vec != 2) 1655 new->intrspec_vec = vec; 1656 else 1657 /* 1658 * irq 2 on the PC bus is tied to irq 9 1659 * on ISA, EISA and MicroChannel 1660 */ 1661 new->intrspec_vec = 9; 1662 new++; 1663 } 1664 1665 return (DDI_SUCCESS); 1666 } else { 1667 /* the new style "interrupts" property... */ 1668 1669 /* 1670 * The list consists of <vec> elements 1671 */ 1672 if ((n = (*in++)) < 1) 1673 return (DDI_FAILURE); 1674 1675 pdptr->par_nintr = n; 1676 size = n * sizeof (struct intrspec); 1677 new = pdptr->par_intr = kmem_zalloc(size, KM_SLEEP); 1678 1679 /* XXX check for "interrupt-priorities" property... */ 1680 if (ddi_getlongprop(DDI_DEV_T_ANY, rdip, DDI_PROP_DONTPASS, 1681 "interrupt-priorities", (caddr_t)&got_prop, &got_len) 1682 == DDI_PROP_SUCCESS) { 1683 if (n != (got_len / sizeof (int))) { 1684 cmn_err(CE_CONT, 1685 "rootnex: bad interrupt-priorities length" 1686 " from %s%d: expected %d, got %d\n", 1687 DEVI(rdip)->devi_name, 1688 DEVI(rdip)->devi_instance, n, 1689 (int)(got_len / sizeof (int))); 1690 goto broken; 1691 } 1692 inpri = (int *)got_prop; 1693 } 1694 1695 while (n--) { 1696 int level; 1697 int vec = *in++; 1698 1699 if (inpri == NULL) 1700 level = 5; 1701 else 1702 level = *inpri++; 1703 1704 if (level < 1 || level > MAXIPL || 1705 vec < VEC_MIN || vec > VEC_MAX) { 1706 cmn_err(CE_CONT, bad_intr_fmt, 1707 DEVI(rdip)->devi_name, 1708 DEVI(rdip)->devi_instance, level, vec); 1709 goto broken; 1710 } 1711 new->intrspec_pri = level; 1712 if (vec != 2) 1713 new->intrspec_vec = vec; 1714 else 1715 /* 1716 * irq 2 on the PC bus is tied to irq 9 1717 * on ISA, EISA and MicroChannel 1718 */ 1719 new->intrspec_vec = 9; 1720 new++; 1721 } 1722 1723 if (inpri != NULL) 1724 kmem_free(got_prop, got_len); 1725 return (DDI_SUCCESS); 1726 } 1727 1728 broken: 1729 kmem_free(pdptr->par_intr, size); 1730 pdptr->par_intr = NULL; 1731 pdptr->par_nintr = 0; 1732 if (inpri != NULL) 1733 kmem_free(got_prop, got_len); 1734 return (DDI_FAILURE); 1735 } 1736 1737 1738 /* 1739 * ****************** 1740 * dma related code 1741 * ****************** 1742 */ 1743 1744 /* 1745 * rootnex_dma_allochdl() 1746 * called from ddi_dma_alloc_handle(). 1747 */ 1748 /*ARGSUSED*/ 1749 static int 1750 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1751 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1752 { 1753 uint64_t maxsegmentsize_ll; 1754 uint_t maxsegmentsize; 1755 ddi_dma_impl_t *hp; 1756 rootnex_dma_t *dma; 1757 uint64_t count_max; 1758 uint64_t seg; 1759 int kmflag; 1760 int e; 1761 1762 1763 /* convert our sleep flags */ 1764 if (waitfp == DDI_DMA_SLEEP) { 1765 kmflag = KM_SLEEP; 1766 } else { 1767 kmflag = KM_NOSLEEP; 1768 } 1769 1770 /* 1771 * We try to do only one memory allocation here. We'll do a little 1772 * pointer manipulation later. If the bind ends up taking more than 1773 * our prealloc's space, we'll have to allocate more memory in the 1774 * bind operation. Not great, but much better than before and the 1775 * best we can do with the current bind interfaces. 1776 */ 1777 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1778 if (hp == NULL) { 1779 if (waitfp != DDI_DMA_DONTWAIT) { 1780 ddi_set_callback(waitfp, arg, 1781 &rootnex_state->r_dvma_call_list_id); 1782 } 1783 return (DDI_DMA_NORESOURCES); 1784 } 1785 1786 /* Do our pointer manipulation now, align the structures */ 1787 hp->dmai_private = (void *)(((uintptr_t)hp + 1788 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1789 dma = (rootnex_dma_t *)hp->dmai_private; 1790 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1791 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1792 1793 /* setup the handle */ 1794 rootnex_clean_dmahdl(hp); 1795 dma->dp_dip = rdip; 1796 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1797 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1798 hp->dmai_minxfer = attr->dma_attr_minxfer; 1799 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1800 hp->dmai_rdip = rdip; 1801 hp->dmai_attr = *attr; 1802 1803 /* we don't need to worry about the SPL since we do a tryenter */ 1804 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1805 1806 /* 1807 * Figure out our maximum segment size. If the segment size is greater 1808 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1809 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1810 * dma_attr_count_max are size-1 type values. 1811 * 1812 * Maximum segment size is the largest physically contiguous chunk of 1813 * memory that we can return from a bind (i.e. the maximum size of a 1814 * single cookie). 1815 */ 1816 1817 /* handle the rollover cases */ 1818 seg = attr->dma_attr_seg + 1; 1819 if (seg < attr->dma_attr_seg) { 1820 seg = attr->dma_attr_seg; 1821 } 1822 count_max = attr->dma_attr_count_max + 1; 1823 if (count_max < attr->dma_attr_count_max) { 1824 count_max = attr->dma_attr_count_max; 1825 } 1826 1827 /* 1828 * granularity may or may not be a power of two. If it isn't, we can't 1829 * use a simple mask. 1830 */ 1831 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1832 dma->dp_granularity_power_2 = B_FALSE; 1833 } else { 1834 dma->dp_granularity_power_2 = B_TRUE; 1835 } 1836 1837 /* 1838 * maxxfer should be a whole multiple of granularity. If we're going to 1839 * break up a window because we're greater than maxxfer, we might as 1840 * well make sure it's maxxfer is a whole multiple so we don't have to 1841 * worry about triming the window later on for this case. 1842 */ 1843 if (attr->dma_attr_granular > 1) { 1844 if (dma->dp_granularity_power_2) { 1845 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1846 (attr->dma_attr_maxxfer & 1847 (attr->dma_attr_granular - 1)); 1848 } else { 1849 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1850 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1851 } 1852 } else { 1853 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1854 } 1855 1856 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1857 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1858 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1859 maxsegmentsize = 0xFFFFFFFF; 1860 } else { 1861 maxsegmentsize = maxsegmentsize_ll; 1862 } 1863 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1864 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1865 1866 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1867 if (rootnex_alloc_check_parms) { 1868 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1869 if (e != DDI_SUCCESS) { 1870 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1871 (void) rootnex_dma_freehdl(dip, rdip, 1872 (ddi_dma_handle_t)hp); 1873 return (e); 1874 } 1875 } 1876 1877 *handlep = (ddi_dma_handle_t)hp; 1878 1879 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1880 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1881 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1882 1883 return (DDI_SUCCESS); 1884 } 1885 1886 1887 /* 1888 * rootnex_dma_freehdl() 1889 * called from ddi_dma_free_handle(). 1890 */ 1891 /*ARGSUSED*/ 1892 static int 1893 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1894 { 1895 ddi_dma_impl_t *hp; 1896 rootnex_dma_t *dma; 1897 1898 1899 hp = (ddi_dma_impl_t *)handle; 1900 dma = (rootnex_dma_t *)hp->dmai_private; 1901 1902 /* unbind should have been called first */ 1903 ASSERT(!dma->dp_inuse); 1904 1905 mutex_destroy(&dma->dp_mutex); 1906 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1907 1908 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1909 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1910 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1911 1912 if (rootnex_state->r_dvma_call_list_id) 1913 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1914 1915 return (DDI_SUCCESS); 1916 } 1917 1918 1919 /* 1920 * rootnex_dma_bindhdl() 1921 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1922 */ 1923 /*ARGSUSED*/ 1924 static int 1925 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1926 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1927 { 1928 rootnex_sglinfo_t *sinfo; 1929 ddi_dma_attr_t *attr; 1930 ddi_dma_impl_t *hp; 1931 rootnex_dma_t *dma; 1932 int kmflag; 1933 int e; 1934 1935 1936 hp = (ddi_dma_impl_t *)handle; 1937 dma = (rootnex_dma_t *)hp->dmai_private; 1938 sinfo = &dma->dp_sglinfo; 1939 attr = &hp->dmai_attr; 1940 1941 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1942 1943 /* 1944 * This is useful for debugging a driver. Not as useful in a production 1945 * system. The only time this will fail is if you have a driver bug. 1946 */ 1947 if (rootnex_bind_check_inuse) { 1948 /* 1949 * No one else should ever have this lock unless someone else 1950 * is trying to use this handle. So contention on the lock 1951 * is the same as inuse being set. 1952 */ 1953 e = mutex_tryenter(&dma->dp_mutex); 1954 if (e == 0) { 1955 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1956 return (DDI_DMA_INUSE); 1957 } 1958 if (dma->dp_inuse) { 1959 mutex_exit(&dma->dp_mutex); 1960 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1961 return (DDI_DMA_INUSE); 1962 } 1963 dma->dp_inuse = B_TRUE; 1964 mutex_exit(&dma->dp_mutex); 1965 } 1966 1967 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1968 if (rootnex_bind_check_parms) { 1969 e = rootnex_valid_bind_parms(dmareq, attr); 1970 if (e != DDI_SUCCESS) { 1971 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1972 rootnex_clean_dmahdl(hp); 1973 return (e); 1974 } 1975 } 1976 1977 /* save away the original bind info */ 1978 dma->dp_dma = dmareq->dmar_object; 1979 1980 /* 1981 * Figure out a rough estimate of what maximum number of pages this 1982 * buffer could use (a high estimate of course). 1983 */ 1984 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1985 1986 /* 1987 * We'll use the pre-allocated cookies for any bind that will *always* 1988 * fit (more important to be consistent, we don't want to create 1989 * additional degenerate cases). 1990 */ 1991 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1992 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1993 dma->dp_need_to_free_cookie = B_FALSE; 1994 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1995 uint_t, sinfo->si_max_pages); 1996 1997 /* 1998 * For anything larger than that, we'll go ahead and allocate the 1999 * maximum number of pages we expect to see. Hopefuly, we won't be 2000 * seeing this path in the fast path for high performance devices very 2001 * frequently. 2002 * 2003 * a ddi bind interface that allowed the driver to provide storage to 2004 * the bind interface would speed this case up. 2005 */ 2006 } else { 2007 /* convert the sleep flags */ 2008 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2009 kmflag = KM_SLEEP; 2010 } else { 2011 kmflag = KM_NOSLEEP; 2012 } 2013 2014 /* 2015 * Save away how much memory we allocated. If we're doing a 2016 * nosleep, the alloc could fail... 2017 */ 2018 dma->dp_cookie_size = sinfo->si_max_pages * 2019 sizeof (ddi_dma_cookie_t); 2020 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2021 if (dma->dp_cookies == NULL) { 2022 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2023 rootnex_clean_dmahdl(hp); 2024 return (DDI_DMA_NORESOURCES); 2025 } 2026 dma->dp_need_to_free_cookie = B_TRUE; 2027 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 2028 sinfo->si_max_pages); 2029 } 2030 hp->dmai_cookie = dma->dp_cookies; 2031 2032 /* 2033 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2034 * looking at the contraints in the dma structure. It will then put some 2035 * additional state about the sgl in the dma struct (i.e. is the sgl 2036 * clean, or do we need to do some munging; how many pages need to be 2037 * copied, etc.) 2038 */ 2039 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 2040 &dma->dp_sglinfo); 2041 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2042 2043 /* if we don't need a copy buffer, we don't need to sync */ 2044 if (sinfo->si_copybuf_req == 0) { 2045 hp->dmai_rflags |= DMP_NOSYNC; 2046 } 2047 2048 /* 2049 * if we don't need the copybuf and we don't need to do a partial, we 2050 * hit the fast path. All the high performance devices should be trying 2051 * to hit this path. To hit this path, a device should be able to reach 2052 * all of memory, shouldn't try to bind more than it can transfer, and 2053 * the buffer shouldn't require more cookies than the driver/device can 2054 * handle [sgllen]). 2055 */ 2056 if ((sinfo->si_copybuf_req == 0) && 2057 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 2058 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 2059 /* 2060 * copy out the first cookie and ccountp, set the cookie 2061 * pointer to the second cookie. The first cookie is passed 2062 * back on the stack. Additional cookies are accessed via 2063 * ddi_dma_nextcookie() 2064 */ 2065 *cookiep = dma->dp_cookies[0]; 2066 *ccountp = sinfo->si_sgl_size; 2067 hp->dmai_cookie++; 2068 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2069 hp->dmai_nwin = 1; 2070 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2071 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 2072 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2073 dma->dp_dma.dmao_size); 2074 return (DDI_DMA_MAPPED); 2075 } 2076 2077 /* 2078 * go to the slow path, we may need to alloc more memory, create 2079 * multiple windows, and munge up a sgl to make the device happy. 2080 */ 2081 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2082 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2083 if (dma->dp_need_to_free_cookie) { 2084 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2085 } 2086 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2087 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2088 return (e); 2089 } 2090 2091 /* if the first window uses the copy buffer, sync it for the device */ 2092 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2093 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2094 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2095 DDI_DMA_SYNC_FORDEV); 2096 } 2097 2098 /* 2099 * copy out the first cookie and ccountp, set the cookie pointer to the 2100 * second cookie. Make sure the partial flag is set/cleared correctly. 2101 * If we have a partial map (i.e. multiple windows), the number of 2102 * cookies we return is the number of cookies in the first window. 2103 */ 2104 if (e == DDI_DMA_MAPPED) { 2105 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2106 *ccountp = sinfo->si_sgl_size; 2107 } else { 2108 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2109 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2110 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2111 } 2112 *cookiep = dma->dp_cookies[0]; 2113 hp->dmai_cookie++; 2114 2115 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2116 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2117 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2118 dma->dp_dma.dmao_size); 2119 return (e); 2120 } 2121 2122 2123 /* 2124 * rootnex_dma_unbindhdl() 2125 * called from ddi_dma_unbind_handle() 2126 */ 2127 /*ARGSUSED*/ 2128 static int 2129 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2130 ddi_dma_handle_t handle) 2131 { 2132 ddi_dma_impl_t *hp; 2133 rootnex_dma_t *dma; 2134 int e; 2135 2136 2137 hp = (ddi_dma_impl_t *)handle; 2138 dma = (rootnex_dma_t *)hp->dmai_private; 2139 2140 /* make sure the buffer wasn't free'd before calling unbind */ 2141 if (rootnex_unbind_verify_buffer) { 2142 e = rootnex_verify_buffer(dma); 2143 if (e != DDI_SUCCESS) { 2144 ASSERT(0); 2145 return (DDI_FAILURE); 2146 } 2147 } 2148 2149 /* sync the current window before unbinding the buffer */ 2150 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2151 (hp->dmai_rflags & DDI_DMA_READ)) { 2152 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2153 DDI_DMA_SYNC_FORCPU); 2154 } 2155 2156 /* 2157 * cleanup and copy buffer or window state. if we didn't use the copy 2158 * buffer or windows, there won't be much to do :-) 2159 */ 2160 rootnex_teardown_copybuf(dma); 2161 rootnex_teardown_windows(dma); 2162 2163 /* 2164 * If we had to allocate space to for the worse case sgl (it didn't 2165 * fit into our pre-allocate buffer), free that up now 2166 */ 2167 if (dma->dp_need_to_free_cookie) { 2168 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2169 } 2170 2171 /* 2172 * clean up the handle so it's ready for the next bind (i.e. if the 2173 * handle is reused). 2174 */ 2175 rootnex_clean_dmahdl(hp); 2176 2177 if (rootnex_state->r_dvma_call_list_id) 2178 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2179 2180 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2181 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2182 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2183 2184 return (DDI_SUCCESS); 2185 } 2186 2187 2188 /* 2189 * rootnex_verify_buffer() 2190 * verify buffer wasn't free'd 2191 */ 2192 static int 2193 rootnex_verify_buffer(rootnex_dma_t *dma) 2194 { 2195 peekpoke_ctlops_t peek; 2196 page_t **pplist; 2197 caddr_t vaddr; 2198 uint_t pcnt; 2199 uint_t poff; 2200 page_t *pp; 2201 uint8_t b; 2202 int i; 2203 int e; 2204 2205 2206 /* Figure out how many pages this buffer occupies */ 2207 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2208 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2209 } else { 2210 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2211 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2212 } 2213 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2214 2215 switch (dma->dp_dma.dmao_type) { 2216 case DMA_OTYP_PAGES: 2217 /* 2218 * for a linked list of pp's walk through them to make sure 2219 * they're locked and not free. 2220 */ 2221 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2222 for (i = 0; i < pcnt; i++) { 2223 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2224 return (DDI_FAILURE); 2225 } 2226 pp = pp->p_next; 2227 } 2228 break; 2229 2230 case DMA_OTYP_VADDR: 2231 case DMA_OTYP_BUFVADDR: 2232 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2233 /* 2234 * for an array of pp's walk through them to make sure they're 2235 * not free. It's possible that they may not be locked. 2236 */ 2237 if (pplist) { 2238 for (i = 0; i < pcnt; i++) { 2239 if (PP_ISFREE(pplist[i])) { 2240 return (DDI_FAILURE); 2241 } 2242 } 2243 2244 /* For a virtual address, try to peek at each page */ 2245 } else { 2246 if (dma->dp_sglinfo.si_asp == &kas) { 2247 bzero(&peek, sizeof (peekpoke_ctlops_t)); 2248 peek.host_addr = (uintptr_t)&b; 2249 peek.size = sizeof (uint8_t); 2250 peek.dev_addr = (uintptr_t)vaddr; 2251 for (i = 0; i < pcnt; i++) { 2252 e = rootnex_ctlops_peek(&peek, &b); 2253 if (e != DDI_SUCCESS) { 2254 return (DDI_FAILURE); 2255 } 2256 peek.dev_addr += MMU_PAGESIZE; 2257 } 2258 } 2259 } 2260 break; 2261 2262 default: 2263 ASSERT(0); 2264 break; 2265 } 2266 2267 return (DDI_SUCCESS); 2268 } 2269 2270 2271 /* 2272 * rootnex_clean_dmahdl() 2273 * Clean the dma handle. This should be called on a handle alloc and an 2274 * unbind handle. Set the handle state to the default settings. 2275 */ 2276 static void 2277 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2278 { 2279 rootnex_dma_t *dma; 2280 2281 2282 dma = (rootnex_dma_t *)hp->dmai_private; 2283 2284 hp->dmai_nwin = 0; 2285 dma->dp_current_cookie = 0; 2286 dma->dp_copybuf_size = 0; 2287 dma->dp_window = NULL; 2288 dma->dp_cbaddr = NULL; 2289 dma->dp_inuse = B_FALSE; 2290 dma->dp_need_to_free_cookie = B_FALSE; 2291 dma->dp_need_to_free_window = B_FALSE; 2292 dma->dp_partial_required = B_FALSE; 2293 dma->dp_trim_required = B_FALSE; 2294 dma->dp_sglinfo.si_copybuf_req = 0; 2295 #if !defined(__amd64) 2296 dma->dp_cb_remaping = B_FALSE; 2297 dma->dp_kva = NULL; 2298 #endif 2299 2300 /* FMA related initialization */ 2301 hp->dmai_fault = 0; 2302 hp->dmai_fault_check = NULL; 2303 hp->dmai_fault_notify = NULL; 2304 hp->dmai_error.err_ena = 0; 2305 hp->dmai_error.err_status = DDI_FM_OK; 2306 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2307 hp->dmai_error.err_ontrap = NULL; 2308 hp->dmai_error.err_fep = NULL; 2309 } 2310 2311 2312 /* 2313 * rootnex_valid_alloc_parms() 2314 * Called in ddi_dma_alloc_handle path to validate its parameters. 2315 */ 2316 static int 2317 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2318 { 2319 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2320 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2321 (attr->dma_attr_granular > MMU_PAGESIZE) || 2322 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2323 return (DDI_DMA_BADATTR); 2324 } 2325 2326 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2327 return (DDI_DMA_BADATTR); 2328 } 2329 2330 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2331 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2332 attr->dma_attr_sgllen <= 0) { 2333 return (DDI_DMA_BADATTR); 2334 } 2335 2336 /* We should be able to DMA into every byte offset in a page */ 2337 if (maxsegmentsize < MMU_PAGESIZE) { 2338 return (DDI_DMA_BADATTR); 2339 } 2340 2341 return (DDI_SUCCESS); 2342 } 2343 2344 2345 /* 2346 * rootnex_valid_bind_parms() 2347 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2348 */ 2349 /* ARGSUSED */ 2350 static int 2351 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2352 { 2353 #if !defined(__amd64) 2354 /* 2355 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2356 * we can track the offset for the obsoleted interfaces. 2357 */ 2358 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2359 return (DDI_DMA_TOOBIG); 2360 } 2361 #endif 2362 2363 return (DDI_SUCCESS); 2364 } 2365 2366 2367 /* 2368 * rootnex_get_sgl() 2369 * Called in bind fastpath to get the sgl. Most of this will be replaced 2370 * with a call to the vm layer when vm2.0 comes around... 2371 */ 2372 static void 2373 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2374 rootnex_sglinfo_t *sglinfo) 2375 { 2376 ddi_dma_atyp_t buftype; 2377 uint64_t last_page; 2378 uint64_t offset; 2379 uint64_t addrhi; 2380 uint64_t addrlo; 2381 uint64_t maxseg; 2382 page_t **pplist; 2383 uint64_t paddr; 2384 uint32_t psize; 2385 uint32_t size; 2386 caddr_t vaddr; 2387 uint_t pcnt; 2388 page_t *pp; 2389 uint_t cnt; 2390 2391 2392 /* shortcuts */ 2393 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2394 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2395 maxseg = sglinfo->si_max_cookie_size; 2396 buftype = dmar_object->dmao_type; 2397 addrhi = sglinfo->si_max_addr; 2398 addrlo = sglinfo->si_min_addr; 2399 size = dmar_object->dmao_size; 2400 2401 pcnt = 0; 2402 cnt = 0; 2403 2404 /* 2405 * if we were passed down a linked list of pages, i.e. pointer to 2406 * page_t, use this to get our physical address and buf offset. 2407 */ 2408 if (buftype == DMA_OTYP_PAGES) { 2409 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2410 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2411 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2412 MMU_PAGEOFFSET; 2413 paddr = ptob64(pp->p_pagenum) + offset; 2414 psize = MIN(size, (MMU_PAGESIZE - offset)); 2415 pp = pp->p_next; 2416 sglinfo->si_asp = NULL; 2417 2418 /* 2419 * We weren't passed down a linked list of pages, but if we were passed 2420 * down an array of pages, use this to get our physical address and buf 2421 * offset. 2422 */ 2423 } else if (pplist != NULL) { 2424 ASSERT((buftype == DMA_OTYP_VADDR) || 2425 (buftype == DMA_OTYP_BUFVADDR)); 2426 2427 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2428 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2429 if (sglinfo->si_asp == NULL) { 2430 sglinfo->si_asp = &kas; 2431 } 2432 2433 ASSERT(!PP_ISFREE(pplist[pcnt])); 2434 paddr = ptob64(pplist[pcnt]->p_pagenum); 2435 paddr += offset; 2436 psize = MIN(size, (MMU_PAGESIZE - offset)); 2437 pcnt++; 2438 2439 /* 2440 * All we have is a virtual address, we'll need to call into the VM 2441 * to get the physical address. 2442 */ 2443 } else { 2444 ASSERT((buftype == DMA_OTYP_VADDR) || 2445 (buftype == DMA_OTYP_BUFVADDR)); 2446 2447 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2448 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2449 if (sglinfo->si_asp == NULL) { 2450 sglinfo->si_asp = &kas; 2451 } 2452 2453 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2454 paddr += offset; 2455 psize = MIN(size, (MMU_PAGESIZE - offset)); 2456 vaddr += psize; 2457 } 2458 2459 /* 2460 * Setup the first cookie with the physical address of the page and the 2461 * size of the page (which takes into account the initial offset into 2462 * the page. 2463 */ 2464 sgl[cnt].dmac_laddress = paddr; 2465 sgl[cnt].dmac_size = psize; 2466 sgl[cnt].dmac_type = 0; 2467 2468 /* 2469 * Save away the buffer offset into the page. We'll need this later in 2470 * the copy buffer code to help figure out the page index within the 2471 * buffer and the offset into the current page. 2472 */ 2473 sglinfo->si_buf_offset = offset; 2474 2475 /* 2476 * If the DMA engine can't reach the physical address, increase how 2477 * much copy buffer we need. We always increase by pagesize so we don't 2478 * have to worry about converting offsets. Set a flag in the cookies 2479 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2480 * last cookie, go to the next cookie (since we separate each page which 2481 * uses the copy buffer in case the copy buffer is not physically 2482 * contiguous. 2483 */ 2484 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2485 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2486 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2487 if ((cnt + 1) < sglinfo->si_max_pages) { 2488 cnt++; 2489 sgl[cnt].dmac_laddress = 0; 2490 sgl[cnt].dmac_size = 0; 2491 sgl[cnt].dmac_type = 0; 2492 } 2493 } 2494 2495 /* 2496 * save this page's physical address so we can figure out if the next 2497 * page is physically contiguous. Keep decrementing size until we are 2498 * done with the buffer. 2499 */ 2500 last_page = paddr & MMU_PAGEMASK; 2501 size -= psize; 2502 2503 while (size > 0) { 2504 /* Get the size for this page (i.e. partial or full page) */ 2505 psize = MIN(size, MMU_PAGESIZE); 2506 2507 if (buftype == DMA_OTYP_PAGES) { 2508 /* get the paddr from the page_t */ 2509 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2510 paddr = ptob64(pp->p_pagenum); 2511 pp = pp->p_next; 2512 } else if (pplist != NULL) { 2513 /* index into the array of page_t's to get the paddr */ 2514 ASSERT(!PP_ISFREE(pplist[pcnt])); 2515 paddr = ptob64(pplist[pcnt]->p_pagenum); 2516 pcnt++; 2517 } else { 2518 /* call into the VM to get the paddr */ 2519 paddr = ptob64(hat_getpfnum(sglinfo->si_asp->a_hat, 2520 vaddr)); 2521 vaddr += psize; 2522 } 2523 2524 /* check to see if this page needs the copy buffer */ 2525 if ((paddr < addrlo) || ((paddr + psize) > addrhi)) { 2526 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2527 2528 /* 2529 * if there is something in the current cookie, go to 2530 * the next one. We only want one page in a cookie which 2531 * uses the copybuf since the copybuf doesn't have to 2532 * be physically contiguous. 2533 */ 2534 if (sgl[cnt].dmac_size != 0) { 2535 cnt++; 2536 } 2537 sgl[cnt].dmac_laddress = paddr; 2538 sgl[cnt].dmac_size = psize; 2539 #if defined(__amd64) 2540 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2541 #else 2542 /* 2543 * save the buf offset for 32-bit kernel. used in the 2544 * obsoleted interfaces. 2545 */ 2546 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2547 (dmar_object->dmao_size - size); 2548 #endif 2549 /* if this isn't the last cookie, go to the next one */ 2550 if ((cnt + 1) < sglinfo->si_max_pages) { 2551 cnt++; 2552 sgl[cnt].dmac_laddress = 0; 2553 sgl[cnt].dmac_size = 0; 2554 sgl[cnt].dmac_type = 0; 2555 } 2556 2557 /* 2558 * this page didn't need the copy buffer, if it's not physically 2559 * contiguous, or it would put us over a segment boundary, or it 2560 * puts us over the max cookie size, or the current sgl doesn't 2561 * have anything in it. 2562 */ 2563 } else if (((last_page + MMU_PAGESIZE) != paddr) || 2564 !(paddr & sglinfo->si_segmask) || 2565 ((sgl[cnt].dmac_size + psize) > maxseg) || 2566 (sgl[cnt].dmac_size == 0)) { 2567 /* 2568 * if we're not already in a new cookie, go to the next 2569 * cookie. 2570 */ 2571 if (sgl[cnt].dmac_size != 0) { 2572 cnt++; 2573 } 2574 2575 /* save the cookie information */ 2576 sgl[cnt].dmac_laddress = paddr; 2577 sgl[cnt].dmac_size = psize; 2578 #if defined(__amd64) 2579 sgl[cnt].dmac_type = 0; 2580 #else 2581 /* 2582 * save the buf offset for 32-bit kernel. used in the 2583 * obsoleted interfaces. 2584 */ 2585 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2586 #endif 2587 2588 /* 2589 * this page didn't need the copy buffer, it is physically 2590 * contiguous with the last page, and it's <= the max cookie 2591 * size. 2592 */ 2593 } else { 2594 sgl[cnt].dmac_size += psize; 2595 2596 /* 2597 * if this exactly == the maximum cookie size, and 2598 * it isn't the last cookie, go to the next cookie. 2599 */ 2600 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2601 ((cnt + 1) < sglinfo->si_max_pages)) { 2602 cnt++; 2603 sgl[cnt].dmac_laddress = 0; 2604 sgl[cnt].dmac_size = 0; 2605 sgl[cnt].dmac_type = 0; 2606 } 2607 } 2608 2609 /* 2610 * save this page's physical address so we can figure out if the 2611 * next page is physically contiguous. Keep decrementing size 2612 * until we are done with the buffer. 2613 */ 2614 last_page = paddr; 2615 size -= psize; 2616 } 2617 2618 /* we're done, save away how many cookies the sgl has */ 2619 if (sgl[cnt].dmac_size == 0) { 2620 ASSERT(cnt < sglinfo->si_max_pages); 2621 sglinfo->si_sgl_size = cnt; 2622 } else { 2623 sglinfo->si_sgl_size = cnt + 1; 2624 } 2625 } 2626 2627 2628 /* 2629 * rootnex_bind_slowpath() 2630 * Call in the bind path if the calling driver can't use the sgl without 2631 * modifying it. We either need to use the copy buffer and/or we will end up 2632 * with a partial bind. 2633 */ 2634 static int 2635 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2636 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2637 { 2638 rootnex_sglinfo_t *sinfo; 2639 rootnex_window_t *window; 2640 ddi_dma_cookie_t *cookie; 2641 size_t copybuf_used; 2642 size_t dmac_size; 2643 boolean_t partial; 2644 off_t cur_offset; 2645 page_t *cur_pp; 2646 major_t mnum; 2647 int e; 2648 int i; 2649 2650 2651 sinfo = &dma->dp_sglinfo; 2652 copybuf_used = 0; 2653 partial = B_FALSE; 2654 2655 /* 2656 * If we're using the copybuf, set the copybuf state in dma struct. 2657 * Needs to be first since it sets the copy buffer size. 2658 */ 2659 if (sinfo->si_copybuf_req != 0) { 2660 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2661 if (e != DDI_SUCCESS) { 2662 return (e); 2663 } 2664 } else { 2665 dma->dp_copybuf_size = 0; 2666 } 2667 2668 /* 2669 * Figure out if we need to do a partial mapping. If so, figure out 2670 * if we need to trim the buffers when we munge the sgl. 2671 */ 2672 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2673 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2674 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2675 dma->dp_partial_required = B_TRUE; 2676 if (attr->dma_attr_granular != 1) { 2677 dma->dp_trim_required = B_TRUE; 2678 } 2679 } else { 2680 dma->dp_partial_required = B_FALSE; 2681 dma->dp_trim_required = B_FALSE; 2682 } 2683 2684 /* If we need to do a partial bind, make sure the driver supports it */ 2685 if (dma->dp_partial_required && 2686 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2687 2688 mnum = ddi_driver_major(dma->dp_dip); 2689 /* 2690 * patchable which allows us to print one warning per major 2691 * number. 2692 */ 2693 if ((rootnex_bind_warn) && 2694 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2695 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2696 cmn_err(CE_WARN, "!%s: coding error detected, the " 2697 "driver is using ddi_dma_attr(9S) incorrectly. " 2698 "There is a small risk of data corruption in " 2699 "particular with large I/Os. The driver should be " 2700 "replaced with a corrected version for proper " 2701 "system operation. To disable this warning, add " 2702 "'set rootnex:rootnex_bind_warn=0' to " 2703 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2704 } 2705 return (DDI_DMA_TOOBIG); 2706 } 2707 2708 /* 2709 * we might need multiple windows, setup state to handle them. In this 2710 * code path, we will have at least one window. 2711 */ 2712 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2713 if (e != DDI_SUCCESS) { 2714 rootnex_teardown_copybuf(dma); 2715 return (e); 2716 } 2717 2718 window = &dma->dp_window[0]; 2719 cookie = &dma->dp_cookies[0]; 2720 cur_offset = 0; 2721 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2722 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2723 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2724 } 2725 2726 /* loop though all the cookies we got back from get_sgl() */ 2727 for (i = 0; i < sinfo->si_sgl_size; i++) { 2728 /* 2729 * If we're using the copy buffer, check this cookie and setup 2730 * its associated copy buffer state. If this cookie uses the 2731 * copy buffer, make sure we sync this window during dma_sync. 2732 */ 2733 if (dma->dp_copybuf_size > 0) { 2734 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2735 cur_offset, ©buf_used, &cur_pp); 2736 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2737 window->wd_dosync = B_TRUE; 2738 } 2739 } 2740 2741 /* 2742 * save away the cookie size, since it could be modified in 2743 * the windowing code. 2744 */ 2745 dmac_size = cookie->dmac_size; 2746 2747 /* if we went over max copybuf size */ 2748 if (dma->dp_copybuf_size && 2749 (copybuf_used > dma->dp_copybuf_size)) { 2750 partial = B_TRUE; 2751 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2752 cookie, cur_offset, ©buf_used); 2753 if (e != DDI_SUCCESS) { 2754 rootnex_teardown_copybuf(dma); 2755 rootnex_teardown_windows(dma); 2756 return (e); 2757 } 2758 2759 /* 2760 * if the coookie uses the copy buffer, make sure the 2761 * new window we just moved to is set to sync. 2762 */ 2763 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2764 window->wd_dosync = B_TRUE; 2765 } 2766 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2767 dma->dp_dip); 2768 2769 /* if the cookie cnt == max sgllen, move to the next window */ 2770 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2771 partial = B_TRUE; 2772 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2773 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2774 cookie, attr, cur_offset); 2775 if (e != DDI_SUCCESS) { 2776 rootnex_teardown_copybuf(dma); 2777 rootnex_teardown_windows(dma); 2778 return (e); 2779 } 2780 2781 /* 2782 * if the coookie uses the copy buffer, make sure the 2783 * new window we just moved to is set to sync. 2784 */ 2785 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2786 window->wd_dosync = B_TRUE; 2787 } 2788 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2789 dma->dp_dip); 2790 2791 /* else if we will be over maxxfer */ 2792 } else if ((window->wd_size + dmac_size) > 2793 dma->dp_maxxfer) { 2794 partial = B_TRUE; 2795 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2796 cookie); 2797 if (e != DDI_SUCCESS) { 2798 rootnex_teardown_copybuf(dma); 2799 rootnex_teardown_windows(dma); 2800 return (e); 2801 } 2802 2803 /* 2804 * if the coookie uses the copy buffer, make sure the 2805 * new window we just moved to is set to sync. 2806 */ 2807 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2808 window->wd_dosync = B_TRUE; 2809 } 2810 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2811 dma->dp_dip); 2812 2813 /* else this cookie fits in the current window */ 2814 } else { 2815 window->wd_cookie_cnt++; 2816 window->wd_size += dmac_size; 2817 } 2818 2819 /* track our offset into the buffer, go to the next cookie */ 2820 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2821 ASSERT(cookie->dmac_size <= dmac_size); 2822 cur_offset += dmac_size; 2823 cookie++; 2824 } 2825 2826 /* if we ended up with a zero sized window in the end, clean it up */ 2827 if (window->wd_size == 0) { 2828 hp->dmai_nwin--; 2829 window--; 2830 } 2831 2832 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2833 2834 if (!partial) { 2835 return (DDI_DMA_MAPPED); 2836 } 2837 2838 ASSERT(dma->dp_partial_required); 2839 return (DDI_DMA_PARTIAL_MAP); 2840 } 2841 2842 2843 /* 2844 * rootnex_setup_copybuf() 2845 * Called in bind slowpath. Figures out if we're going to use the copy 2846 * buffer, and if we do, sets up the basic state to handle it. 2847 */ 2848 static int 2849 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2850 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2851 { 2852 rootnex_sglinfo_t *sinfo; 2853 ddi_dma_attr_t lattr; 2854 size_t max_copybuf; 2855 int cansleep; 2856 int e; 2857 #if !defined(__amd64) 2858 int vmflag; 2859 #endif 2860 2861 2862 sinfo = &dma->dp_sglinfo; 2863 2864 /* 2865 * read this first so it's consistent through the routine so we can 2866 * patch it on the fly. 2867 */ 2868 max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK; 2869 2870 /* We need to call into the rootnex on ddi_dma_sync() */ 2871 hp->dmai_rflags &= ~DMP_NOSYNC; 2872 2873 /* make sure the copybuf size <= the max size */ 2874 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2875 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2876 2877 #if !defined(__amd64) 2878 /* 2879 * if we don't have kva space to copy to/from, allocate the KVA space 2880 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2881 * the 64-bit kernel. 2882 */ 2883 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2884 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2885 2886 /* convert the sleep flags */ 2887 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2888 vmflag = VM_SLEEP; 2889 } else { 2890 vmflag = VM_NOSLEEP; 2891 } 2892 2893 /* allocate Kernel VA space that we can bcopy to/from */ 2894 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2895 vmflag); 2896 if (dma->dp_kva == NULL) { 2897 return (DDI_DMA_NORESOURCES); 2898 } 2899 } 2900 #endif 2901 2902 /* convert the sleep flags */ 2903 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2904 cansleep = 1; 2905 } else { 2906 cansleep = 0; 2907 } 2908 2909 /* 2910 * Allocated the actual copy buffer. This needs to fit within the DMA 2911 * engines limits, so we can't use kmem_alloc... 2912 */ 2913 lattr = *attr; 2914 lattr.dma_attr_align = MMU_PAGESIZE; 2915 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2916 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2917 if (e != DDI_SUCCESS) { 2918 #if !defined(__amd64) 2919 if (dma->dp_kva != NULL) { 2920 vmem_free(heap_arena, dma->dp_kva, 2921 dma->dp_copybuf_size); 2922 } 2923 #endif 2924 return (DDI_DMA_NORESOURCES); 2925 } 2926 2927 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2928 size_t, dma->dp_copybuf_size); 2929 2930 return (DDI_SUCCESS); 2931 } 2932 2933 2934 /* 2935 * rootnex_setup_windows() 2936 * Called in bind slowpath to setup the window state. We always have windows 2937 * in the slowpath. Even if the window count = 1. 2938 */ 2939 static int 2940 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2941 ddi_dma_attr_t *attr, int kmflag) 2942 { 2943 rootnex_window_t *windowp; 2944 rootnex_sglinfo_t *sinfo; 2945 size_t copy_state_size; 2946 size_t win_state_size; 2947 size_t state_available; 2948 size_t space_needed; 2949 uint_t copybuf_win; 2950 uint_t maxxfer_win; 2951 size_t space_used; 2952 uint_t sglwin; 2953 2954 2955 sinfo = &dma->dp_sglinfo; 2956 2957 dma->dp_current_win = 0; 2958 hp->dmai_nwin = 0; 2959 2960 /* If we don't need to do a partial, we only have one window */ 2961 if (!dma->dp_partial_required) { 2962 dma->dp_max_win = 1; 2963 2964 /* 2965 * we need multiple windows, need to figure out the worse case number 2966 * of windows. 2967 */ 2968 } else { 2969 /* 2970 * if we need windows because we need more copy buffer that 2971 * we allow, the worse case number of windows we could need 2972 * here would be (copybuf space required / copybuf space that 2973 * we have) plus one for remainder, and plus 2 to handle the 2974 * extra pages on the trim for the first and last pages of the 2975 * buffer (a page is the minimum window size so under the right 2976 * attr settings, you could have a window for each page). 2977 * The last page will only be hit here if the size is not a 2978 * multiple of the granularity (which theoretically shouldn't 2979 * be the case but never has been enforced, so we could have 2980 * broken things without it). 2981 */ 2982 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2983 ASSERT(dma->dp_copybuf_size > 0); 2984 copybuf_win = (sinfo->si_copybuf_req / 2985 dma->dp_copybuf_size) + 1 + 2; 2986 } else { 2987 copybuf_win = 0; 2988 } 2989 2990 /* 2991 * if we need windows because we have more cookies than the H/W 2992 * can handle, the number of windows we would need here would 2993 * be (cookie count / cookies count H/W supports) plus one for 2994 * remainder, and plus 2 to handle the extra pages on the trim 2995 * (see above comment about trim) 2996 */ 2997 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2998 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2999 + 1) + 2; 3000 } else { 3001 sglwin = 0; 3002 } 3003 3004 /* 3005 * if we need windows because we're binding more memory than the 3006 * H/W can transfer at once, the number of windows we would need 3007 * here would be (xfer count / max xfer H/W supports) plus one 3008 * for remainder, and plus 2 to handle the extra pages on the 3009 * trim (see above comment about trim) 3010 */ 3011 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3012 maxxfer_win = (dma->dp_dma.dmao_size / 3013 dma->dp_maxxfer) + 1 + 2; 3014 } else { 3015 maxxfer_win = 0; 3016 } 3017 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3018 ASSERT(dma->dp_max_win > 0); 3019 } 3020 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3021 3022 /* 3023 * Get space for window and potential copy buffer state. Before we 3024 * go and allocate memory, see if we can get away with using what's 3025 * left in the pre-allocted state or the dynamically allocated sgl. 3026 */ 3027 space_used = (uintptr_t)(sinfo->si_sgl_size * 3028 sizeof (ddi_dma_cookie_t)); 3029 3030 /* if we dynamically allocated space for the cookies */ 3031 if (dma->dp_need_to_free_cookie) { 3032 /* if we have more space in the pre-allocted buffer, use it */ 3033 ASSERT(space_used <= dma->dp_cookie_size); 3034 if ((dma->dp_cookie_size - space_used) <= 3035 rootnex_state->r_prealloc_size) { 3036 state_available = rootnex_state->r_prealloc_size; 3037 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3038 3039 /* 3040 * else, we have more free space in the dynamically allocated 3041 * buffer, i.e. the buffer wasn't worse case fragmented so we 3042 * didn't need a lot of cookies. 3043 */ 3044 } else { 3045 state_available = dma->dp_cookie_size - space_used; 3046 windowp = (rootnex_window_t *) 3047 &dma->dp_cookies[sinfo->si_sgl_size]; 3048 } 3049 3050 /* we used the pre-alloced buffer */ 3051 } else { 3052 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3053 state_available = rootnex_state->r_prealloc_size - space_used; 3054 windowp = (rootnex_window_t *) 3055 &dma->dp_cookies[sinfo->si_sgl_size]; 3056 } 3057 3058 /* 3059 * figure out how much state we need to track the copy buffer. Add an 3060 * addition 8 bytes for pointer alignemnt later. 3061 */ 3062 if (dma->dp_copybuf_size > 0) { 3063 copy_state_size = sinfo->si_max_pages * 3064 sizeof (rootnex_pgmap_t); 3065 } else { 3066 copy_state_size = 0; 3067 } 3068 /* add an additional 8 bytes for pointer alignment */ 3069 space_needed = win_state_size + copy_state_size + 0x8; 3070 3071 /* if we have enough space already, use it */ 3072 if (state_available >= space_needed) { 3073 dma->dp_window = windowp; 3074 dma->dp_need_to_free_window = B_FALSE; 3075 3076 /* not enough space, need to allocate more. */ 3077 } else { 3078 dma->dp_window = kmem_alloc(space_needed, kmflag); 3079 if (dma->dp_window == NULL) { 3080 return (DDI_DMA_NORESOURCES); 3081 } 3082 dma->dp_need_to_free_window = B_TRUE; 3083 dma->dp_window_size = space_needed; 3084 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3085 dma->dp_dip, size_t, space_needed); 3086 } 3087 3088 /* 3089 * we allocate copy buffer state and window state at the same time. 3090 * setup our copy buffer state pointers. Make sure it's aligned. 3091 */ 3092 if (dma->dp_copybuf_size > 0) { 3093 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3094 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3095 3096 #if !defined(__amd64) 3097 /* 3098 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3099 * false/NULL. Should be quicker to bzero vs loop and set. 3100 */ 3101 bzero(dma->dp_pgmap, copy_state_size); 3102 #endif 3103 } else { 3104 dma->dp_pgmap = NULL; 3105 } 3106 3107 return (DDI_SUCCESS); 3108 } 3109 3110 3111 /* 3112 * rootnex_teardown_copybuf() 3113 * cleans up after rootnex_setup_copybuf() 3114 */ 3115 static void 3116 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3117 { 3118 #if !defined(__amd64) 3119 int i; 3120 3121 /* 3122 * if we allocated kernel heap VMEM space, go through all the pages and 3123 * map out any of the ones that we're mapped into the kernel heap VMEM 3124 * arena. Then free the VMEM space. 3125 */ 3126 if (dma->dp_kva != NULL) { 3127 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3128 if (dma->dp_pgmap[i].pm_mapped) { 3129 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3130 MMU_PAGESIZE, HAT_UNLOAD); 3131 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3132 } 3133 } 3134 3135 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3136 } 3137 3138 #endif 3139 3140 /* if we allocated a copy buffer, free it */ 3141 if (dma->dp_cbaddr != NULL) { 3142 i_ddi_mem_free(dma->dp_cbaddr, 0); 3143 } 3144 } 3145 3146 3147 /* 3148 * rootnex_teardown_windows() 3149 * cleans up after rootnex_setup_windows() 3150 */ 3151 static void 3152 rootnex_teardown_windows(rootnex_dma_t *dma) 3153 { 3154 /* 3155 * if we had to allocate window state on the last bind (because we 3156 * didn't have enough pre-allocated space in the handle), free it. 3157 */ 3158 if (dma->dp_need_to_free_window) { 3159 kmem_free(dma->dp_window, dma->dp_window_size); 3160 } 3161 } 3162 3163 3164 /* 3165 * rootnex_init_win() 3166 * Called in bind slow path during creation of a new window. Initializes 3167 * window state to default values. 3168 */ 3169 /*ARGSUSED*/ 3170 static void 3171 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3172 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3173 { 3174 hp->dmai_nwin++; 3175 window->wd_dosync = B_FALSE; 3176 window->wd_offset = cur_offset; 3177 window->wd_size = 0; 3178 window->wd_first_cookie = cookie; 3179 window->wd_cookie_cnt = 0; 3180 window->wd_trim.tr_trim_first = B_FALSE; 3181 window->wd_trim.tr_trim_last = B_FALSE; 3182 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3183 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3184 #if !defined(__amd64) 3185 window->wd_remap_copybuf = dma->dp_cb_remaping; 3186 #endif 3187 } 3188 3189 3190 /* 3191 * rootnex_setup_cookie() 3192 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3193 * the sgl uses the copy buffer, we need to go through each cookie, figure 3194 * out if it uses the copy buffer, and if it does, save away everything we'll 3195 * need during sync. 3196 */ 3197 static void 3198 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3199 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3200 page_t **cur_pp) 3201 { 3202 boolean_t copybuf_sz_power_2; 3203 rootnex_sglinfo_t *sinfo; 3204 uint_t pidx; 3205 uint_t pcnt; 3206 off_t poff; 3207 #if defined(__amd64) 3208 pfn_t pfn; 3209 #else 3210 page_t **pplist; 3211 #endif 3212 3213 sinfo = &dma->dp_sglinfo; 3214 3215 /* 3216 * Calculate the page index relative to the start of the buffer. The 3217 * index to the current page for our buffer is the offset into the 3218 * first page of the buffer plus our current offset into the buffer 3219 * itself, shifted of course... 3220 */ 3221 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3222 ASSERT(pidx < sinfo->si_max_pages); 3223 3224 /* if this cookie uses the copy buffer */ 3225 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3226 /* 3227 * NOTE: we know that since this cookie uses the copy buffer, it 3228 * is <= MMU_PAGESIZE. 3229 */ 3230 3231 /* 3232 * get the offset into the page. For the 64-bit kernel, get the 3233 * pfn which we'll use with seg kpm. 3234 */ 3235 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3236 #if defined(__amd64) 3237 pfn = cookie->_dmu._dmac_ll >> MMU_PAGESHIFT; 3238 #endif 3239 3240 /* figure out if the copybuf size is a power of 2 */ 3241 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3242 copybuf_sz_power_2 = B_FALSE; 3243 } else { 3244 copybuf_sz_power_2 = B_TRUE; 3245 } 3246 3247 /* This page uses the copy buffer */ 3248 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3249 3250 /* 3251 * save the copy buffer KVA that we'll use with this page. 3252 * if we still fit within the copybuf, it's a simple add. 3253 * otherwise, we need to wrap over using & or % accordingly. 3254 */ 3255 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3256 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3257 *copybuf_used; 3258 } else { 3259 if (copybuf_sz_power_2) { 3260 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3261 (uintptr_t)dma->dp_cbaddr + 3262 (*copybuf_used & 3263 (dma->dp_copybuf_size - 1))); 3264 } else { 3265 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3266 (uintptr_t)dma->dp_cbaddr + 3267 (*copybuf_used % dma->dp_copybuf_size)); 3268 } 3269 } 3270 3271 /* 3272 * over write the cookie physical address with the address of 3273 * the physical address of the copy buffer page that we will 3274 * use. 3275 */ 3276 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3277 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3278 3279 /* if we have a kernel VA, it's easy, just save that address */ 3280 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3281 (sinfo->si_asp == &kas)) { 3282 /* 3283 * save away the page aligned virtual address of the 3284 * driver buffer. Offsets are handled in the sync code. 3285 */ 3286 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3287 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3288 & MMU_PAGEMASK); 3289 #if !defined(__amd64) 3290 /* 3291 * we didn't need to, and will never need to map this 3292 * page. 3293 */ 3294 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3295 #endif 3296 3297 /* we don't have a kernel VA. We need one for the bcopy. */ 3298 } else { 3299 #if defined(__amd64) 3300 /* 3301 * for the 64-bit kernel, it's easy. We use seg kpm to 3302 * get a Kernel VA for the corresponding pfn. 3303 */ 3304 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3305 #else 3306 /* 3307 * for the 32-bit kernel, this is a pain. First we'll 3308 * save away the page_t or user VA for this page. This 3309 * is needed in rootnex_dma_win() when we switch to a 3310 * new window which requires us to re-map the copy 3311 * buffer. 3312 */ 3313 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3314 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3315 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3316 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3317 } else if (pplist != NULL) { 3318 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3319 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3320 } else { 3321 dma->dp_pgmap[pidx].pm_pp = NULL; 3322 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3323 (((uintptr_t) 3324 dmar_object->dmao_obj.virt_obj.v_addr + 3325 cur_offset) & MMU_PAGEMASK); 3326 } 3327 3328 /* 3329 * save away the page aligned virtual address which was 3330 * allocated from the kernel heap arena (taking into 3331 * account if we need more copy buffer than we alloced 3332 * and use multiple windows to handle this, i.e. &,%). 3333 * NOTE: there isn't and physical memory backing up this 3334 * virtual address space currently. 3335 */ 3336 if ((*copybuf_used + MMU_PAGESIZE) <= 3337 dma->dp_copybuf_size) { 3338 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3339 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3340 MMU_PAGEMASK); 3341 } else { 3342 if (copybuf_sz_power_2) { 3343 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3344 (((uintptr_t)dma->dp_kva + 3345 (*copybuf_used & 3346 (dma->dp_copybuf_size - 1))) & 3347 MMU_PAGEMASK); 3348 } else { 3349 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3350 (((uintptr_t)dma->dp_kva + 3351 (*copybuf_used % 3352 dma->dp_copybuf_size)) & 3353 MMU_PAGEMASK); 3354 } 3355 } 3356 3357 /* 3358 * if we haven't used up the available copy buffer yet, 3359 * map the kva to the physical page. 3360 */ 3361 if (!dma->dp_cb_remaping && ((*copybuf_used + 3362 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3363 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3364 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3365 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3366 dma->dp_pgmap[pidx].pm_kaddr); 3367 } else { 3368 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3369 sinfo->si_asp, 3370 dma->dp_pgmap[pidx].pm_kaddr); 3371 } 3372 3373 /* 3374 * we've used up the available copy buffer, this page 3375 * will have to be mapped during rootnex_dma_win() when 3376 * we switch to a new window which requires a re-map 3377 * the copy buffer. (32-bit kernel only) 3378 */ 3379 } else { 3380 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3381 } 3382 #endif 3383 /* go to the next page_t */ 3384 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3385 *cur_pp = (*cur_pp)->p_next; 3386 } 3387 } 3388 3389 /* add to the copy buffer count */ 3390 *copybuf_used += MMU_PAGESIZE; 3391 3392 /* 3393 * This cookie doesn't use the copy buffer. Walk through the pages this 3394 * cookie occupies to reflect this. 3395 */ 3396 } else { 3397 /* 3398 * figure out how many pages the cookie occupies. We need to 3399 * use the original page offset of the buffer and the cookies 3400 * offset in the buffer to do this. 3401 */ 3402 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3403 pcnt = mmu_btopr(cookie->dmac_size + poff); 3404 3405 while (pcnt > 0) { 3406 #if !defined(__amd64) 3407 /* 3408 * the 32-bit kernel doesn't have seg kpm, so we need 3409 * to map in the driver buffer (if it didn't come down 3410 * with a kernel VA) on the fly. Since this page doesn't 3411 * use the copy buffer, it's not, or will it ever, have 3412 * to be mapped in. 3413 */ 3414 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3415 #endif 3416 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3417 3418 /* 3419 * we need to update pidx and cur_pp or we'll loose 3420 * track of where we are. 3421 */ 3422 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3423 *cur_pp = (*cur_pp)->p_next; 3424 } 3425 pidx++; 3426 pcnt--; 3427 } 3428 } 3429 } 3430 3431 3432 /* 3433 * rootnex_sgllen_window_boundary() 3434 * Called in the bind slow path when the next cookie causes us to exceed (in 3435 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3436 * length supported by the DMA H/W. 3437 */ 3438 static int 3439 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3440 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3441 off_t cur_offset) 3442 { 3443 off_t new_offset; 3444 size_t trim_sz; 3445 off_t coffset; 3446 3447 3448 /* 3449 * if we know we'll never have to trim, it's pretty easy. Just move to 3450 * the next window and init it. We're done. 3451 */ 3452 if (!dma->dp_trim_required) { 3453 (*windowp)++; 3454 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3455 (*windowp)->wd_cookie_cnt++; 3456 (*windowp)->wd_size = cookie->dmac_size; 3457 return (DDI_SUCCESS); 3458 } 3459 3460 /* figure out how much we need to trim from the window */ 3461 ASSERT(attr->dma_attr_granular != 0); 3462 if (dma->dp_granularity_power_2) { 3463 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3464 } else { 3465 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3466 } 3467 3468 /* The window's a whole multiple of granularity. We're done */ 3469 if (trim_sz == 0) { 3470 (*windowp)++; 3471 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3472 (*windowp)->wd_cookie_cnt++; 3473 (*windowp)->wd_size = cookie->dmac_size; 3474 return (DDI_SUCCESS); 3475 } 3476 3477 /* 3478 * The window's not a whole multiple of granularity, since we know this 3479 * is due to the sgllen, we need to go back to the last cookie and trim 3480 * that one, add the left over part of the old cookie into the new 3481 * window, and then add in the new cookie into the new window. 3482 */ 3483 3484 /* 3485 * make sure the driver isn't making us do something bad... Trimming and 3486 * sgllen == 1 don't go together. 3487 */ 3488 if (attr->dma_attr_sgllen == 1) { 3489 return (DDI_DMA_NOMAPPING); 3490 } 3491 3492 /* 3493 * first, setup the current window to account for the trim. Need to go 3494 * back to the last cookie for this. 3495 */ 3496 cookie--; 3497 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3498 (*windowp)->wd_trim.tr_last_cookie = cookie; 3499 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3500 ASSERT(cookie->dmac_size > trim_sz); 3501 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3502 (*windowp)->wd_size -= trim_sz; 3503 3504 /* save the buffer offsets for the next window */ 3505 coffset = cookie->dmac_size - trim_sz; 3506 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3507 3508 /* 3509 * set this now in case this is the first window. all other cases are 3510 * set in dma_win() 3511 */ 3512 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3513 3514 /* 3515 * initialize the next window using what's left over in the previous 3516 * cookie. 3517 */ 3518 (*windowp)++; 3519 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3520 (*windowp)->wd_cookie_cnt++; 3521 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3522 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3523 (*windowp)->wd_trim.tr_first_size = trim_sz; 3524 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3525 (*windowp)->wd_dosync = B_TRUE; 3526 } 3527 3528 /* 3529 * now go back to the current cookie and add it to the new window. set 3530 * the new window size to the what was left over from the previous 3531 * cookie and what's in the current cookie. 3532 */ 3533 cookie++; 3534 (*windowp)->wd_cookie_cnt++; 3535 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3536 3537 /* 3538 * trim plus the next cookie could put us over maxxfer (a cookie can be 3539 * a max size of maxxfer). Handle that case. 3540 */ 3541 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3542 /* 3543 * maxxfer is already a whole multiple of granularity, and this 3544 * trim will be <= the previous trim (since a cookie can't be 3545 * larger than maxxfer). Make things simple here. 3546 */ 3547 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3548 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3549 (*windowp)->wd_trim.tr_last_cookie = cookie; 3550 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3551 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3552 (*windowp)->wd_size -= trim_sz; 3553 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3554 3555 /* save the buffer offsets for the next window */ 3556 coffset = cookie->dmac_size - trim_sz; 3557 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3558 3559 /* setup the next window */ 3560 (*windowp)++; 3561 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3562 (*windowp)->wd_cookie_cnt++; 3563 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3564 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3565 coffset; 3566 (*windowp)->wd_trim.tr_first_size = trim_sz; 3567 } 3568 3569 return (DDI_SUCCESS); 3570 } 3571 3572 3573 /* 3574 * rootnex_copybuf_window_boundary() 3575 * Called in bind slowpath when we get to a window boundary because we used 3576 * up all the copy buffer that we have. 3577 */ 3578 static int 3579 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3580 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3581 size_t *copybuf_used) 3582 { 3583 rootnex_sglinfo_t *sinfo; 3584 off_t new_offset; 3585 size_t trim_sz; 3586 off_t coffset; 3587 uint_t pidx; 3588 off_t poff; 3589 3590 3591 sinfo = &dma->dp_sglinfo; 3592 3593 /* 3594 * the copy buffer should be a whole multiple of page size. We know that 3595 * this cookie is <= MMU_PAGESIZE. 3596 */ 3597 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3598 3599 /* 3600 * from now on, all new windows in this bind need to be re-mapped during 3601 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3602 * space... 3603 */ 3604 #if !defined(__amd64) 3605 dma->dp_cb_remaping = B_TRUE; 3606 #endif 3607 3608 /* reset copybuf used */ 3609 *copybuf_used = 0; 3610 3611 /* 3612 * if we don't have to trim (since granularity is set to 1), go to the 3613 * next window and add the current cookie to it. We know the current 3614 * cookie uses the copy buffer since we're in this code path. 3615 */ 3616 if (!dma->dp_trim_required) { 3617 (*windowp)++; 3618 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3619 3620 /* Add this cookie to the new window */ 3621 (*windowp)->wd_cookie_cnt++; 3622 (*windowp)->wd_size += cookie->dmac_size; 3623 *copybuf_used += MMU_PAGESIZE; 3624 return (DDI_SUCCESS); 3625 } 3626 3627 /* 3628 * *** may need to trim, figure it out. 3629 */ 3630 3631 /* figure out how much we need to trim from the window */ 3632 if (dma->dp_granularity_power_2) { 3633 trim_sz = (*windowp)->wd_size & 3634 (hp->dmai_attr.dma_attr_granular - 1); 3635 } else { 3636 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3637 } 3638 3639 /* 3640 * if the window's a whole multiple of granularity, go to the next 3641 * window, init it, then add in the current cookie. We know the current 3642 * cookie uses the copy buffer since we're in this code path. 3643 */ 3644 if (trim_sz == 0) { 3645 (*windowp)++; 3646 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3647 3648 /* Add this cookie to the new window */ 3649 (*windowp)->wd_cookie_cnt++; 3650 (*windowp)->wd_size += cookie->dmac_size; 3651 *copybuf_used += MMU_PAGESIZE; 3652 return (DDI_SUCCESS); 3653 } 3654 3655 /* 3656 * *** We figured it out, we definitly need to trim 3657 */ 3658 3659 /* 3660 * make sure the driver isn't making us do something bad... 3661 * Trimming and sgllen == 1 don't go together. 3662 */ 3663 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3664 return (DDI_DMA_NOMAPPING); 3665 } 3666 3667 /* 3668 * first, setup the current window to account for the trim. Need to go 3669 * back to the last cookie for this. Some of the last cookie will be in 3670 * the current window, and some of the last cookie will be in the new 3671 * window. All of the current cookie will be in the new window. 3672 */ 3673 cookie--; 3674 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3675 (*windowp)->wd_trim.tr_last_cookie = cookie; 3676 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3677 ASSERT(cookie->dmac_size > trim_sz); 3678 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3679 (*windowp)->wd_size -= trim_sz; 3680 3681 /* 3682 * we're trimming the last cookie (not the current cookie). So that 3683 * last cookie may have or may not have been using the copy buffer ( 3684 * we know the cookie passed in uses the copy buffer since we're in 3685 * this code path). 3686 * 3687 * If the last cookie doesn't use the copy buffer, nothing special to 3688 * do. However, if it does uses the copy buffer, it will be both the 3689 * last page in the current window and the first page in the next 3690 * window. Since we are reusing the copy buffer (and KVA space on the 3691 * 32-bit kernel), this page will use the end of the copy buffer in the 3692 * current window, and the start of the copy buffer in the next window. 3693 * Track that info... The cookie physical address was already set to 3694 * the copy buffer physical address in setup_cookie.. 3695 */ 3696 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3697 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3698 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3699 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3700 (*windowp)->wd_trim.tr_last_pidx = pidx; 3701 (*windowp)->wd_trim.tr_last_cbaddr = 3702 dma->dp_pgmap[pidx].pm_cbaddr; 3703 #if !defined(__amd64) 3704 (*windowp)->wd_trim.tr_last_kaddr = 3705 dma->dp_pgmap[pidx].pm_kaddr; 3706 #endif 3707 } 3708 3709 /* save the buffer offsets for the next window */ 3710 coffset = cookie->dmac_size - trim_sz; 3711 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3712 3713 /* 3714 * set this now in case this is the first window. all other cases are 3715 * set in dma_win() 3716 */ 3717 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3718 3719 /* 3720 * initialize the next window using what's left over in the previous 3721 * cookie. 3722 */ 3723 (*windowp)++; 3724 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3725 (*windowp)->wd_cookie_cnt++; 3726 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3727 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + coffset; 3728 (*windowp)->wd_trim.tr_first_size = trim_sz; 3729 3730 /* 3731 * again, we're tracking if the last cookie uses the copy buffer. 3732 * read the comment above for more info on why we need to track 3733 * additional state. 3734 * 3735 * For the first cookie in the new window, we need reset the physical 3736 * address to DMA into to the start of the copy buffer plus any 3737 * initial page offset which may be present. 3738 */ 3739 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3740 (*windowp)->wd_dosync = B_TRUE; 3741 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3742 (*windowp)->wd_trim.tr_first_pidx = pidx; 3743 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3744 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3745 (*windowp)->wd_trim.tr_first_paddr = ptob64(hat_getpfnum( 3746 kas.a_hat, dma->dp_cbaddr)) + poff; 3747 #if !defined(__amd64) 3748 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3749 #endif 3750 /* account for the cookie copybuf usage in the new window */ 3751 *copybuf_used += MMU_PAGESIZE; 3752 3753 /* 3754 * every piece of code has to have a hack, and here is this 3755 * ones :-) 3756 * 3757 * There is a complex interaction between setup_cookie and the 3758 * copybuf window boundary. The complexity had to be in either 3759 * the maxxfer window, or the copybuf window, and I chose the 3760 * copybuf code. 3761 * 3762 * So in this code path, we have taken the last cookie, 3763 * virtually broken it in half due to the trim, and it happens 3764 * to use the copybuf which further complicates life. At the 3765 * same time, we have already setup the current cookie, which 3766 * is now wrong. More background info: the current cookie uses 3767 * the copybuf, so it is only a page long max. So we need to 3768 * fix the current cookies copy buffer address, physical 3769 * address, and kva for the 32-bit kernel. We due this by 3770 * bumping them by page size (of course, we can't due this on 3771 * the physical address since the copy buffer may not be 3772 * physically contiguous). 3773 */ 3774 cookie++; 3775 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3776 poff = cookie->_dmu._dmac_ll & MMU_PAGEOFFSET; 3777 cookie->_dmu._dmac_ll = ptob64(hat_getpfnum(kas.a_hat, 3778 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3779 #if !defined(__amd64) 3780 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3781 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3782 #endif 3783 } else { 3784 /* go back to the current cookie */ 3785 cookie++; 3786 } 3787 3788 /* 3789 * add the current cookie to the new window. set the new window size to 3790 * the what was left over from the previous cookie and what's in the 3791 * current cookie. 3792 */ 3793 (*windowp)->wd_cookie_cnt++; 3794 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3795 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3796 3797 /* 3798 * we know that the cookie passed in always uses the copy buffer. We 3799 * wouldn't be here if it didn't. 3800 */ 3801 *copybuf_used += MMU_PAGESIZE; 3802 3803 return (DDI_SUCCESS); 3804 } 3805 3806 3807 /* 3808 * rootnex_maxxfer_window_boundary() 3809 * Called in bind slowpath when we get to a window boundary because we will 3810 * go over maxxfer. 3811 */ 3812 static int 3813 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3814 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3815 { 3816 size_t dmac_size; 3817 off_t new_offset; 3818 size_t trim_sz; 3819 off_t coffset; 3820 3821 3822 /* 3823 * calculate how much we have to trim off of the current cookie to equal 3824 * maxxfer. We don't have to account for granularity here since our 3825 * maxxfer already takes that into account. 3826 */ 3827 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3828 ASSERT(trim_sz <= cookie->dmac_size); 3829 ASSERT(trim_sz <= dma->dp_maxxfer); 3830 3831 /* save cookie size since we need it later and we might change it */ 3832 dmac_size = cookie->dmac_size; 3833 3834 /* 3835 * if we're not trimming the entire cookie, setup the current window to 3836 * account for the trim. 3837 */ 3838 if (trim_sz < cookie->dmac_size) { 3839 (*windowp)->wd_cookie_cnt++; 3840 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3841 (*windowp)->wd_trim.tr_last_cookie = cookie; 3842 (*windowp)->wd_trim.tr_last_paddr = cookie->_dmu._dmac_ll; 3843 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3844 (*windowp)->wd_size = dma->dp_maxxfer; 3845 3846 /* 3847 * set the adjusted cookie size now in case this is the first 3848 * window. All other windows are taken care of in get win 3849 */ 3850 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3851 } 3852 3853 /* 3854 * coffset is the current offset within the cookie, new_offset is the 3855 * current offset with the entire buffer. 3856 */ 3857 coffset = dmac_size - trim_sz; 3858 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3859 3860 /* initialize the next window */ 3861 (*windowp)++; 3862 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3863 (*windowp)->wd_cookie_cnt++; 3864 (*windowp)->wd_size = trim_sz; 3865 if (trim_sz < dmac_size) { 3866 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3867 (*windowp)->wd_trim.tr_first_paddr = cookie->_dmu._dmac_ll + 3868 coffset; 3869 (*windowp)->wd_trim.tr_first_size = trim_sz; 3870 } 3871 3872 return (DDI_SUCCESS); 3873 } 3874 3875 3876 /* 3877 * rootnex_dma_sync() 3878 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3879 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3880 * is set, ddi_dma_sync() returns immediately passing back success. 3881 */ 3882 /*ARGSUSED*/ 3883 static int 3884 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3885 off_t off, size_t len, uint_t cache_flags) 3886 { 3887 rootnex_sglinfo_t *sinfo; 3888 rootnex_pgmap_t *cbpage; 3889 rootnex_window_t *win; 3890 ddi_dma_impl_t *hp; 3891 rootnex_dma_t *dma; 3892 caddr_t fromaddr; 3893 caddr_t toaddr; 3894 uint_t psize; 3895 off_t offset; 3896 uint_t pidx; 3897 size_t size; 3898 off_t poff; 3899 int e; 3900 3901 3902 hp = (ddi_dma_impl_t *)handle; 3903 dma = (rootnex_dma_t *)hp->dmai_private; 3904 sinfo = &dma->dp_sglinfo; 3905 3906 /* 3907 * if we don't have any windows, we don't need to sync. A copybuf 3908 * will cause us to have at least one window. 3909 */ 3910 if (dma->dp_window == NULL) { 3911 return (DDI_SUCCESS); 3912 } 3913 3914 /* This window may not need to be sync'd */ 3915 win = &dma->dp_window[dma->dp_current_win]; 3916 if (!win->wd_dosync) { 3917 return (DDI_SUCCESS); 3918 } 3919 3920 /* handle off and len special cases */ 3921 if ((off == 0) || (rootnex_sync_ignore_params)) { 3922 offset = win->wd_offset; 3923 } else { 3924 offset = off; 3925 } 3926 if ((len == 0) || (rootnex_sync_ignore_params)) { 3927 size = win->wd_size; 3928 } else { 3929 size = len; 3930 } 3931 3932 /* check the sync args to make sure they make a little sense */ 3933 if (rootnex_sync_check_parms) { 3934 e = rootnex_valid_sync_parms(hp, win, offset, size, 3935 cache_flags); 3936 if (e != DDI_SUCCESS) { 3937 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3938 return (DDI_FAILURE); 3939 } 3940 } 3941 3942 /* 3943 * special case the first page to handle the offset into the page. The 3944 * offset to the current page for our buffer is the offset into the 3945 * first page of the buffer plus our current offset into the buffer 3946 * itself, masked of course. 3947 */ 3948 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3949 psize = MIN((MMU_PAGESIZE - poff), size); 3950 3951 /* go through all the pages that we want to sync */ 3952 while (size > 0) { 3953 /* 3954 * Calculate the page index relative to the start of the buffer. 3955 * The index to the current page for our buffer is the offset 3956 * into the first page of the buffer plus our current offset 3957 * into the buffer itself, shifted of course... 3958 */ 3959 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3960 ASSERT(pidx < sinfo->si_max_pages); 3961 3962 /* 3963 * if this page uses the copy buffer, we need to sync it, 3964 * otherwise, go on to the next page. 3965 */ 3966 cbpage = &dma->dp_pgmap[pidx]; 3967 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3968 (cbpage->pm_uses_copybuf == B_FALSE)); 3969 if (cbpage->pm_uses_copybuf) { 3970 /* cbaddr and kaddr should be page aligned */ 3971 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3972 MMU_PAGEOFFSET) == 0); 3973 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3974 MMU_PAGEOFFSET) == 0); 3975 3976 /* 3977 * if we're copying for the device, we are going to 3978 * copy from the drivers buffer and to the rootnex 3979 * allocated copy buffer. 3980 */ 3981 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3982 fromaddr = cbpage->pm_kaddr + poff; 3983 toaddr = cbpage->pm_cbaddr + poff; 3984 DTRACE_PROBE2(rootnex__sync__dev, 3985 dev_info_t *, dma->dp_dip, size_t, psize); 3986 3987 /* 3988 * if we're copying for the cpu/kernel, we are going to 3989 * copy from the rootnex allocated copy buffer to the 3990 * drivers buffer. 3991 */ 3992 } else { 3993 fromaddr = cbpage->pm_cbaddr + poff; 3994 toaddr = cbpage->pm_kaddr + poff; 3995 DTRACE_PROBE2(rootnex__sync__cpu, 3996 dev_info_t *, dma->dp_dip, size_t, psize); 3997 } 3998 3999 bcopy(fromaddr, toaddr, psize); 4000 } 4001 4002 /* 4003 * decrement size until we're done, update our offset into the 4004 * buffer, and get the next page size. 4005 */ 4006 size -= psize; 4007 offset += psize; 4008 psize = MIN(MMU_PAGESIZE, size); 4009 4010 /* page offset is zero for the rest of this loop */ 4011 poff = 0; 4012 } 4013 4014 return (DDI_SUCCESS); 4015 } 4016 4017 4018 /* 4019 * rootnex_valid_sync_parms() 4020 * checks the parameters passed to sync to verify they are correct. 4021 */ 4022 static int 4023 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4024 off_t offset, size_t size, uint_t cache_flags) 4025 { 4026 off_t woffset; 4027 4028 4029 /* 4030 * the first part of the test to make sure the offset passed in is 4031 * within the window. 4032 */ 4033 if (offset < win->wd_offset) { 4034 return (DDI_FAILURE); 4035 } 4036 4037 /* 4038 * second and last part of the test to make sure the offset and length 4039 * passed in is within the window. 4040 */ 4041 woffset = offset - win->wd_offset; 4042 if ((woffset + size) > win->wd_size) { 4043 return (DDI_FAILURE); 4044 } 4045 4046 /* 4047 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4048 * be set too. 4049 */ 4050 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4051 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4052 return (DDI_SUCCESS); 4053 } 4054 4055 /* 4056 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4057 * should be set. Also DDI_DMA_READ should be set in the flags. 4058 */ 4059 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4060 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4061 (hp->dmai_rflags & DDI_DMA_READ)) { 4062 return (DDI_SUCCESS); 4063 } 4064 4065 return (DDI_FAILURE); 4066 } 4067 4068 4069 /* 4070 * rootnex_dma_win() 4071 * called from ddi_dma_getwin() 4072 */ 4073 /*ARGSUSED*/ 4074 static int 4075 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4076 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4077 uint_t *ccountp) 4078 { 4079 rootnex_window_t *window; 4080 rootnex_trim_t *trim; 4081 ddi_dma_impl_t *hp; 4082 rootnex_dma_t *dma; 4083 #if !defined(__amd64) 4084 rootnex_sglinfo_t *sinfo; 4085 rootnex_pgmap_t *pmap; 4086 uint_t pidx; 4087 uint_t pcnt; 4088 off_t poff; 4089 int i; 4090 #endif 4091 4092 4093 hp = (ddi_dma_impl_t *)handle; 4094 dma = (rootnex_dma_t *)hp->dmai_private; 4095 #if !defined(__amd64) 4096 sinfo = &dma->dp_sglinfo; 4097 #endif 4098 4099 /* If we try and get a window which doesn't exist, return failure */ 4100 if (win >= hp->dmai_nwin) { 4101 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4102 return (DDI_FAILURE); 4103 } 4104 4105 /* 4106 * if we don't have any windows, and they're asking for the first 4107 * window, setup the cookie pointer to the first cookie in the bind. 4108 * setup our return values, then increment the cookie since we return 4109 * the first cookie on the stack. 4110 */ 4111 if (dma->dp_window == NULL) { 4112 if (win != 0) { 4113 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4114 return (DDI_FAILURE); 4115 } 4116 hp->dmai_cookie = dma->dp_cookies; 4117 *offp = 0; 4118 *lenp = dma->dp_dma.dmao_size; 4119 *ccountp = dma->dp_sglinfo.si_sgl_size; 4120 *cookiep = hp->dmai_cookie[0]; 4121 hp->dmai_cookie++; 4122 return (DDI_SUCCESS); 4123 } 4124 4125 /* sync the old window before moving on to the new one */ 4126 window = &dma->dp_window[dma->dp_current_win]; 4127 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4128 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4129 DDI_DMA_SYNC_FORCPU); 4130 } 4131 4132 #if !defined(__amd64) 4133 /* 4134 * before we move to the next window, if we need to re-map, unmap all 4135 * the pages in this window. 4136 */ 4137 if (dma->dp_cb_remaping) { 4138 /* 4139 * If we switch to this window again, we'll need to map in 4140 * on the fly next time. 4141 */ 4142 window->wd_remap_copybuf = B_TRUE; 4143 4144 /* 4145 * calculate the page index into the buffer where this window 4146 * starts, and the number of pages this window takes up. 4147 */ 4148 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4149 MMU_PAGESHIFT; 4150 poff = (sinfo->si_buf_offset + window->wd_offset) & 4151 MMU_PAGEOFFSET; 4152 pcnt = mmu_btopr(window->wd_size + poff); 4153 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4154 4155 /* unmap pages which are currently mapped in this window */ 4156 for (i = 0; i < pcnt; i++) { 4157 if (dma->dp_pgmap[pidx].pm_mapped) { 4158 hat_unload(kas.a_hat, 4159 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4160 HAT_UNLOAD); 4161 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4162 } 4163 pidx++; 4164 } 4165 } 4166 #endif 4167 4168 /* 4169 * Move to the new window. 4170 * NOTE: current_win must be set for sync to work right 4171 */ 4172 dma->dp_current_win = win; 4173 window = &dma->dp_window[win]; 4174 4175 /* if needed, adjust the first and/or last cookies for trim */ 4176 trim = &window->wd_trim; 4177 if (trim->tr_trim_first) { 4178 window->wd_first_cookie->_dmu._dmac_ll = trim->tr_first_paddr; 4179 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4180 #if !defined(__amd64) 4181 window->wd_first_cookie->dmac_type = 4182 (window->wd_first_cookie->dmac_type & 4183 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4184 #endif 4185 if (trim->tr_first_copybuf_win) { 4186 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4187 trim->tr_first_cbaddr; 4188 #if !defined(__amd64) 4189 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4190 trim->tr_first_kaddr; 4191 #endif 4192 } 4193 } 4194 if (trim->tr_trim_last) { 4195 trim->tr_last_cookie->_dmu._dmac_ll = trim->tr_last_paddr; 4196 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4197 if (trim->tr_last_copybuf_win) { 4198 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4199 trim->tr_last_cbaddr; 4200 #if !defined(__amd64) 4201 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4202 trim->tr_last_kaddr; 4203 #endif 4204 } 4205 } 4206 4207 /* 4208 * setup the cookie pointer to the first cookie in the window. setup 4209 * our return values, then increment the cookie since we return the 4210 * first cookie on the stack. 4211 */ 4212 hp->dmai_cookie = window->wd_first_cookie; 4213 *offp = window->wd_offset; 4214 *lenp = window->wd_size; 4215 *ccountp = window->wd_cookie_cnt; 4216 *cookiep = hp->dmai_cookie[0]; 4217 hp->dmai_cookie++; 4218 4219 #if !defined(__amd64) 4220 /* re-map copybuf if required for this window */ 4221 if (dma->dp_cb_remaping) { 4222 /* 4223 * calculate the page index into the buffer where this 4224 * window starts. 4225 */ 4226 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4227 MMU_PAGESHIFT; 4228 ASSERT(pidx < sinfo->si_max_pages); 4229 4230 /* 4231 * the first page can get unmapped if it's shared with the 4232 * previous window. Even if the rest of this window is already 4233 * mapped in, we need to still check this one. 4234 */ 4235 pmap = &dma->dp_pgmap[pidx]; 4236 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4237 if (pmap->pm_pp != NULL) { 4238 pmap->pm_mapped = B_TRUE; 4239 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4240 } else if (pmap->pm_vaddr != NULL) { 4241 pmap->pm_mapped = B_TRUE; 4242 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4243 pmap->pm_kaddr); 4244 } 4245 } 4246 pidx++; 4247 4248 /* map in the rest of the pages if required */ 4249 if (window->wd_remap_copybuf) { 4250 window->wd_remap_copybuf = B_FALSE; 4251 4252 /* figure out many pages this window takes up */ 4253 poff = (sinfo->si_buf_offset + window->wd_offset) & 4254 MMU_PAGEOFFSET; 4255 pcnt = mmu_btopr(window->wd_size + poff); 4256 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4257 4258 /* map pages which require it */ 4259 for (i = 1; i < pcnt; i++) { 4260 pmap = &dma->dp_pgmap[pidx]; 4261 if (pmap->pm_uses_copybuf) { 4262 ASSERT(pmap->pm_mapped == B_FALSE); 4263 if (pmap->pm_pp != NULL) { 4264 pmap->pm_mapped = B_TRUE; 4265 i86_pp_map(pmap->pm_pp, 4266 pmap->pm_kaddr); 4267 } else if (pmap->pm_vaddr != NULL) { 4268 pmap->pm_mapped = B_TRUE; 4269 i86_va_map(pmap->pm_vaddr, 4270 sinfo->si_asp, 4271 pmap->pm_kaddr); 4272 } 4273 } 4274 pidx++; 4275 } 4276 } 4277 } 4278 #endif 4279 4280 /* if the new window uses the copy buffer, sync it for the device */ 4281 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4282 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4283 DDI_DMA_SYNC_FORDEV); 4284 } 4285 4286 return (DDI_SUCCESS); 4287 } 4288 4289 4290 4291 /* 4292 * ************************ 4293 * obsoleted dma routines 4294 * ************************ 4295 */ 4296 4297 /* 4298 * rootnex_dma_map() 4299 * called from ddi_dma_setup() 4300 */ 4301 /* ARGSUSED */ 4302 static int 4303 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4304 ddi_dma_handle_t *handlep) 4305 { 4306 #if defined(__amd64) 4307 /* 4308 * this interface is not supported in 64-bit x86 kernel. See comment in 4309 * rootnex_dma_mctl() 4310 */ 4311 ASSERT(0); 4312 return (DDI_DMA_NORESOURCES); 4313 4314 #else /* 32-bit x86 kernel */ 4315 ddi_dma_handle_t *lhandlep; 4316 ddi_dma_handle_t lhandle; 4317 ddi_dma_cookie_t cookie; 4318 ddi_dma_attr_t dma_attr; 4319 ddi_dma_lim_t *dma_lim; 4320 uint_t ccnt; 4321 int e; 4322 4323 4324 /* 4325 * if the driver is just testing to see if it's possible to do the bind, 4326 * we'll use local state. Otherwise, use the handle pointer passed in. 4327 */ 4328 if (handlep == NULL) { 4329 lhandlep = &lhandle; 4330 } else { 4331 lhandlep = handlep; 4332 } 4333 4334 /* convert the limit structure to a dma_attr one */ 4335 dma_lim = dmareq->dmar_limits; 4336 dma_attr.dma_attr_version = DMA_ATTR_V0; 4337 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4338 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4339 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4340 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4341 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4342 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4343 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4344 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4345 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4346 dma_attr.dma_attr_align = MMU_PAGESIZE; 4347 dma_attr.dma_attr_flags = 0; 4348 4349 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4350 dmareq->dmar_arg, lhandlep); 4351 if (e != DDI_SUCCESS) { 4352 return (e); 4353 } 4354 4355 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4356 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4357 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4358 return (e); 4359 } 4360 4361 /* 4362 * if the driver is just testing to see if it's possible to do the bind, 4363 * free up the local state and return the result. 4364 */ 4365 if (handlep == NULL) { 4366 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4367 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4368 if (e == DDI_DMA_MAPPED) { 4369 return (DDI_DMA_MAPOK); 4370 } else { 4371 return (DDI_DMA_NOMAPPING); 4372 } 4373 } 4374 4375 return (e); 4376 #endif /* defined(__amd64) */ 4377 } 4378 4379 4380 /* 4381 * rootnex_dma_mctl() 4382 * 4383 */ 4384 /* ARGSUSED */ 4385 static int 4386 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4387 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4388 uint_t cache_flags) 4389 { 4390 #if defined(__amd64) 4391 /* 4392 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4393 * common implementation in genunix, so they no longer have x86 4394 * specific functionality which called into dma_ctl. 4395 * 4396 * The rest of the obsoleted interfaces were never supported in the 4397 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4398 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4399 * implementation issues. 4400 * 4401 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4402 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4403 * reflect that now too... 4404 * 4405 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4406 * not going to put this functionality into the 64-bit x86 kernel now. 4407 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4408 * that in a future release. 4409 */ 4410 ASSERT(0); 4411 return (DDI_FAILURE); 4412 4413 #else /* 32-bit x86 kernel */ 4414 ddi_dma_cookie_t lcookie; 4415 ddi_dma_cookie_t *cookie; 4416 rootnex_window_t *window; 4417 ddi_dma_impl_t *hp; 4418 rootnex_dma_t *dma; 4419 uint_t nwin; 4420 uint_t ccnt; 4421 size_t len; 4422 off_t off; 4423 int e; 4424 4425 4426 /* 4427 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4428 * hacky since were optimizing for the current interfaces and so we can 4429 * cleanup the mess in genunix. Hopefully we will remove the this 4430 * obsoleted routines someday soon. 4431 */ 4432 4433 switch (request) { 4434 4435 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4436 hp = (ddi_dma_impl_t *)handle; 4437 cookie = (ddi_dma_cookie_t *)objpp; 4438 4439 /* 4440 * convert segment to cookie. We don't distinguish between the 4441 * two :-) 4442 */ 4443 *cookie = *hp->dmai_cookie; 4444 *lenp = cookie->dmac_size; 4445 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4446 return (DDI_SUCCESS); 4447 4448 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4449 hp = (ddi_dma_impl_t *)handle; 4450 dma = (rootnex_dma_t *)hp->dmai_private; 4451 4452 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4453 return (DDI_DMA_STALE); 4454 } 4455 4456 /* handle the case where we don't have any windows */ 4457 if (dma->dp_window == NULL) { 4458 /* 4459 * if seg == NULL, and we don't have any windows, 4460 * return the first cookie in the sgl. 4461 */ 4462 if (*lenp == NULL) { 4463 dma->dp_current_cookie = 0; 4464 hp->dmai_cookie = dma->dp_cookies; 4465 *objpp = (caddr_t)handle; 4466 return (DDI_SUCCESS); 4467 4468 /* if we have more cookies, go to the next cookie */ 4469 } else { 4470 if ((dma->dp_current_cookie + 1) >= 4471 dma->dp_sglinfo.si_sgl_size) { 4472 return (DDI_DMA_DONE); 4473 } 4474 dma->dp_current_cookie++; 4475 hp->dmai_cookie++; 4476 return (DDI_SUCCESS); 4477 } 4478 } 4479 4480 /* We have one or more windows */ 4481 window = &dma->dp_window[dma->dp_current_win]; 4482 4483 /* 4484 * if seg == NULL, return the first cookie in the current 4485 * window 4486 */ 4487 if (*lenp == NULL) { 4488 dma->dp_current_cookie = 0; 4489 hp->dmai_cookie = dma->dp_cookies; 4490 4491 /* 4492 * go to the next cookie in the window then see if we done with 4493 * this window. 4494 */ 4495 } else { 4496 if ((dma->dp_current_cookie + 1) >= 4497 window->wd_cookie_cnt) { 4498 return (DDI_DMA_DONE); 4499 } 4500 dma->dp_current_cookie++; 4501 hp->dmai_cookie++; 4502 } 4503 *objpp = (caddr_t)handle; 4504 return (DDI_SUCCESS); 4505 4506 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4507 hp = (ddi_dma_impl_t *)handle; 4508 dma = (rootnex_dma_t *)hp->dmai_private; 4509 4510 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4511 return (DDI_DMA_STALE); 4512 } 4513 4514 /* if win == NULL, return the first window in the bind */ 4515 if (*offp == NULL) { 4516 nwin = 0; 4517 4518 /* 4519 * else, go to the next window then see if we're done with all 4520 * the windows. 4521 */ 4522 } else { 4523 nwin = dma->dp_current_win + 1; 4524 if (nwin >= hp->dmai_nwin) { 4525 return (DDI_DMA_DONE); 4526 } 4527 } 4528 4529 /* switch to the next window */ 4530 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4531 &lcookie, &ccnt); 4532 ASSERT(e == DDI_SUCCESS); 4533 if (e != DDI_SUCCESS) { 4534 return (DDI_DMA_STALE); 4535 } 4536 4537 /* reset the cookie back to the first cookie in the window */ 4538 if (dma->dp_window != NULL) { 4539 window = &dma->dp_window[dma->dp_current_win]; 4540 hp->dmai_cookie = window->wd_first_cookie; 4541 } else { 4542 hp->dmai_cookie = dma->dp_cookies; 4543 } 4544 4545 *objpp = (caddr_t)handle; 4546 return (DDI_SUCCESS); 4547 4548 case DDI_DMA_FREE: /* ddi_dma_free() */ 4549 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4550 (void) rootnex_dma_freehdl(dip, rdip, handle); 4551 if (rootnex_state->r_dvma_call_list_id) { 4552 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4553 } 4554 return (DDI_SUCCESS); 4555 4556 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4557 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4558 /* should never get here, handled in genunix */ 4559 ASSERT(0); 4560 return (DDI_FAILURE); 4561 4562 case DDI_DMA_KVADDR: 4563 case DDI_DMA_GETERR: 4564 case DDI_DMA_COFF: 4565 return (DDI_FAILURE); 4566 } 4567 4568 return (DDI_FAILURE); 4569 #endif /* defined(__amd64) */ 4570 } 4571