1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * x86 root nexus driver 30 */ 31 32 #include <sys/sysmacros.h> 33 #include <sys/conf.h> 34 #include <sys/autoconf.h> 35 #include <sys/sysmacros.h> 36 #include <sys/debug.h> 37 #include <sys/psw.h> 38 #include <sys/ddidmareq.h> 39 #include <sys/promif.h> 40 #include <sys/devops.h> 41 #include <sys/kmem.h> 42 #include <sys/cmn_err.h> 43 #include <vm/seg.h> 44 #include <vm/seg_kmem.h> 45 #include <vm/seg_dev.h> 46 #include <sys/vmem.h> 47 #include <sys/mman.h> 48 #include <vm/hat.h> 49 #include <vm/as.h> 50 #include <vm/page.h> 51 #include <sys/avintr.h> 52 #include <sys/errno.h> 53 #include <sys/modctl.h> 54 #include <sys/ddi_impldefs.h> 55 #include <sys/sunddi.h> 56 #include <sys/sunndi.h> 57 #include <sys/mach_intr.h> 58 #include <sys/psm.h> 59 #include <sys/ontrap.h> 60 #include <sys/atomic.h> 61 #include <sys/sdt.h> 62 #include <sys/rootnex.h> 63 #include <vm/hat_i86.h> 64 #include <sys/ddifm.h> 65 #include <sys/ddi_isa.h> 66 67 #ifdef __xpv 68 #include <sys/bootinfo.h> 69 #include <sys/hypervisor.h> 70 #include <sys/bootconf.h> 71 #include <vm/kboot_mmu.h> 72 #endif 73 74 /* 75 * enable/disable extra checking of function parameters. Useful for debugging 76 * drivers. 77 */ 78 #ifdef DEBUG 79 int rootnex_alloc_check_parms = 1; 80 int rootnex_bind_check_parms = 1; 81 int rootnex_bind_check_inuse = 1; 82 int rootnex_unbind_verify_buffer = 0; 83 int rootnex_sync_check_parms = 1; 84 #else 85 int rootnex_alloc_check_parms = 0; 86 int rootnex_bind_check_parms = 0; 87 int rootnex_bind_check_inuse = 0; 88 int rootnex_unbind_verify_buffer = 0; 89 int rootnex_sync_check_parms = 0; 90 #endif 91 92 /* Master Abort and Target Abort panic flag */ 93 int rootnex_fm_ma_ta_panic_flag = 0; 94 95 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 96 int rootnex_bind_fail = 1; 97 int rootnex_bind_warn = 1; 98 uint8_t *rootnex_warn_list; 99 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 100 #define ROOTNEX_BIND_WARNING (0x1 << 0) 101 102 /* 103 * revert back to old broken behavior of always sync'ing entire copy buffer. 104 * This is useful if be have a buggy driver which doesn't correctly pass in 105 * the offset and size into ddi_dma_sync(). 106 */ 107 int rootnex_sync_ignore_params = 0; 108 109 /* 110 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 111 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 112 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 113 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 114 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 115 * (< 8K). We will still need to allocate the copy buffer during bind though 116 * (if we need one). These can only be modified in /etc/system before rootnex 117 * attach. 118 */ 119 #if defined(__amd64) 120 int rootnex_prealloc_cookies = 65; 121 int rootnex_prealloc_windows = 4; 122 int rootnex_prealloc_copybuf = 2; 123 #else 124 int rootnex_prealloc_cookies = 33; 125 int rootnex_prealloc_windows = 4; 126 int rootnex_prealloc_copybuf = 2; 127 #endif 128 129 /* driver global state */ 130 static rootnex_state_t *rootnex_state; 131 132 /* shortcut to rootnex counters */ 133 static uint64_t *rootnex_cnt; 134 135 /* 136 * XXX - does x86 even need these or are they left over from the SPARC days? 137 */ 138 /* statically defined integer/boolean properties for the root node */ 139 static rootnex_intprop_t rootnex_intprp[] = { 140 { "PAGESIZE", PAGESIZE }, 141 { "MMU_PAGESIZE", MMU_PAGESIZE }, 142 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 143 { DDI_RELATIVE_ADDRESSING, 1 }, 144 }; 145 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 146 147 #ifdef __xpv 148 typedef maddr_t rootnex_addr_t; 149 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 150 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 151 #else 152 typedef paddr_t rootnex_addr_t; 153 #endif 154 155 156 static struct cb_ops rootnex_cb_ops = { 157 nodev, /* open */ 158 nodev, /* close */ 159 nodev, /* strategy */ 160 nodev, /* print */ 161 nodev, /* dump */ 162 nodev, /* read */ 163 nodev, /* write */ 164 nodev, /* ioctl */ 165 nodev, /* devmap */ 166 nodev, /* mmap */ 167 nodev, /* segmap */ 168 nochpoll, /* chpoll */ 169 ddi_prop_op, /* cb_prop_op */ 170 NULL, /* struct streamtab */ 171 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 172 CB_REV, /* Rev */ 173 nodev, /* cb_aread */ 174 nodev /* cb_awrite */ 175 }; 176 177 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 178 off_t offset, off_t len, caddr_t *vaddrp); 179 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 180 struct hat *hat, struct seg *seg, caddr_t addr, 181 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 182 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 183 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 184 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 185 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 186 ddi_dma_handle_t *handlep); 187 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 188 ddi_dma_handle_t handle); 189 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 190 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 191 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 192 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 193 ddi_dma_handle_t handle); 194 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 195 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 196 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 197 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 198 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 199 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 201 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 202 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 203 ddi_ctl_enum_t ctlop, void *arg, void *result); 204 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 205 ddi_iblock_cookie_t *ibc); 206 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 207 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 208 209 210 static struct bus_ops rootnex_bus_ops = { 211 BUSO_REV, 212 rootnex_map, 213 NULL, 214 NULL, 215 NULL, 216 rootnex_map_fault, 217 rootnex_dma_map, 218 rootnex_dma_allochdl, 219 rootnex_dma_freehdl, 220 rootnex_dma_bindhdl, 221 rootnex_dma_unbindhdl, 222 rootnex_dma_sync, 223 rootnex_dma_win, 224 rootnex_dma_mctl, 225 rootnex_ctlops, 226 ddi_bus_prop_op, 227 i_ddi_rootnex_get_eventcookie, 228 i_ddi_rootnex_add_eventcall, 229 i_ddi_rootnex_remove_eventcall, 230 i_ddi_rootnex_post_event, 231 0, /* bus_intr_ctl */ 232 0, /* bus_config */ 233 0, /* bus_unconfig */ 234 rootnex_fm_init, /* bus_fm_init */ 235 NULL, /* bus_fm_fini */ 236 NULL, /* bus_fm_access_enter */ 237 NULL, /* bus_fm_access_exit */ 238 NULL, /* bus_powr */ 239 rootnex_intr_ops /* bus_intr_op */ 240 }; 241 242 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 243 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 244 245 static struct dev_ops rootnex_ops = { 246 DEVO_REV, 247 0, 248 ddi_no_info, 249 nulldev, 250 nulldev, 251 rootnex_attach, 252 rootnex_detach, 253 nulldev, 254 &rootnex_cb_ops, 255 &rootnex_bus_ops 256 }; 257 258 static struct modldrv rootnex_modldrv = { 259 &mod_driverops, 260 "i86pc root nexus %I%", 261 &rootnex_ops 262 }; 263 264 static struct modlinkage rootnex_modlinkage = { 265 MODREV_1, 266 (void *)&rootnex_modldrv, 267 NULL 268 }; 269 270 271 /* 272 * extern hacks 273 */ 274 extern struct seg_ops segdev_ops; 275 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 276 #ifdef DDI_MAP_DEBUG 277 extern int ddi_map_debug_flag; 278 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 279 #endif 280 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 281 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 282 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 283 psm_intr_op_t, int *); 284 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 285 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 286 287 /* 288 * Use device arena to use for device control register mappings. 289 * Various kernel memory walkers (debugger, dtrace) need to know 290 * to avoid this address range to prevent undesired device activity. 291 */ 292 extern void *device_arena_alloc(size_t size, int vm_flag); 293 extern void device_arena_free(void * vaddr, size_t size); 294 295 296 /* 297 * Internal functions 298 */ 299 static int rootnex_dma_init(); 300 static void rootnex_add_props(dev_info_t *); 301 static int rootnex_ctl_reportdev(dev_info_t *dip); 302 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 303 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 304 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 305 static int rootnex_map_handle(ddi_map_req_t *mp); 306 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 307 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 308 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 309 ddi_dma_attr_t *attr); 310 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 311 rootnex_sglinfo_t *sglinfo); 312 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 313 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 314 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 315 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 316 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 317 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 318 ddi_dma_attr_t *attr, int kmflag); 319 static void rootnex_teardown_windows(rootnex_dma_t *dma); 320 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 321 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 322 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 323 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 324 size_t *copybuf_used, page_t **cur_pp); 325 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 326 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 327 ddi_dma_attr_t *attr, off_t cur_offset); 328 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 329 rootnex_dma_t *dma, rootnex_window_t **windowp, 330 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 331 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 332 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 333 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 334 off_t offset, size_t size, uint_t cache_flags); 335 static int rootnex_verify_buffer(rootnex_dma_t *dma); 336 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 337 const void *comp_addr, const void *not_used); 338 339 /* 340 * _init() 341 * 342 */ 343 int 344 _init(void) 345 { 346 347 rootnex_state = NULL; 348 return (mod_install(&rootnex_modlinkage)); 349 } 350 351 352 /* 353 * _info() 354 * 355 */ 356 int 357 _info(struct modinfo *modinfop) 358 { 359 return (mod_info(&rootnex_modlinkage, modinfop)); 360 } 361 362 363 /* 364 * _fini() 365 * 366 */ 367 int 368 _fini(void) 369 { 370 return (EBUSY); 371 } 372 373 374 /* 375 * rootnex_attach() 376 * 377 */ 378 static int 379 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 380 { 381 int fmcap; 382 int e; 383 384 switch (cmd) { 385 case DDI_ATTACH: 386 break; 387 case DDI_RESUME: 388 return (DDI_SUCCESS); 389 default: 390 return (DDI_FAILURE); 391 } 392 393 /* 394 * We should only have one instance of rootnex. Save it away since we 395 * don't have an easy way to get it back later. 396 */ 397 ASSERT(rootnex_state == NULL); 398 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 399 400 rootnex_state->r_dip = dip; 401 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 402 rootnex_state->r_reserved_msg_printed = B_FALSE; 403 rootnex_cnt = &rootnex_state->r_counters[0]; 404 405 /* 406 * Set minimum fm capability level for i86pc platforms and then 407 * initialize error handling. Since we're the rootnex, we don't 408 * care what's returned in the fmcap field. 409 */ 410 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 411 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 412 fmcap = ddi_system_fmcap; 413 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 414 415 /* initialize DMA related state */ 416 e = rootnex_dma_init(); 417 if (e != DDI_SUCCESS) { 418 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 419 return (DDI_FAILURE); 420 } 421 422 /* Add static root node properties */ 423 rootnex_add_props(dip); 424 425 /* since we can't call ddi_report_dev() */ 426 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 427 428 /* Initialize rootnex event handle */ 429 i_ddi_rootnex_init_events(dip); 430 431 return (DDI_SUCCESS); 432 } 433 434 435 /* 436 * rootnex_detach() 437 * 438 */ 439 /*ARGSUSED*/ 440 static int 441 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 442 { 443 switch (cmd) { 444 case DDI_SUSPEND: 445 break; 446 default: 447 return (DDI_FAILURE); 448 } 449 450 return (DDI_SUCCESS); 451 } 452 453 454 /* 455 * rootnex_dma_init() 456 * 457 */ 458 /*ARGSUSED*/ 459 static int 460 rootnex_dma_init() 461 { 462 size_t bufsize; 463 464 465 /* 466 * size of our cookie/window/copybuf state needed in dma bind that we 467 * pre-alloc in dma_alloc_handle 468 */ 469 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 470 rootnex_state->r_prealloc_size = 471 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 472 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 473 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 474 475 /* 476 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 477 * allocate 16 extra bytes for struct pointer alignment 478 * (p->dmai_private & dma->dp_prealloc_buffer) 479 */ 480 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 481 rootnex_state->r_prealloc_size + 0x10; 482 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 483 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 484 if (rootnex_state->r_dmahdl_cache == NULL) { 485 return (DDI_FAILURE); 486 } 487 488 /* 489 * allocate array to track which major numbers we have printed warnings 490 * for. 491 */ 492 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 493 KM_SLEEP); 494 495 return (DDI_SUCCESS); 496 } 497 498 499 /* 500 * rootnex_add_props() 501 * 502 */ 503 static void 504 rootnex_add_props(dev_info_t *dip) 505 { 506 rootnex_intprop_t *rpp; 507 int i; 508 509 /* Add static integer/boolean properties to the root node */ 510 rpp = rootnex_intprp; 511 for (i = 0; i < NROOT_INTPROPS; i++) { 512 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 513 rpp[i].prop_name, rpp[i].prop_value); 514 } 515 } 516 517 518 519 /* 520 * ************************* 521 * ctlops related routines 522 * ************************* 523 */ 524 525 /* 526 * rootnex_ctlops() 527 * 528 */ 529 /*ARGSUSED*/ 530 static int 531 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 532 void *arg, void *result) 533 { 534 int n, *ptr; 535 struct ddi_parent_private_data *pdp; 536 537 switch (ctlop) { 538 case DDI_CTLOPS_DMAPMAPC: 539 /* 540 * Return 'partial' to indicate that dma mapping 541 * has to be done in the main MMU. 542 */ 543 return (DDI_DMA_PARTIAL); 544 545 case DDI_CTLOPS_BTOP: 546 /* 547 * Convert byte count input to physical page units. 548 * (byte counts that are not a page-size multiple 549 * are rounded down) 550 */ 551 *(ulong_t *)result = btop(*(ulong_t *)arg); 552 return (DDI_SUCCESS); 553 554 case DDI_CTLOPS_PTOB: 555 /* 556 * Convert size in physical pages to bytes 557 */ 558 *(ulong_t *)result = ptob(*(ulong_t *)arg); 559 return (DDI_SUCCESS); 560 561 case DDI_CTLOPS_BTOPR: 562 /* 563 * Convert byte count input to physical page units 564 * (byte counts that are not a page-size multiple 565 * are rounded up) 566 */ 567 *(ulong_t *)result = btopr(*(ulong_t *)arg); 568 return (DDI_SUCCESS); 569 570 case DDI_CTLOPS_INITCHILD: 571 return (impl_ddi_sunbus_initchild(arg)); 572 573 case DDI_CTLOPS_UNINITCHILD: 574 impl_ddi_sunbus_removechild(arg); 575 return (DDI_SUCCESS); 576 577 case DDI_CTLOPS_REPORTDEV: 578 return (rootnex_ctl_reportdev(rdip)); 579 580 case DDI_CTLOPS_IOMIN: 581 /* 582 * Nothing to do here but reflect back.. 583 */ 584 return (DDI_SUCCESS); 585 586 case DDI_CTLOPS_REGSIZE: 587 case DDI_CTLOPS_NREGS: 588 break; 589 590 case DDI_CTLOPS_SIDDEV: 591 if (ndi_dev_is_prom_node(rdip)) 592 return (DDI_SUCCESS); 593 if (ndi_dev_is_persistent_node(rdip)) 594 return (DDI_SUCCESS); 595 return (DDI_FAILURE); 596 597 case DDI_CTLOPS_POWER: 598 return ((*pm_platform_power)((power_req_t *)arg)); 599 600 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 601 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 602 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 603 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 604 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 605 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 606 if (!rootnex_state->r_reserved_msg_printed) { 607 rootnex_state->r_reserved_msg_printed = B_TRUE; 608 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 609 "1 or more reserved/obsolete operations."); 610 } 611 return (DDI_FAILURE); 612 613 default: 614 return (DDI_FAILURE); 615 } 616 /* 617 * The rest are for "hardware" properties 618 */ 619 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 620 return (DDI_FAILURE); 621 622 if (ctlop == DDI_CTLOPS_NREGS) { 623 ptr = (int *)result; 624 *ptr = pdp->par_nreg; 625 } else { 626 off_t *size = (off_t *)result; 627 628 ptr = (int *)arg; 629 n = *ptr; 630 if (n >= pdp->par_nreg) { 631 return (DDI_FAILURE); 632 } 633 *size = (off_t)pdp->par_reg[n].regspec_size; 634 } 635 return (DDI_SUCCESS); 636 } 637 638 639 /* 640 * rootnex_ctl_reportdev() 641 * 642 */ 643 static int 644 rootnex_ctl_reportdev(dev_info_t *dev) 645 { 646 int i, n, len, f_len = 0; 647 char *buf; 648 649 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 650 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 651 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 652 len = strlen(buf); 653 654 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 655 656 struct regspec *rp = sparc_pd_getreg(dev, i); 657 658 if (i == 0) 659 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 660 ": "); 661 else 662 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 663 " and "); 664 len = strlen(buf); 665 666 switch (rp->regspec_bustype) { 667 668 case BTEISA: 669 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 670 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 671 break; 672 673 case BTISA: 674 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 675 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 676 break; 677 678 default: 679 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 680 "space %x offset %x", 681 rp->regspec_bustype, rp->regspec_addr); 682 break; 683 } 684 len = strlen(buf); 685 } 686 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 687 int pri; 688 689 if (i != 0) { 690 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 691 ","); 692 len = strlen(buf); 693 } 694 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 695 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 696 " sparc ipl %d", pri); 697 len = strlen(buf); 698 } 699 #ifdef DEBUG 700 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 701 cmn_err(CE_NOTE, "next message is truncated: " 702 "printed length 1024, real length %d", f_len); 703 } 704 #endif /* DEBUG */ 705 cmn_err(CE_CONT, "?%s\n", buf); 706 kmem_free(buf, REPORTDEV_BUFSIZE); 707 return (DDI_SUCCESS); 708 } 709 710 711 /* 712 * ****************** 713 * map related code 714 * ****************** 715 */ 716 717 /* 718 * rootnex_map() 719 * 720 */ 721 static int 722 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 723 off_t len, caddr_t *vaddrp) 724 { 725 struct regspec *rp, tmp_reg; 726 ddi_map_req_t mr = *mp; /* Get private copy of request */ 727 int error; 728 729 mp = &mr; 730 731 switch (mp->map_op) { 732 case DDI_MO_MAP_LOCKED: 733 case DDI_MO_UNMAP: 734 case DDI_MO_MAP_HANDLE: 735 break; 736 default: 737 #ifdef DDI_MAP_DEBUG 738 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 739 mp->map_op); 740 #endif /* DDI_MAP_DEBUG */ 741 return (DDI_ME_UNIMPLEMENTED); 742 } 743 744 if (mp->map_flags & DDI_MF_USER_MAPPING) { 745 #ifdef DDI_MAP_DEBUG 746 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 747 #endif /* DDI_MAP_DEBUG */ 748 return (DDI_ME_UNIMPLEMENTED); 749 } 750 751 /* 752 * First, if given an rnumber, convert it to a regspec... 753 * (Presumably, this is on behalf of a child of the root node?) 754 */ 755 756 if (mp->map_type == DDI_MT_RNUMBER) { 757 758 int rnumber = mp->map_obj.rnumber; 759 #ifdef DDI_MAP_DEBUG 760 static char *out_of_range = 761 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 762 #endif /* DDI_MAP_DEBUG */ 763 764 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 765 if (rp == NULL) { 766 #ifdef DDI_MAP_DEBUG 767 cmn_err(CE_WARN, out_of_range, rnumber, 768 ddi_get_name(rdip)); 769 #endif /* DDI_MAP_DEBUG */ 770 return (DDI_ME_RNUMBER_RANGE); 771 } 772 773 /* 774 * Convert the given ddi_map_req_t from rnumber to regspec... 775 */ 776 777 mp->map_type = DDI_MT_REGSPEC; 778 mp->map_obj.rp = rp; 779 } 780 781 /* 782 * Adjust offset and length correspnding to called values... 783 * XXX: A non-zero length means override the one in the regspec 784 * XXX: (regardless of what's in the parent's range?) 785 */ 786 787 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 788 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 789 790 #ifdef DDI_MAP_DEBUG 791 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 792 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 793 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 794 len, mp->map_handlep); 795 #endif /* DDI_MAP_DEBUG */ 796 797 /* 798 * I/O or memory mapping: 799 * 800 * <bustype=0, addr=x, len=x>: memory 801 * <bustype=1, addr=x, len=x>: i/o 802 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 803 */ 804 805 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 806 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 807 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 808 ddi_get_name(rdip), rp->regspec_bustype, 809 rp->regspec_addr, rp->regspec_size); 810 return (DDI_ME_INVAL); 811 } 812 813 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 814 /* 815 * compatibility i/o mapping 816 */ 817 rp->regspec_bustype += (uint_t)offset; 818 } else { 819 /* 820 * Normal memory or i/o mapping 821 */ 822 rp->regspec_addr += (uint_t)offset; 823 } 824 825 if (len != 0) 826 rp->regspec_size = (uint_t)len; 827 828 #ifdef DDI_MAP_DEBUG 829 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 830 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 831 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 832 offset, len, mp->map_handlep); 833 #endif /* DDI_MAP_DEBUG */ 834 835 /* 836 * Apply any parent ranges at this level, if applicable. 837 * (This is where nexus specific regspec translation takes place. 838 * Use of this function is implicit agreement that translation is 839 * provided via ddi_apply_range.) 840 */ 841 842 #ifdef DDI_MAP_DEBUG 843 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 844 ddi_get_name(dip), ddi_get_name(rdip)); 845 #endif /* DDI_MAP_DEBUG */ 846 847 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 848 return (error); 849 850 switch (mp->map_op) { 851 case DDI_MO_MAP_LOCKED: 852 853 /* 854 * Set up the locked down kernel mapping to the regspec... 855 */ 856 857 return (rootnex_map_regspec(mp, vaddrp)); 858 859 case DDI_MO_UNMAP: 860 861 /* 862 * Release mapping... 863 */ 864 865 return (rootnex_unmap_regspec(mp, vaddrp)); 866 867 case DDI_MO_MAP_HANDLE: 868 869 return (rootnex_map_handle(mp)); 870 871 default: 872 return (DDI_ME_UNIMPLEMENTED); 873 } 874 } 875 876 877 /* 878 * rootnex_map_fault() 879 * 880 * fault in mappings for requestors 881 */ 882 /*ARGSUSED*/ 883 static int 884 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 885 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 886 uint_t lock) 887 { 888 889 #ifdef DDI_MAP_DEBUG 890 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 891 ddi_map_debug(" Seg <%s>\n", 892 seg->s_ops == &segdev_ops ? "segdev" : 893 seg == &kvseg ? "segkmem" : "NONE!"); 894 #endif /* DDI_MAP_DEBUG */ 895 896 /* 897 * This is all terribly broken, but it is a start 898 * 899 * XXX Note that this test means that segdev_ops 900 * must be exported from seg_dev.c. 901 * XXX What about devices with their own segment drivers? 902 */ 903 if (seg->s_ops == &segdev_ops) { 904 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 905 906 if (hat == NULL) { 907 /* 908 * This is one plausible interpretation of 909 * a null hat i.e. use the first hat on the 910 * address space hat list which by convention is 911 * the hat of the system MMU. At alternative 912 * would be to panic .. this might well be better .. 913 */ 914 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 915 hat = seg->s_as->a_hat; 916 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 917 } 918 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 919 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 920 } else if (seg == &kvseg && dp == NULL) { 921 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 922 HAT_LOAD_LOCK); 923 } else 924 return (DDI_FAILURE); 925 return (DDI_SUCCESS); 926 } 927 928 929 /* 930 * rootnex_map_regspec() 931 * we don't support mapping of I/O cards above 4Gb 932 */ 933 static int 934 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 935 { 936 rootnex_addr_t rbase; 937 void *cvaddr; 938 uint_t npages, pgoffset; 939 struct regspec *rp; 940 ddi_acc_hdl_t *hp; 941 ddi_acc_impl_t *ap; 942 uint_t hat_acc_flags; 943 paddr_t pbase; 944 945 rp = mp->map_obj.rp; 946 hp = mp->map_handlep; 947 948 #ifdef DDI_MAP_DEBUG 949 ddi_map_debug( 950 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 951 rp->regspec_bustype, rp->regspec_addr, 952 rp->regspec_size, mp->map_handlep); 953 #endif /* DDI_MAP_DEBUG */ 954 955 /* 956 * I/O or memory mapping 957 * 958 * <bustype=0, addr=x, len=x>: memory 959 * <bustype=1, addr=x, len=x>: i/o 960 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 961 */ 962 963 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 964 cmn_err(CE_WARN, "rootnex: invalid register spec" 965 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 966 rp->regspec_addr, rp->regspec_size); 967 return (DDI_FAILURE); 968 } 969 970 if (rp->regspec_bustype != 0) { 971 /* 972 * I/O space - needs a handle. 973 */ 974 if (hp == NULL) { 975 return (DDI_FAILURE); 976 } 977 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 978 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 979 impl_acc_hdl_init(hp); 980 981 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 982 #ifdef DDI_MAP_DEBUG 983 ddi_map_debug("rootnex_map_regspec: mmap() " 984 "to I/O space is not supported.\n"); 985 #endif /* DDI_MAP_DEBUG */ 986 return (DDI_ME_INVAL); 987 } else { 988 /* 989 * 1275-compliant vs. compatibility i/o mapping 990 */ 991 *vaddrp = 992 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 993 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 994 ((caddr_t)(uintptr_t)rp->regspec_addr); 995 #ifdef __xpv 996 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 997 hp->ah_pfn = xen_assign_pfn( 998 mmu_btop((ulong_t)rp->regspec_addr & 999 MMU_PAGEMASK)); 1000 } else { 1001 hp->ah_pfn = mmu_btop( 1002 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1003 } 1004 #else 1005 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1006 MMU_PAGEMASK); 1007 #endif 1008 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1009 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1010 } 1011 1012 #ifdef DDI_MAP_DEBUG 1013 ddi_map_debug( 1014 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1015 rp->regspec_size, *vaddrp); 1016 #endif /* DDI_MAP_DEBUG */ 1017 return (DDI_SUCCESS); 1018 } 1019 1020 /* 1021 * Memory space 1022 */ 1023 1024 if (hp != NULL) { 1025 /* 1026 * hat layer ignores 1027 * hp->ah_acc.devacc_attr_endian_flags. 1028 */ 1029 switch (hp->ah_acc.devacc_attr_dataorder) { 1030 case DDI_STRICTORDER_ACC: 1031 hat_acc_flags = HAT_STRICTORDER; 1032 break; 1033 case DDI_UNORDERED_OK_ACC: 1034 hat_acc_flags = HAT_UNORDERED_OK; 1035 break; 1036 case DDI_MERGING_OK_ACC: 1037 hat_acc_flags = HAT_MERGING_OK; 1038 break; 1039 case DDI_LOADCACHING_OK_ACC: 1040 hat_acc_flags = HAT_LOADCACHING_OK; 1041 break; 1042 case DDI_STORECACHING_OK_ACC: 1043 hat_acc_flags = HAT_STORECACHING_OK; 1044 break; 1045 } 1046 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1047 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1048 impl_acc_hdl_init(hp); 1049 hp->ah_hat_flags = hat_acc_flags; 1050 } else { 1051 hat_acc_flags = HAT_STRICTORDER; 1052 } 1053 1054 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1055 #ifdef __xpv 1056 /* 1057 * If we're dom0, we're using a real device so we need to translate 1058 * the MA to a PA. 1059 */ 1060 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1061 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1062 } else { 1063 pbase = rbase; 1064 } 1065 #else 1066 pbase = rbase; 1067 #endif 1068 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1069 1070 if (rp->regspec_size == 0) { 1071 #ifdef DDI_MAP_DEBUG 1072 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1073 #endif /* DDI_MAP_DEBUG */ 1074 return (DDI_ME_INVAL); 1075 } 1076 1077 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1078 /* extra cast to make gcc happy */ 1079 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1080 } else { 1081 npages = mmu_btopr(rp->regspec_size + pgoffset); 1082 1083 #ifdef DDI_MAP_DEBUG 1084 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1085 "physical %llx", npages, pbase); 1086 #endif /* DDI_MAP_DEBUG */ 1087 1088 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1089 if (cvaddr == NULL) 1090 return (DDI_ME_NORESOURCES); 1091 1092 /* 1093 * Now map in the pages we've allocated... 1094 */ 1095 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1096 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1097 HAT_LOAD_LOCK); 1098 *vaddrp = (caddr_t)cvaddr + pgoffset; 1099 1100 /* save away pfn and npages for FMA */ 1101 hp = mp->map_handlep; 1102 if (hp) { 1103 hp->ah_pfn = mmu_btop(pbase); 1104 hp->ah_pnum = npages; 1105 } 1106 } 1107 1108 #ifdef DDI_MAP_DEBUG 1109 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1110 #endif /* DDI_MAP_DEBUG */ 1111 return (DDI_SUCCESS); 1112 } 1113 1114 1115 /* 1116 * rootnex_unmap_regspec() 1117 * 1118 */ 1119 static int 1120 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1121 { 1122 caddr_t addr = (caddr_t)*vaddrp; 1123 uint_t npages, pgoffset; 1124 struct regspec *rp; 1125 1126 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1127 return (0); 1128 1129 rp = mp->map_obj.rp; 1130 1131 if (rp->regspec_size == 0) { 1132 #ifdef DDI_MAP_DEBUG 1133 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1134 #endif /* DDI_MAP_DEBUG */ 1135 return (DDI_ME_INVAL); 1136 } 1137 1138 /* 1139 * I/O or memory mapping: 1140 * 1141 * <bustype=0, addr=x, len=x>: memory 1142 * <bustype=1, addr=x, len=x>: i/o 1143 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1144 */ 1145 if (rp->regspec_bustype != 0) { 1146 /* 1147 * This is I/O space, which requires no particular 1148 * processing on unmap since it isn't mapped in the 1149 * first place. 1150 */ 1151 return (DDI_SUCCESS); 1152 } 1153 1154 /* 1155 * Memory space 1156 */ 1157 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1158 npages = mmu_btopr(rp->regspec_size + pgoffset); 1159 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1160 device_arena_free(addr - pgoffset, ptob(npages)); 1161 1162 /* 1163 * Destroy the pointer - the mapping has logically gone 1164 */ 1165 *vaddrp = NULL; 1166 1167 return (DDI_SUCCESS); 1168 } 1169 1170 1171 /* 1172 * rootnex_map_handle() 1173 * 1174 */ 1175 static int 1176 rootnex_map_handle(ddi_map_req_t *mp) 1177 { 1178 rootnex_addr_t rbase; 1179 ddi_acc_hdl_t *hp; 1180 uint_t pgoffset; 1181 struct regspec *rp; 1182 paddr_t pbase; 1183 1184 rp = mp->map_obj.rp; 1185 1186 #ifdef DDI_MAP_DEBUG 1187 ddi_map_debug( 1188 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1189 rp->regspec_bustype, rp->regspec_addr, 1190 rp->regspec_size, mp->map_handlep); 1191 #endif /* DDI_MAP_DEBUG */ 1192 1193 /* 1194 * I/O or memory mapping: 1195 * 1196 * <bustype=0, addr=x, len=x>: memory 1197 * <bustype=1, addr=x, len=x>: i/o 1198 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1199 */ 1200 if (rp->regspec_bustype != 0) { 1201 /* 1202 * This refers to I/O space, and we don't support "mapping" 1203 * I/O space to a user. 1204 */ 1205 return (DDI_FAILURE); 1206 } 1207 1208 /* 1209 * Set up the hat_flags for the mapping. 1210 */ 1211 hp = mp->map_handlep; 1212 1213 switch (hp->ah_acc.devacc_attr_endian_flags) { 1214 case DDI_NEVERSWAP_ACC: 1215 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1216 break; 1217 case DDI_STRUCTURE_LE_ACC: 1218 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1219 break; 1220 case DDI_STRUCTURE_BE_ACC: 1221 return (DDI_FAILURE); 1222 default: 1223 return (DDI_REGS_ACC_CONFLICT); 1224 } 1225 1226 switch (hp->ah_acc.devacc_attr_dataorder) { 1227 case DDI_STRICTORDER_ACC: 1228 break; 1229 case DDI_UNORDERED_OK_ACC: 1230 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1231 break; 1232 case DDI_MERGING_OK_ACC: 1233 hp->ah_hat_flags |= HAT_MERGING_OK; 1234 break; 1235 case DDI_LOADCACHING_OK_ACC: 1236 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1237 break; 1238 case DDI_STORECACHING_OK_ACC: 1239 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1240 break; 1241 default: 1242 return (DDI_FAILURE); 1243 } 1244 1245 rbase = (rootnex_addr_t)rp->regspec_addr & 1246 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1247 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1248 1249 if (rp->regspec_size == 0) 1250 return (DDI_ME_INVAL); 1251 1252 #ifdef __xpv 1253 /* 1254 * If we're dom0, we're using a real device so we need to translate 1255 * the MA to a PA. 1256 */ 1257 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1258 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1259 (rbase & MMU_PAGEOFFSET); 1260 } else { 1261 pbase = rbase; 1262 } 1263 #else 1264 pbase = rbase; 1265 #endif 1266 1267 hp->ah_pfn = mmu_btop(pbase); 1268 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1269 1270 return (DDI_SUCCESS); 1271 } 1272 1273 1274 1275 /* 1276 * ************************ 1277 * interrupt related code 1278 * ************************ 1279 */ 1280 1281 /* 1282 * rootnex_intr_ops() 1283 * bus_intr_op() function for interrupt support 1284 */ 1285 /* ARGSUSED */ 1286 static int 1287 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1288 ddi_intr_handle_impl_t *hdlp, void *result) 1289 { 1290 struct intrspec *ispec; 1291 struct ddi_parent_private_data *pdp; 1292 1293 DDI_INTR_NEXDBG((CE_CONT, 1294 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1295 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1296 1297 /* Process the interrupt operation */ 1298 switch (intr_op) { 1299 case DDI_INTROP_GETCAP: 1300 /* First check with pcplusmp */ 1301 if (psm_intr_ops == NULL) 1302 return (DDI_FAILURE); 1303 1304 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1305 *(int *)result = 0; 1306 return (DDI_FAILURE); 1307 } 1308 break; 1309 case DDI_INTROP_SETCAP: 1310 if (psm_intr_ops == NULL) 1311 return (DDI_FAILURE); 1312 1313 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1314 return (DDI_FAILURE); 1315 break; 1316 case DDI_INTROP_ALLOC: 1317 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1318 return (DDI_FAILURE); 1319 hdlp->ih_pri = ispec->intrspec_pri; 1320 *(int *)result = hdlp->ih_scratch1; 1321 break; 1322 case DDI_INTROP_FREE: 1323 pdp = ddi_get_parent_data(rdip); 1324 /* 1325 * Special case for 'pcic' driver' only. 1326 * If an intrspec was created for it, clean it up here 1327 * See detailed comments on this in the function 1328 * rootnex_get_ispec(). 1329 */ 1330 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1331 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1332 pdp->par_nintr); 1333 /* 1334 * Set it to zero; so that 1335 * DDI framework doesn't free it again 1336 */ 1337 pdp->par_intr = NULL; 1338 pdp->par_nintr = 0; 1339 } 1340 break; 1341 case DDI_INTROP_GETPRI: 1342 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1343 return (DDI_FAILURE); 1344 *(int *)result = ispec->intrspec_pri; 1345 break; 1346 case DDI_INTROP_SETPRI: 1347 /* Validate the interrupt priority passed to us */ 1348 if (*(int *)result > LOCK_LEVEL) 1349 return (DDI_FAILURE); 1350 1351 /* Ensure that PSM is all initialized and ispec is ok */ 1352 if ((psm_intr_ops == NULL) || 1353 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1354 return (DDI_FAILURE); 1355 1356 /* Change the priority */ 1357 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1358 PSM_FAILURE) 1359 return (DDI_FAILURE); 1360 1361 /* update the ispec with the new priority */ 1362 ispec->intrspec_pri = *(int *)result; 1363 break; 1364 case DDI_INTROP_ADDISR: 1365 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1366 return (DDI_FAILURE); 1367 ispec->intrspec_func = hdlp->ih_cb_func; 1368 break; 1369 case DDI_INTROP_REMISR: 1370 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1371 return (DDI_FAILURE); 1372 ispec->intrspec_func = (uint_t (*)()) 0; 1373 break; 1374 case DDI_INTROP_ENABLE: 1375 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1376 return (DDI_FAILURE); 1377 1378 /* Call psmi to translate irq with the dip */ 1379 if (psm_intr_ops == NULL) 1380 return (DDI_FAILURE); 1381 1382 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1383 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1384 (int *)&hdlp->ih_vector); 1385 1386 /* Add the interrupt handler */ 1387 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1388 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1389 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1390 return (DDI_FAILURE); 1391 break; 1392 case DDI_INTROP_DISABLE: 1393 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1394 return (DDI_FAILURE); 1395 1396 /* Call psm_ops() to translate irq with the dip */ 1397 if (psm_intr_ops == NULL) 1398 return (DDI_FAILURE); 1399 1400 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1401 (void) (*psm_intr_ops)(rdip, hdlp, 1402 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1403 1404 /* Remove the interrupt handler */ 1405 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1406 hdlp->ih_cb_func, hdlp->ih_vector); 1407 break; 1408 case DDI_INTROP_SETMASK: 1409 if (psm_intr_ops == NULL) 1410 return (DDI_FAILURE); 1411 1412 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1413 return (DDI_FAILURE); 1414 break; 1415 case DDI_INTROP_CLRMASK: 1416 if (psm_intr_ops == NULL) 1417 return (DDI_FAILURE); 1418 1419 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1420 return (DDI_FAILURE); 1421 break; 1422 case DDI_INTROP_GETPENDING: 1423 if (psm_intr_ops == NULL) 1424 return (DDI_FAILURE); 1425 1426 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1427 result)) { 1428 *(int *)result = 0; 1429 return (DDI_FAILURE); 1430 } 1431 break; 1432 case DDI_INTROP_NAVAIL: 1433 case DDI_INTROP_NINTRS: 1434 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1435 if (*(int *)result == 0) { 1436 /* 1437 * Special case for 'pcic' driver' only. This driver 1438 * driver is a child of 'isa' and 'rootnex' drivers. 1439 * 1440 * See detailed comments on this in the function 1441 * rootnex_get_ispec(). 1442 * 1443 * Children of 'pcic' send 'NINITR' request all the 1444 * way to rootnex driver. But, the 'pdp->par_nintr' 1445 * field may not initialized. So, we fake it here 1446 * to return 1 (a la what PCMCIA nexus does). 1447 */ 1448 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1449 *(int *)result = 1; 1450 else 1451 return (DDI_FAILURE); 1452 } 1453 break; 1454 case DDI_INTROP_SUPPORTED_TYPES: 1455 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1456 break; 1457 default: 1458 return (DDI_FAILURE); 1459 } 1460 1461 return (DDI_SUCCESS); 1462 } 1463 1464 1465 /* 1466 * rootnex_get_ispec() 1467 * convert an interrupt number to an interrupt specification. 1468 * The interrupt number determines which interrupt spec will be 1469 * returned if more than one exists. 1470 * 1471 * Look into the parent private data area of the 'rdip' to find out 1472 * the interrupt specification. First check to make sure there is 1473 * one that matchs "inumber" and then return a pointer to it. 1474 * 1475 * Return NULL if one could not be found. 1476 * 1477 * NOTE: This is needed for rootnex_intr_ops() 1478 */ 1479 static struct intrspec * 1480 rootnex_get_ispec(dev_info_t *rdip, int inum) 1481 { 1482 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1483 1484 /* 1485 * Special case handling for drivers that provide their own 1486 * intrspec structures instead of relying on the DDI framework. 1487 * 1488 * A broken hardware driver in ON could potentially provide its 1489 * own intrspec structure, instead of relying on the hardware. 1490 * If these drivers are children of 'rootnex' then we need to 1491 * continue to provide backward compatibility to them here. 1492 * 1493 * Following check is a special case for 'pcic' driver which 1494 * was found to have broken hardwre andby provides its own intrspec. 1495 * 1496 * Verbatim comments from this driver are shown here: 1497 * "Don't use the ddi_add_intr since we don't have a 1498 * default intrspec in all cases." 1499 * 1500 * Since an 'ispec' may not be always created for it, 1501 * check for that and create one if so. 1502 * 1503 * NOTE: Currently 'pcic' is the only driver found to do this. 1504 */ 1505 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1506 pdp->par_nintr = 1; 1507 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1508 pdp->par_nintr, KM_SLEEP); 1509 } 1510 1511 /* Validate the interrupt number */ 1512 if (inum >= pdp->par_nintr) 1513 return (NULL); 1514 1515 /* Get the interrupt structure pointer and return that */ 1516 return ((struct intrspec *)&pdp->par_intr[inum]); 1517 } 1518 1519 1520 /* 1521 * ****************** 1522 * dma related code 1523 * ****************** 1524 */ 1525 1526 /* 1527 * rootnex_dma_allochdl() 1528 * called from ddi_dma_alloc_handle(). 1529 */ 1530 /*ARGSUSED*/ 1531 static int 1532 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1533 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1534 { 1535 uint64_t maxsegmentsize_ll; 1536 uint_t maxsegmentsize; 1537 ddi_dma_impl_t *hp; 1538 rootnex_dma_t *dma; 1539 uint64_t count_max; 1540 uint64_t seg; 1541 int kmflag; 1542 int e; 1543 1544 1545 /* convert our sleep flags */ 1546 if (waitfp == DDI_DMA_SLEEP) { 1547 kmflag = KM_SLEEP; 1548 } else { 1549 kmflag = KM_NOSLEEP; 1550 } 1551 1552 /* 1553 * We try to do only one memory allocation here. We'll do a little 1554 * pointer manipulation later. If the bind ends up taking more than 1555 * our prealloc's space, we'll have to allocate more memory in the 1556 * bind operation. Not great, but much better than before and the 1557 * best we can do with the current bind interfaces. 1558 */ 1559 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1560 if (hp == NULL) { 1561 if (waitfp != DDI_DMA_DONTWAIT) { 1562 ddi_set_callback(waitfp, arg, 1563 &rootnex_state->r_dvma_call_list_id); 1564 } 1565 return (DDI_DMA_NORESOURCES); 1566 } 1567 1568 /* Do our pointer manipulation now, align the structures */ 1569 hp->dmai_private = (void *)(((uintptr_t)hp + 1570 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1571 dma = (rootnex_dma_t *)hp->dmai_private; 1572 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1573 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1574 1575 /* setup the handle */ 1576 rootnex_clean_dmahdl(hp); 1577 dma->dp_dip = rdip; 1578 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1579 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1580 hp->dmai_minxfer = attr->dma_attr_minxfer; 1581 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1582 hp->dmai_rdip = rdip; 1583 hp->dmai_attr = *attr; 1584 1585 /* we don't need to worry about the SPL since we do a tryenter */ 1586 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1587 1588 /* 1589 * Figure out our maximum segment size. If the segment size is greater 1590 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1591 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1592 * dma_attr_count_max are size-1 type values. 1593 * 1594 * Maximum segment size is the largest physically contiguous chunk of 1595 * memory that we can return from a bind (i.e. the maximum size of a 1596 * single cookie). 1597 */ 1598 1599 /* handle the rollover cases */ 1600 seg = attr->dma_attr_seg + 1; 1601 if (seg < attr->dma_attr_seg) { 1602 seg = attr->dma_attr_seg; 1603 } 1604 count_max = attr->dma_attr_count_max + 1; 1605 if (count_max < attr->dma_attr_count_max) { 1606 count_max = attr->dma_attr_count_max; 1607 } 1608 1609 /* 1610 * granularity may or may not be a power of two. If it isn't, we can't 1611 * use a simple mask. 1612 */ 1613 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1614 dma->dp_granularity_power_2 = B_FALSE; 1615 } else { 1616 dma->dp_granularity_power_2 = B_TRUE; 1617 } 1618 1619 /* 1620 * maxxfer should be a whole multiple of granularity. If we're going to 1621 * break up a window because we're greater than maxxfer, we might as 1622 * well make sure it's maxxfer is a whole multiple so we don't have to 1623 * worry about triming the window later on for this case. 1624 */ 1625 if (attr->dma_attr_granular > 1) { 1626 if (dma->dp_granularity_power_2) { 1627 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1628 (attr->dma_attr_maxxfer & 1629 (attr->dma_attr_granular - 1)); 1630 } else { 1631 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1632 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1633 } 1634 } else { 1635 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1636 } 1637 1638 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1639 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1640 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1641 maxsegmentsize = 0xFFFFFFFF; 1642 } else { 1643 maxsegmentsize = maxsegmentsize_ll; 1644 } 1645 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1646 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1647 1648 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1649 if (rootnex_alloc_check_parms) { 1650 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1651 if (e != DDI_SUCCESS) { 1652 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1653 (void) rootnex_dma_freehdl(dip, rdip, 1654 (ddi_dma_handle_t)hp); 1655 return (e); 1656 } 1657 } 1658 1659 *handlep = (ddi_dma_handle_t)hp; 1660 1661 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1662 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1663 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1664 1665 return (DDI_SUCCESS); 1666 } 1667 1668 1669 /* 1670 * rootnex_dma_freehdl() 1671 * called from ddi_dma_free_handle(). 1672 */ 1673 /*ARGSUSED*/ 1674 static int 1675 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1676 { 1677 ddi_dma_impl_t *hp; 1678 rootnex_dma_t *dma; 1679 1680 1681 hp = (ddi_dma_impl_t *)handle; 1682 dma = (rootnex_dma_t *)hp->dmai_private; 1683 1684 /* unbind should have been called first */ 1685 ASSERT(!dma->dp_inuse); 1686 1687 mutex_destroy(&dma->dp_mutex); 1688 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1689 1690 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1691 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1692 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1693 1694 if (rootnex_state->r_dvma_call_list_id) 1695 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1696 1697 return (DDI_SUCCESS); 1698 } 1699 1700 1701 /* 1702 * rootnex_dma_bindhdl() 1703 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1704 */ 1705 /*ARGSUSED*/ 1706 static int 1707 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1708 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1709 { 1710 rootnex_sglinfo_t *sinfo; 1711 ddi_dma_attr_t *attr; 1712 ddi_dma_impl_t *hp; 1713 rootnex_dma_t *dma; 1714 int kmflag; 1715 int e; 1716 1717 1718 hp = (ddi_dma_impl_t *)handle; 1719 dma = (rootnex_dma_t *)hp->dmai_private; 1720 sinfo = &dma->dp_sglinfo; 1721 attr = &hp->dmai_attr; 1722 1723 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1724 1725 /* 1726 * This is useful for debugging a driver. Not as useful in a production 1727 * system. The only time this will fail is if you have a driver bug. 1728 */ 1729 if (rootnex_bind_check_inuse) { 1730 /* 1731 * No one else should ever have this lock unless someone else 1732 * is trying to use this handle. So contention on the lock 1733 * is the same as inuse being set. 1734 */ 1735 e = mutex_tryenter(&dma->dp_mutex); 1736 if (e == 0) { 1737 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1738 return (DDI_DMA_INUSE); 1739 } 1740 if (dma->dp_inuse) { 1741 mutex_exit(&dma->dp_mutex); 1742 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1743 return (DDI_DMA_INUSE); 1744 } 1745 dma->dp_inuse = B_TRUE; 1746 mutex_exit(&dma->dp_mutex); 1747 } 1748 1749 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1750 if (rootnex_bind_check_parms) { 1751 e = rootnex_valid_bind_parms(dmareq, attr); 1752 if (e != DDI_SUCCESS) { 1753 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1754 rootnex_clean_dmahdl(hp); 1755 return (e); 1756 } 1757 } 1758 1759 /* save away the original bind info */ 1760 dma->dp_dma = dmareq->dmar_object; 1761 1762 /* 1763 * Figure out a rough estimate of what maximum number of pages this 1764 * buffer could use (a high estimate of course). 1765 */ 1766 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1767 1768 /* 1769 * We'll use the pre-allocated cookies for any bind that will *always* 1770 * fit (more important to be consistent, we don't want to create 1771 * additional degenerate cases). 1772 */ 1773 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1774 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1775 dma->dp_need_to_free_cookie = B_FALSE; 1776 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1777 uint_t, sinfo->si_max_pages); 1778 1779 /* 1780 * For anything larger than that, we'll go ahead and allocate the 1781 * maximum number of pages we expect to see. Hopefuly, we won't be 1782 * seeing this path in the fast path for high performance devices very 1783 * frequently. 1784 * 1785 * a ddi bind interface that allowed the driver to provide storage to 1786 * the bind interface would speed this case up. 1787 */ 1788 } else { 1789 /* convert the sleep flags */ 1790 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1791 kmflag = KM_SLEEP; 1792 } else { 1793 kmflag = KM_NOSLEEP; 1794 } 1795 1796 /* 1797 * Save away how much memory we allocated. If we're doing a 1798 * nosleep, the alloc could fail... 1799 */ 1800 dma->dp_cookie_size = sinfo->si_max_pages * 1801 sizeof (ddi_dma_cookie_t); 1802 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1803 if (dma->dp_cookies == NULL) { 1804 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1805 rootnex_clean_dmahdl(hp); 1806 return (DDI_DMA_NORESOURCES); 1807 } 1808 dma->dp_need_to_free_cookie = B_TRUE; 1809 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1810 sinfo->si_max_pages); 1811 } 1812 hp->dmai_cookie = dma->dp_cookies; 1813 1814 /* 1815 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1816 * looking at the contraints in the dma structure. It will then put some 1817 * additional state about the sgl in the dma struct (i.e. is the sgl 1818 * clean, or do we need to do some munging; how many pages need to be 1819 * copied, etc.) 1820 */ 1821 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1822 &dma->dp_sglinfo); 1823 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1824 1825 /* if we don't need a copy buffer, we don't need to sync */ 1826 if (sinfo->si_copybuf_req == 0) { 1827 hp->dmai_rflags |= DMP_NOSYNC; 1828 } 1829 1830 /* 1831 * if we don't need the copybuf and we don't need to do a partial, we 1832 * hit the fast path. All the high performance devices should be trying 1833 * to hit this path. To hit this path, a device should be able to reach 1834 * all of memory, shouldn't try to bind more than it can transfer, and 1835 * the buffer shouldn't require more cookies than the driver/device can 1836 * handle [sgllen]). 1837 */ 1838 if ((sinfo->si_copybuf_req == 0) && 1839 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1840 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1841 /* 1842 * If the driver supports FMA, insert the handle in the FMA DMA 1843 * handle cache. 1844 */ 1845 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1846 hp->dmai_error.err_cf = rootnex_dma_check; 1847 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1848 } 1849 1850 /* 1851 * copy out the first cookie and ccountp, set the cookie 1852 * pointer to the second cookie. The first cookie is passed 1853 * back on the stack. Additional cookies are accessed via 1854 * ddi_dma_nextcookie() 1855 */ 1856 *cookiep = dma->dp_cookies[0]; 1857 *ccountp = sinfo->si_sgl_size; 1858 hp->dmai_cookie++; 1859 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1860 hp->dmai_nwin = 1; 1861 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1862 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1863 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1864 dma->dp_dma.dmao_size); 1865 return (DDI_DMA_MAPPED); 1866 } 1867 1868 /* 1869 * go to the slow path, we may need to alloc more memory, create 1870 * multiple windows, and munge up a sgl to make the device happy. 1871 */ 1872 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 1873 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 1874 if (dma->dp_need_to_free_cookie) { 1875 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1876 } 1877 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1878 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 1879 return (e); 1880 } 1881 1882 /* 1883 * If the driver supports FMA, insert the handle in the FMA DMA handle 1884 * cache. 1885 */ 1886 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1887 hp->dmai_error.err_cf = rootnex_dma_check; 1888 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1889 } 1890 1891 /* if the first window uses the copy buffer, sync it for the device */ 1892 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 1893 (hp->dmai_rflags & DDI_DMA_WRITE)) { 1894 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1895 DDI_DMA_SYNC_FORDEV); 1896 } 1897 1898 /* 1899 * copy out the first cookie and ccountp, set the cookie pointer to the 1900 * second cookie. Make sure the partial flag is set/cleared correctly. 1901 * If we have a partial map (i.e. multiple windows), the number of 1902 * cookies we return is the number of cookies in the first window. 1903 */ 1904 if (e == DDI_DMA_MAPPED) { 1905 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1906 *ccountp = sinfo->si_sgl_size; 1907 } else { 1908 hp->dmai_rflags |= DDI_DMA_PARTIAL; 1909 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 1910 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 1911 } 1912 *cookiep = dma->dp_cookies[0]; 1913 hp->dmai_cookie++; 1914 1915 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1916 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 1917 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1918 dma->dp_dma.dmao_size); 1919 return (e); 1920 } 1921 1922 1923 /* 1924 * rootnex_dma_unbindhdl() 1925 * called from ddi_dma_unbind_handle() 1926 */ 1927 /*ARGSUSED*/ 1928 static int 1929 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 1930 ddi_dma_handle_t handle) 1931 { 1932 ddi_dma_impl_t *hp; 1933 rootnex_dma_t *dma; 1934 int e; 1935 1936 1937 hp = (ddi_dma_impl_t *)handle; 1938 dma = (rootnex_dma_t *)hp->dmai_private; 1939 1940 /* make sure the buffer wasn't free'd before calling unbind */ 1941 if (rootnex_unbind_verify_buffer) { 1942 e = rootnex_verify_buffer(dma); 1943 if (e != DDI_SUCCESS) { 1944 ASSERT(0); 1945 return (DDI_FAILURE); 1946 } 1947 } 1948 1949 /* sync the current window before unbinding the buffer */ 1950 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 1951 (hp->dmai_rflags & DDI_DMA_READ)) { 1952 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1953 DDI_DMA_SYNC_FORCPU); 1954 } 1955 1956 /* 1957 * If the driver supports FMA, remove the handle in the FMA DMA handle 1958 * cache. 1959 */ 1960 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 1961 if ((DEVI(rdip)->devi_fmhdl != NULL) && 1962 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 1963 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 1964 } 1965 } 1966 1967 /* 1968 * cleanup and copy buffer or window state. if we didn't use the copy 1969 * buffer or windows, there won't be much to do :-) 1970 */ 1971 rootnex_teardown_copybuf(dma); 1972 rootnex_teardown_windows(dma); 1973 1974 /* 1975 * If we had to allocate space to for the worse case sgl (it didn't 1976 * fit into our pre-allocate buffer), free that up now 1977 */ 1978 if (dma->dp_need_to_free_cookie) { 1979 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1980 } 1981 1982 /* 1983 * clean up the handle so it's ready for the next bind (i.e. if the 1984 * handle is reused). 1985 */ 1986 rootnex_clean_dmahdl(hp); 1987 1988 if (rootnex_state->r_dvma_call_list_id) 1989 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1990 1991 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1992 DTRACE_PROBE1(rootnex__unbind, uint64_t, 1993 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1994 1995 return (DDI_SUCCESS); 1996 } 1997 1998 1999 /* 2000 * rootnex_verify_buffer() 2001 * verify buffer wasn't free'd 2002 */ 2003 static int 2004 rootnex_verify_buffer(rootnex_dma_t *dma) 2005 { 2006 page_t **pplist; 2007 caddr_t vaddr; 2008 uint_t pcnt; 2009 uint_t poff; 2010 page_t *pp; 2011 char b; 2012 int i; 2013 2014 /* Figure out how many pages this buffer occupies */ 2015 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2016 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2017 } else { 2018 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2019 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2020 } 2021 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2022 2023 switch (dma->dp_dma.dmao_type) { 2024 case DMA_OTYP_PAGES: 2025 /* 2026 * for a linked list of pp's walk through them to make sure 2027 * they're locked and not free. 2028 */ 2029 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2030 for (i = 0; i < pcnt; i++) { 2031 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2032 return (DDI_FAILURE); 2033 } 2034 pp = pp->p_next; 2035 } 2036 break; 2037 2038 case DMA_OTYP_VADDR: 2039 case DMA_OTYP_BUFVADDR: 2040 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2041 /* 2042 * for an array of pp's walk through them to make sure they're 2043 * not free. It's possible that they may not be locked. 2044 */ 2045 if (pplist) { 2046 for (i = 0; i < pcnt; i++) { 2047 if (PP_ISFREE(pplist[i])) { 2048 return (DDI_FAILURE); 2049 } 2050 } 2051 2052 /* For a virtual address, try to peek at each page */ 2053 } else { 2054 if (dma->dp_sglinfo.si_asp == &kas) { 2055 for (i = 0; i < pcnt; i++) { 2056 if (ddi_peek8(NULL, vaddr, &b) == 2057 DDI_FAILURE) 2058 return (DDI_FAILURE); 2059 vaddr += MMU_PAGESIZE; 2060 } 2061 } 2062 } 2063 break; 2064 2065 default: 2066 ASSERT(0); 2067 break; 2068 } 2069 2070 return (DDI_SUCCESS); 2071 } 2072 2073 2074 /* 2075 * rootnex_clean_dmahdl() 2076 * Clean the dma handle. This should be called on a handle alloc and an 2077 * unbind handle. Set the handle state to the default settings. 2078 */ 2079 static void 2080 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2081 { 2082 rootnex_dma_t *dma; 2083 2084 2085 dma = (rootnex_dma_t *)hp->dmai_private; 2086 2087 hp->dmai_nwin = 0; 2088 dma->dp_current_cookie = 0; 2089 dma->dp_copybuf_size = 0; 2090 dma->dp_window = NULL; 2091 dma->dp_cbaddr = NULL; 2092 dma->dp_inuse = B_FALSE; 2093 dma->dp_need_to_free_cookie = B_FALSE; 2094 dma->dp_need_to_free_window = B_FALSE; 2095 dma->dp_partial_required = B_FALSE; 2096 dma->dp_trim_required = B_FALSE; 2097 dma->dp_sglinfo.si_copybuf_req = 0; 2098 #if !defined(__amd64) 2099 dma->dp_cb_remaping = B_FALSE; 2100 dma->dp_kva = NULL; 2101 #endif 2102 2103 /* FMA related initialization */ 2104 hp->dmai_fault = 0; 2105 hp->dmai_fault_check = NULL; 2106 hp->dmai_fault_notify = NULL; 2107 hp->dmai_error.err_ena = 0; 2108 hp->dmai_error.err_status = DDI_FM_OK; 2109 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2110 hp->dmai_error.err_ontrap = NULL; 2111 hp->dmai_error.err_fep = NULL; 2112 hp->dmai_error.err_cf = NULL; 2113 } 2114 2115 2116 /* 2117 * rootnex_valid_alloc_parms() 2118 * Called in ddi_dma_alloc_handle path to validate its parameters. 2119 */ 2120 static int 2121 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2122 { 2123 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2124 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2125 (attr->dma_attr_granular > MMU_PAGESIZE) || 2126 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2127 return (DDI_DMA_BADATTR); 2128 } 2129 2130 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2131 return (DDI_DMA_BADATTR); 2132 } 2133 2134 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2135 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2136 attr->dma_attr_sgllen <= 0) { 2137 return (DDI_DMA_BADATTR); 2138 } 2139 2140 /* We should be able to DMA into every byte offset in a page */ 2141 if (maxsegmentsize < MMU_PAGESIZE) { 2142 return (DDI_DMA_BADATTR); 2143 } 2144 2145 return (DDI_SUCCESS); 2146 } 2147 2148 2149 /* 2150 * rootnex_valid_bind_parms() 2151 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2152 */ 2153 /* ARGSUSED */ 2154 static int 2155 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2156 { 2157 #if !defined(__amd64) 2158 /* 2159 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2160 * we can track the offset for the obsoleted interfaces. 2161 */ 2162 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2163 return (DDI_DMA_TOOBIG); 2164 } 2165 #endif 2166 2167 return (DDI_SUCCESS); 2168 } 2169 2170 2171 /* 2172 * rootnex_get_sgl() 2173 * Called in bind fastpath to get the sgl. Most of this will be replaced 2174 * with a call to the vm layer when vm2.0 comes around... 2175 */ 2176 static void 2177 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2178 rootnex_sglinfo_t *sglinfo) 2179 { 2180 ddi_dma_atyp_t buftype; 2181 rootnex_addr_t raddr; 2182 uint64_t last_page; 2183 uint64_t offset; 2184 uint64_t addrhi; 2185 uint64_t addrlo; 2186 uint64_t maxseg; 2187 page_t **pplist; 2188 uint64_t paddr; 2189 uint32_t psize; 2190 uint32_t size; 2191 caddr_t vaddr; 2192 uint_t pcnt; 2193 page_t *pp; 2194 uint_t cnt; 2195 2196 2197 /* shortcuts */ 2198 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2199 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2200 maxseg = sglinfo->si_max_cookie_size; 2201 buftype = dmar_object->dmao_type; 2202 addrhi = sglinfo->si_max_addr; 2203 addrlo = sglinfo->si_min_addr; 2204 size = dmar_object->dmao_size; 2205 2206 pcnt = 0; 2207 cnt = 0; 2208 2209 /* 2210 * if we were passed down a linked list of pages, i.e. pointer to 2211 * page_t, use this to get our physical address and buf offset. 2212 */ 2213 if (buftype == DMA_OTYP_PAGES) { 2214 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2215 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2216 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2217 MMU_PAGEOFFSET; 2218 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2219 psize = MIN(size, (MMU_PAGESIZE - offset)); 2220 pp = pp->p_next; 2221 sglinfo->si_asp = NULL; 2222 2223 /* 2224 * We weren't passed down a linked list of pages, but if we were passed 2225 * down an array of pages, use this to get our physical address and buf 2226 * offset. 2227 */ 2228 } else if (pplist != NULL) { 2229 ASSERT((buftype == DMA_OTYP_VADDR) || 2230 (buftype == DMA_OTYP_BUFVADDR)); 2231 2232 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2233 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2234 if (sglinfo->si_asp == NULL) { 2235 sglinfo->si_asp = &kas; 2236 } 2237 2238 ASSERT(!PP_ISFREE(pplist[pcnt])); 2239 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2240 paddr += offset; 2241 psize = MIN(size, (MMU_PAGESIZE - offset)); 2242 pcnt++; 2243 2244 /* 2245 * All we have is a virtual address, we'll need to call into the VM 2246 * to get the physical address. 2247 */ 2248 } else { 2249 ASSERT((buftype == DMA_OTYP_VADDR) || 2250 (buftype == DMA_OTYP_BUFVADDR)); 2251 2252 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2253 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2254 if (sglinfo->si_asp == NULL) { 2255 sglinfo->si_asp = &kas; 2256 } 2257 2258 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2259 paddr += offset; 2260 psize = MIN(size, (MMU_PAGESIZE - offset)); 2261 vaddr += psize; 2262 } 2263 2264 #ifdef __xpv 2265 /* 2266 * If we're dom0, we're using a real device so we need to load 2267 * the cookies with MFNs instead of PFNs. 2268 */ 2269 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2270 #else 2271 raddr = paddr; 2272 #endif 2273 2274 /* 2275 * Setup the first cookie with the physical address of the page and the 2276 * size of the page (which takes into account the initial offset into 2277 * the page. 2278 */ 2279 sgl[cnt].dmac_laddress = raddr; 2280 sgl[cnt].dmac_size = psize; 2281 sgl[cnt].dmac_type = 0; 2282 2283 /* 2284 * Save away the buffer offset into the page. We'll need this later in 2285 * the copy buffer code to help figure out the page index within the 2286 * buffer and the offset into the current page. 2287 */ 2288 sglinfo->si_buf_offset = offset; 2289 2290 /* 2291 * If the DMA engine can't reach the physical address, increase how 2292 * much copy buffer we need. We always increase by pagesize so we don't 2293 * have to worry about converting offsets. Set a flag in the cookies 2294 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2295 * last cookie, go to the next cookie (since we separate each page which 2296 * uses the copy buffer in case the copy buffer is not physically 2297 * contiguous. 2298 */ 2299 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2300 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2301 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2302 if ((cnt + 1) < sglinfo->si_max_pages) { 2303 cnt++; 2304 sgl[cnt].dmac_laddress = 0; 2305 sgl[cnt].dmac_size = 0; 2306 sgl[cnt].dmac_type = 0; 2307 } 2308 } 2309 2310 /* 2311 * save this page's physical address so we can figure out if the next 2312 * page is physically contiguous. Keep decrementing size until we are 2313 * done with the buffer. 2314 */ 2315 last_page = raddr & MMU_PAGEMASK; 2316 size -= psize; 2317 2318 while (size > 0) { 2319 /* Get the size for this page (i.e. partial or full page) */ 2320 psize = MIN(size, MMU_PAGESIZE); 2321 2322 if (buftype == DMA_OTYP_PAGES) { 2323 /* get the paddr from the page_t */ 2324 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2325 paddr = pfn_to_pa(pp->p_pagenum); 2326 pp = pp->p_next; 2327 } else if (pplist != NULL) { 2328 /* index into the array of page_t's to get the paddr */ 2329 ASSERT(!PP_ISFREE(pplist[pcnt])); 2330 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2331 pcnt++; 2332 } else { 2333 /* call into the VM to get the paddr */ 2334 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2335 vaddr)); 2336 vaddr += psize; 2337 } 2338 2339 #ifdef __xpv 2340 /* 2341 * If we're dom0, we're using a real device so we need to load 2342 * the cookies with MFNs instead of PFNs. 2343 */ 2344 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2345 #else 2346 raddr = paddr; 2347 #endif 2348 2349 /* check to see if this page needs the copy buffer */ 2350 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2351 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2352 2353 /* 2354 * if there is something in the current cookie, go to 2355 * the next one. We only want one page in a cookie which 2356 * uses the copybuf since the copybuf doesn't have to 2357 * be physically contiguous. 2358 */ 2359 if (sgl[cnt].dmac_size != 0) { 2360 cnt++; 2361 } 2362 sgl[cnt].dmac_laddress = raddr; 2363 sgl[cnt].dmac_size = psize; 2364 #if defined(__amd64) 2365 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2366 #else 2367 /* 2368 * save the buf offset for 32-bit kernel. used in the 2369 * obsoleted interfaces. 2370 */ 2371 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2372 (dmar_object->dmao_size - size); 2373 #endif 2374 /* if this isn't the last cookie, go to the next one */ 2375 if ((cnt + 1) < sglinfo->si_max_pages) { 2376 cnt++; 2377 sgl[cnt].dmac_laddress = 0; 2378 sgl[cnt].dmac_size = 0; 2379 sgl[cnt].dmac_type = 0; 2380 } 2381 2382 /* 2383 * this page didn't need the copy buffer, if it's not physically 2384 * contiguous, or it would put us over a segment boundary, or it 2385 * puts us over the max cookie size, or the current sgl doesn't 2386 * have anything in it. 2387 */ 2388 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2389 !(raddr & sglinfo->si_segmask) || 2390 ((sgl[cnt].dmac_size + psize) > maxseg) || 2391 (sgl[cnt].dmac_size == 0)) { 2392 /* 2393 * if we're not already in a new cookie, go to the next 2394 * cookie. 2395 */ 2396 if (sgl[cnt].dmac_size != 0) { 2397 cnt++; 2398 } 2399 2400 /* save the cookie information */ 2401 sgl[cnt].dmac_laddress = raddr; 2402 sgl[cnt].dmac_size = psize; 2403 #if defined(__amd64) 2404 sgl[cnt].dmac_type = 0; 2405 #else 2406 /* 2407 * save the buf offset for 32-bit kernel. used in the 2408 * obsoleted interfaces. 2409 */ 2410 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2411 #endif 2412 2413 /* 2414 * this page didn't need the copy buffer, it is physically 2415 * contiguous with the last page, and it's <= the max cookie 2416 * size. 2417 */ 2418 } else { 2419 sgl[cnt].dmac_size += psize; 2420 2421 /* 2422 * if this exactly == the maximum cookie size, and 2423 * it isn't the last cookie, go to the next cookie. 2424 */ 2425 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2426 ((cnt + 1) < sglinfo->si_max_pages)) { 2427 cnt++; 2428 sgl[cnt].dmac_laddress = 0; 2429 sgl[cnt].dmac_size = 0; 2430 sgl[cnt].dmac_type = 0; 2431 } 2432 } 2433 2434 /* 2435 * save this page's physical address so we can figure out if the 2436 * next page is physically contiguous. Keep decrementing size 2437 * until we are done with the buffer. 2438 */ 2439 last_page = raddr; 2440 size -= psize; 2441 } 2442 2443 /* we're done, save away how many cookies the sgl has */ 2444 if (sgl[cnt].dmac_size == 0) { 2445 ASSERT(cnt < sglinfo->si_max_pages); 2446 sglinfo->si_sgl_size = cnt; 2447 } else { 2448 sglinfo->si_sgl_size = cnt + 1; 2449 } 2450 } 2451 2452 2453 /* 2454 * rootnex_bind_slowpath() 2455 * Call in the bind path if the calling driver can't use the sgl without 2456 * modifying it. We either need to use the copy buffer and/or we will end up 2457 * with a partial bind. 2458 */ 2459 static int 2460 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2461 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2462 { 2463 rootnex_sglinfo_t *sinfo; 2464 rootnex_window_t *window; 2465 ddi_dma_cookie_t *cookie; 2466 size_t copybuf_used; 2467 size_t dmac_size; 2468 boolean_t partial; 2469 off_t cur_offset; 2470 page_t *cur_pp; 2471 major_t mnum; 2472 int e; 2473 int i; 2474 2475 2476 sinfo = &dma->dp_sglinfo; 2477 copybuf_used = 0; 2478 partial = B_FALSE; 2479 2480 /* 2481 * If we're using the copybuf, set the copybuf state in dma struct. 2482 * Needs to be first since it sets the copy buffer size. 2483 */ 2484 if (sinfo->si_copybuf_req != 0) { 2485 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2486 if (e != DDI_SUCCESS) { 2487 return (e); 2488 } 2489 } else { 2490 dma->dp_copybuf_size = 0; 2491 } 2492 2493 /* 2494 * Figure out if we need to do a partial mapping. If so, figure out 2495 * if we need to trim the buffers when we munge the sgl. 2496 */ 2497 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2498 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2499 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2500 dma->dp_partial_required = B_TRUE; 2501 if (attr->dma_attr_granular != 1) { 2502 dma->dp_trim_required = B_TRUE; 2503 } 2504 } else { 2505 dma->dp_partial_required = B_FALSE; 2506 dma->dp_trim_required = B_FALSE; 2507 } 2508 2509 /* If we need to do a partial bind, make sure the driver supports it */ 2510 if (dma->dp_partial_required && 2511 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2512 2513 mnum = ddi_driver_major(dma->dp_dip); 2514 /* 2515 * patchable which allows us to print one warning per major 2516 * number. 2517 */ 2518 if ((rootnex_bind_warn) && 2519 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2520 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2521 cmn_err(CE_WARN, "!%s: coding error detected, the " 2522 "driver is using ddi_dma_attr(9S) incorrectly. " 2523 "There is a small risk of data corruption in " 2524 "particular with large I/Os. The driver should be " 2525 "replaced with a corrected version for proper " 2526 "system operation. To disable this warning, add " 2527 "'set rootnex:rootnex_bind_warn=0' to " 2528 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2529 } 2530 return (DDI_DMA_TOOBIG); 2531 } 2532 2533 /* 2534 * we might need multiple windows, setup state to handle them. In this 2535 * code path, we will have at least one window. 2536 */ 2537 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2538 if (e != DDI_SUCCESS) { 2539 rootnex_teardown_copybuf(dma); 2540 return (e); 2541 } 2542 2543 window = &dma->dp_window[0]; 2544 cookie = &dma->dp_cookies[0]; 2545 cur_offset = 0; 2546 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2547 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2548 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2549 } 2550 2551 /* loop though all the cookies we got back from get_sgl() */ 2552 for (i = 0; i < sinfo->si_sgl_size; i++) { 2553 /* 2554 * If we're using the copy buffer, check this cookie and setup 2555 * its associated copy buffer state. If this cookie uses the 2556 * copy buffer, make sure we sync this window during dma_sync. 2557 */ 2558 if (dma->dp_copybuf_size > 0) { 2559 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2560 cur_offset, ©buf_used, &cur_pp); 2561 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2562 window->wd_dosync = B_TRUE; 2563 } 2564 } 2565 2566 /* 2567 * save away the cookie size, since it could be modified in 2568 * the windowing code. 2569 */ 2570 dmac_size = cookie->dmac_size; 2571 2572 /* if we went over max copybuf size */ 2573 if (dma->dp_copybuf_size && 2574 (copybuf_used > dma->dp_copybuf_size)) { 2575 partial = B_TRUE; 2576 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2577 cookie, cur_offset, ©buf_used); 2578 if (e != DDI_SUCCESS) { 2579 rootnex_teardown_copybuf(dma); 2580 rootnex_teardown_windows(dma); 2581 return (e); 2582 } 2583 2584 /* 2585 * if the coookie uses the copy buffer, make sure the 2586 * new window we just moved to is set to sync. 2587 */ 2588 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2589 window->wd_dosync = B_TRUE; 2590 } 2591 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2592 dma->dp_dip); 2593 2594 /* if the cookie cnt == max sgllen, move to the next window */ 2595 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2596 partial = B_TRUE; 2597 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2598 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2599 cookie, attr, cur_offset); 2600 if (e != DDI_SUCCESS) { 2601 rootnex_teardown_copybuf(dma); 2602 rootnex_teardown_windows(dma); 2603 return (e); 2604 } 2605 2606 /* 2607 * if the coookie uses the copy buffer, make sure the 2608 * new window we just moved to is set to sync. 2609 */ 2610 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2611 window->wd_dosync = B_TRUE; 2612 } 2613 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2614 dma->dp_dip); 2615 2616 /* else if we will be over maxxfer */ 2617 } else if ((window->wd_size + dmac_size) > 2618 dma->dp_maxxfer) { 2619 partial = B_TRUE; 2620 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2621 cookie); 2622 if (e != DDI_SUCCESS) { 2623 rootnex_teardown_copybuf(dma); 2624 rootnex_teardown_windows(dma); 2625 return (e); 2626 } 2627 2628 /* 2629 * if the coookie uses the copy buffer, make sure the 2630 * new window we just moved to is set to sync. 2631 */ 2632 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2633 window->wd_dosync = B_TRUE; 2634 } 2635 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2636 dma->dp_dip); 2637 2638 /* else this cookie fits in the current window */ 2639 } else { 2640 window->wd_cookie_cnt++; 2641 window->wd_size += dmac_size; 2642 } 2643 2644 /* track our offset into the buffer, go to the next cookie */ 2645 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2646 ASSERT(cookie->dmac_size <= dmac_size); 2647 cur_offset += dmac_size; 2648 cookie++; 2649 } 2650 2651 /* if we ended up with a zero sized window in the end, clean it up */ 2652 if (window->wd_size == 0) { 2653 hp->dmai_nwin--; 2654 window--; 2655 } 2656 2657 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2658 2659 if (!partial) { 2660 return (DDI_DMA_MAPPED); 2661 } 2662 2663 ASSERT(dma->dp_partial_required); 2664 return (DDI_DMA_PARTIAL_MAP); 2665 } 2666 2667 2668 /* 2669 * rootnex_setup_copybuf() 2670 * Called in bind slowpath. Figures out if we're going to use the copy 2671 * buffer, and if we do, sets up the basic state to handle it. 2672 */ 2673 static int 2674 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2675 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2676 { 2677 rootnex_sglinfo_t *sinfo; 2678 ddi_dma_attr_t lattr; 2679 size_t max_copybuf; 2680 int cansleep; 2681 int e; 2682 #if !defined(__amd64) 2683 int vmflag; 2684 #endif 2685 2686 2687 sinfo = &dma->dp_sglinfo; 2688 2689 /* read this first so it's consistent through the routine */ 2690 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 2691 2692 /* We need to call into the rootnex on ddi_dma_sync() */ 2693 hp->dmai_rflags &= ~DMP_NOSYNC; 2694 2695 /* make sure the copybuf size <= the max size */ 2696 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2697 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2698 2699 #if !defined(__amd64) 2700 /* 2701 * if we don't have kva space to copy to/from, allocate the KVA space 2702 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2703 * the 64-bit kernel. 2704 */ 2705 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2706 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2707 2708 /* convert the sleep flags */ 2709 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2710 vmflag = VM_SLEEP; 2711 } else { 2712 vmflag = VM_NOSLEEP; 2713 } 2714 2715 /* allocate Kernel VA space that we can bcopy to/from */ 2716 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2717 vmflag); 2718 if (dma->dp_kva == NULL) { 2719 return (DDI_DMA_NORESOURCES); 2720 } 2721 } 2722 #endif 2723 2724 /* convert the sleep flags */ 2725 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2726 cansleep = 1; 2727 } else { 2728 cansleep = 0; 2729 } 2730 2731 /* 2732 * Allocate the actual copy buffer. This needs to fit within the DMA 2733 * engine limits, so we can't use kmem_alloc... We don't need 2734 * contiguous memory (sgllen) since we will be forcing windows on 2735 * sgllen anyway. 2736 */ 2737 lattr = *attr; 2738 lattr.dma_attr_align = MMU_PAGESIZE; 2739 /* 2740 * this should be < 0 to indicate no limit, but due to a bug in 2741 * the rootnex, we'll set it to the maximum positive int. 2742 */ 2743 lattr.dma_attr_sgllen = 0x7fffffff; 2744 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2745 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2746 if (e != DDI_SUCCESS) { 2747 #if !defined(__amd64) 2748 if (dma->dp_kva != NULL) { 2749 vmem_free(heap_arena, dma->dp_kva, 2750 dma->dp_copybuf_size); 2751 } 2752 #endif 2753 return (DDI_DMA_NORESOURCES); 2754 } 2755 2756 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2757 size_t, dma->dp_copybuf_size); 2758 2759 return (DDI_SUCCESS); 2760 } 2761 2762 2763 /* 2764 * rootnex_setup_windows() 2765 * Called in bind slowpath to setup the window state. We always have windows 2766 * in the slowpath. Even if the window count = 1. 2767 */ 2768 static int 2769 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2770 ddi_dma_attr_t *attr, int kmflag) 2771 { 2772 rootnex_window_t *windowp; 2773 rootnex_sglinfo_t *sinfo; 2774 size_t copy_state_size; 2775 size_t win_state_size; 2776 size_t state_available; 2777 size_t space_needed; 2778 uint_t copybuf_win; 2779 uint_t maxxfer_win; 2780 size_t space_used; 2781 uint_t sglwin; 2782 2783 2784 sinfo = &dma->dp_sglinfo; 2785 2786 dma->dp_current_win = 0; 2787 hp->dmai_nwin = 0; 2788 2789 /* If we don't need to do a partial, we only have one window */ 2790 if (!dma->dp_partial_required) { 2791 dma->dp_max_win = 1; 2792 2793 /* 2794 * we need multiple windows, need to figure out the worse case number 2795 * of windows. 2796 */ 2797 } else { 2798 /* 2799 * if we need windows because we need more copy buffer that 2800 * we allow, the worse case number of windows we could need 2801 * here would be (copybuf space required / copybuf space that 2802 * we have) plus one for remainder, and plus 2 to handle the 2803 * extra pages on the trim for the first and last pages of the 2804 * buffer (a page is the minimum window size so under the right 2805 * attr settings, you could have a window for each page). 2806 * The last page will only be hit here if the size is not a 2807 * multiple of the granularity (which theoretically shouldn't 2808 * be the case but never has been enforced, so we could have 2809 * broken things without it). 2810 */ 2811 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2812 ASSERT(dma->dp_copybuf_size > 0); 2813 copybuf_win = (sinfo->si_copybuf_req / 2814 dma->dp_copybuf_size) + 1 + 2; 2815 } else { 2816 copybuf_win = 0; 2817 } 2818 2819 /* 2820 * if we need windows because we have more cookies than the H/W 2821 * can handle, the number of windows we would need here would 2822 * be (cookie count / cookies count H/W supports) plus one for 2823 * remainder, and plus 2 to handle the extra pages on the trim 2824 * (see above comment about trim) 2825 */ 2826 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2827 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2828 + 1) + 2; 2829 } else { 2830 sglwin = 0; 2831 } 2832 2833 /* 2834 * if we need windows because we're binding more memory than the 2835 * H/W can transfer at once, the number of windows we would need 2836 * here would be (xfer count / max xfer H/W supports) plus one 2837 * for remainder, and plus 2 to handle the extra pages on the 2838 * trim (see above comment about trim) 2839 */ 2840 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 2841 maxxfer_win = (dma->dp_dma.dmao_size / 2842 dma->dp_maxxfer) + 1 + 2; 2843 } else { 2844 maxxfer_win = 0; 2845 } 2846 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 2847 ASSERT(dma->dp_max_win > 0); 2848 } 2849 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 2850 2851 /* 2852 * Get space for window and potential copy buffer state. Before we 2853 * go and allocate memory, see if we can get away with using what's 2854 * left in the pre-allocted state or the dynamically allocated sgl. 2855 */ 2856 space_used = (uintptr_t)(sinfo->si_sgl_size * 2857 sizeof (ddi_dma_cookie_t)); 2858 2859 /* if we dynamically allocated space for the cookies */ 2860 if (dma->dp_need_to_free_cookie) { 2861 /* if we have more space in the pre-allocted buffer, use it */ 2862 ASSERT(space_used <= dma->dp_cookie_size); 2863 if ((dma->dp_cookie_size - space_used) <= 2864 rootnex_state->r_prealloc_size) { 2865 state_available = rootnex_state->r_prealloc_size; 2866 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 2867 2868 /* 2869 * else, we have more free space in the dynamically allocated 2870 * buffer, i.e. the buffer wasn't worse case fragmented so we 2871 * didn't need a lot of cookies. 2872 */ 2873 } else { 2874 state_available = dma->dp_cookie_size - space_used; 2875 windowp = (rootnex_window_t *) 2876 &dma->dp_cookies[sinfo->si_sgl_size]; 2877 } 2878 2879 /* we used the pre-alloced buffer */ 2880 } else { 2881 ASSERT(space_used <= rootnex_state->r_prealloc_size); 2882 state_available = rootnex_state->r_prealloc_size - space_used; 2883 windowp = (rootnex_window_t *) 2884 &dma->dp_cookies[sinfo->si_sgl_size]; 2885 } 2886 2887 /* 2888 * figure out how much state we need to track the copy buffer. Add an 2889 * addition 8 bytes for pointer alignemnt later. 2890 */ 2891 if (dma->dp_copybuf_size > 0) { 2892 copy_state_size = sinfo->si_max_pages * 2893 sizeof (rootnex_pgmap_t); 2894 } else { 2895 copy_state_size = 0; 2896 } 2897 /* add an additional 8 bytes for pointer alignment */ 2898 space_needed = win_state_size + copy_state_size + 0x8; 2899 2900 /* if we have enough space already, use it */ 2901 if (state_available >= space_needed) { 2902 dma->dp_window = windowp; 2903 dma->dp_need_to_free_window = B_FALSE; 2904 2905 /* not enough space, need to allocate more. */ 2906 } else { 2907 dma->dp_window = kmem_alloc(space_needed, kmflag); 2908 if (dma->dp_window == NULL) { 2909 return (DDI_DMA_NORESOURCES); 2910 } 2911 dma->dp_need_to_free_window = B_TRUE; 2912 dma->dp_window_size = space_needed; 2913 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 2914 dma->dp_dip, size_t, space_needed); 2915 } 2916 2917 /* 2918 * we allocate copy buffer state and window state at the same time. 2919 * setup our copy buffer state pointers. Make sure it's aligned. 2920 */ 2921 if (dma->dp_copybuf_size > 0) { 2922 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 2923 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 2924 2925 #if !defined(__amd64) 2926 /* 2927 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 2928 * false/NULL. Should be quicker to bzero vs loop and set. 2929 */ 2930 bzero(dma->dp_pgmap, copy_state_size); 2931 #endif 2932 } else { 2933 dma->dp_pgmap = NULL; 2934 } 2935 2936 return (DDI_SUCCESS); 2937 } 2938 2939 2940 /* 2941 * rootnex_teardown_copybuf() 2942 * cleans up after rootnex_setup_copybuf() 2943 */ 2944 static void 2945 rootnex_teardown_copybuf(rootnex_dma_t *dma) 2946 { 2947 #if !defined(__amd64) 2948 int i; 2949 2950 /* 2951 * if we allocated kernel heap VMEM space, go through all the pages and 2952 * map out any of the ones that we're mapped into the kernel heap VMEM 2953 * arena. Then free the VMEM space. 2954 */ 2955 if (dma->dp_kva != NULL) { 2956 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 2957 if (dma->dp_pgmap[i].pm_mapped) { 2958 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 2959 MMU_PAGESIZE, HAT_UNLOAD); 2960 dma->dp_pgmap[i].pm_mapped = B_FALSE; 2961 } 2962 } 2963 2964 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 2965 } 2966 2967 #endif 2968 2969 /* if we allocated a copy buffer, free it */ 2970 if (dma->dp_cbaddr != NULL) { 2971 i_ddi_mem_free(dma->dp_cbaddr, NULL); 2972 } 2973 } 2974 2975 2976 /* 2977 * rootnex_teardown_windows() 2978 * cleans up after rootnex_setup_windows() 2979 */ 2980 static void 2981 rootnex_teardown_windows(rootnex_dma_t *dma) 2982 { 2983 /* 2984 * if we had to allocate window state on the last bind (because we 2985 * didn't have enough pre-allocated space in the handle), free it. 2986 */ 2987 if (dma->dp_need_to_free_window) { 2988 kmem_free(dma->dp_window, dma->dp_window_size); 2989 } 2990 } 2991 2992 2993 /* 2994 * rootnex_init_win() 2995 * Called in bind slow path during creation of a new window. Initializes 2996 * window state to default values. 2997 */ 2998 /*ARGSUSED*/ 2999 static void 3000 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3001 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3002 { 3003 hp->dmai_nwin++; 3004 window->wd_dosync = B_FALSE; 3005 window->wd_offset = cur_offset; 3006 window->wd_size = 0; 3007 window->wd_first_cookie = cookie; 3008 window->wd_cookie_cnt = 0; 3009 window->wd_trim.tr_trim_first = B_FALSE; 3010 window->wd_trim.tr_trim_last = B_FALSE; 3011 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3012 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3013 #if !defined(__amd64) 3014 window->wd_remap_copybuf = dma->dp_cb_remaping; 3015 #endif 3016 } 3017 3018 3019 /* 3020 * rootnex_setup_cookie() 3021 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3022 * the sgl uses the copy buffer, we need to go through each cookie, figure 3023 * out if it uses the copy buffer, and if it does, save away everything we'll 3024 * need during sync. 3025 */ 3026 static void 3027 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3028 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3029 page_t **cur_pp) 3030 { 3031 boolean_t copybuf_sz_power_2; 3032 rootnex_sglinfo_t *sinfo; 3033 paddr_t paddr; 3034 uint_t pidx; 3035 uint_t pcnt; 3036 off_t poff; 3037 #if defined(__amd64) 3038 pfn_t pfn; 3039 #else 3040 page_t **pplist; 3041 #endif 3042 3043 sinfo = &dma->dp_sglinfo; 3044 3045 /* 3046 * Calculate the page index relative to the start of the buffer. The 3047 * index to the current page for our buffer is the offset into the 3048 * first page of the buffer plus our current offset into the buffer 3049 * itself, shifted of course... 3050 */ 3051 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3052 ASSERT(pidx < sinfo->si_max_pages); 3053 3054 /* if this cookie uses the copy buffer */ 3055 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3056 /* 3057 * NOTE: we know that since this cookie uses the copy buffer, it 3058 * is <= MMU_PAGESIZE. 3059 */ 3060 3061 /* 3062 * get the offset into the page. For the 64-bit kernel, get the 3063 * pfn which we'll use with seg kpm. 3064 */ 3065 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3066 #if defined(__amd64) 3067 /* mfn_to_pfn() is a NOP on i86pc */ 3068 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3069 #endif /* __amd64 */ 3070 3071 /* figure out if the copybuf size is a power of 2 */ 3072 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3073 copybuf_sz_power_2 = B_FALSE; 3074 } else { 3075 copybuf_sz_power_2 = B_TRUE; 3076 } 3077 3078 /* This page uses the copy buffer */ 3079 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3080 3081 /* 3082 * save the copy buffer KVA that we'll use with this page. 3083 * if we still fit within the copybuf, it's a simple add. 3084 * otherwise, we need to wrap over using & or % accordingly. 3085 */ 3086 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3087 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3088 *copybuf_used; 3089 } else { 3090 if (copybuf_sz_power_2) { 3091 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3092 (uintptr_t)dma->dp_cbaddr + 3093 (*copybuf_used & 3094 (dma->dp_copybuf_size - 1))); 3095 } else { 3096 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3097 (uintptr_t)dma->dp_cbaddr + 3098 (*copybuf_used % dma->dp_copybuf_size)); 3099 } 3100 } 3101 3102 /* 3103 * over write the cookie physical address with the address of 3104 * the physical address of the copy buffer page that we will 3105 * use. 3106 */ 3107 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3108 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3109 3110 #ifdef __xpv 3111 /* 3112 * If we're dom0, we're using a real device so we need to load 3113 * the cookies with MAs instead of PAs. 3114 */ 3115 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3116 #else 3117 cookie->dmac_laddress = paddr; 3118 #endif 3119 3120 /* if we have a kernel VA, it's easy, just save that address */ 3121 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3122 (sinfo->si_asp == &kas)) { 3123 /* 3124 * save away the page aligned virtual address of the 3125 * driver buffer. Offsets are handled in the sync code. 3126 */ 3127 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3128 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3129 & MMU_PAGEMASK); 3130 #if !defined(__amd64) 3131 /* 3132 * we didn't need to, and will never need to map this 3133 * page. 3134 */ 3135 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3136 #endif 3137 3138 /* we don't have a kernel VA. We need one for the bcopy. */ 3139 } else { 3140 #if defined(__amd64) 3141 /* 3142 * for the 64-bit kernel, it's easy. We use seg kpm to 3143 * get a Kernel VA for the corresponding pfn. 3144 */ 3145 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3146 #else 3147 /* 3148 * for the 32-bit kernel, this is a pain. First we'll 3149 * save away the page_t or user VA for this page. This 3150 * is needed in rootnex_dma_win() when we switch to a 3151 * new window which requires us to re-map the copy 3152 * buffer. 3153 */ 3154 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3155 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3156 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3157 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3158 } else if (pplist != NULL) { 3159 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3160 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3161 } else { 3162 dma->dp_pgmap[pidx].pm_pp = NULL; 3163 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3164 (((uintptr_t) 3165 dmar_object->dmao_obj.virt_obj.v_addr + 3166 cur_offset) & MMU_PAGEMASK); 3167 } 3168 3169 /* 3170 * save away the page aligned virtual address which was 3171 * allocated from the kernel heap arena (taking into 3172 * account if we need more copy buffer than we alloced 3173 * and use multiple windows to handle this, i.e. &,%). 3174 * NOTE: there isn't and physical memory backing up this 3175 * virtual address space currently. 3176 */ 3177 if ((*copybuf_used + MMU_PAGESIZE) <= 3178 dma->dp_copybuf_size) { 3179 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3180 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3181 MMU_PAGEMASK); 3182 } else { 3183 if (copybuf_sz_power_2) { 3184 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3185 (((uintptr_t)dma->dp_kva + 3186 (*copybuf_used & 3187 (dma->dp_copybuf_size - 1))) & 3188 MMU_PAGEMASK); 3189 } else { 3190 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3191 (((uintptr_t)dma->dp_kva + 3192 (*copybuf_used % 3193 dma->dp_copybuf_size)) & 3194 MMU_PAGEMASK); 3195 } 3196 } 3197 3198 /* 3199 * if we haven't used up the available copy buffer yet, 3200 * map the kva to the physical page. 3201 */ 3202 if (!dma->dp_cb_remaping && ((*copybuf_used + 3203 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3204 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3205 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3206 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3207 dma->dp_pgmap[pidx].pm_kaddr); 3208 } else { 3209 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3210 sinfo->si_asp, 3211 dma->dp_pgmap[pidx].pm_kaddr); 3212 } 3213 3214 /* 3215 * we've used up the available copy buffer, this page 3216 * will have to be mapped during rootnex_dma_win() when 3217 * we switch to a new window which requires a re-map 3218 * the copy buffer. (32-bit kernel only) 3219 */ 3220 } else { 3221 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3222 } 3223 #endif 3224 /* go to the next page_t */ 3225 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3226 *cur_pp = (*cur_pp)->p_next; 3227 } 3228 } 3229 3230 /* add to the copy buffer count */ 3231 *copybuf_used += MMU_PAGESIZE; 3232 3233 /* 3234 * This cookie doesn't use the copy buffer. Walk through the pages this 3235 * cookie occupies to reflect this. 3236 */ 3237 } else { 3238 /* 3239 * figure out how many pages the cookie occupies. We need to 3240 * use the original page offset of the buffer and the cookies 3241 * offset in the buffer to do this. 3242 */ 3243 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3244 pcnt = mmu_btopr(cookie->dmac_size + poff); 3245 3246 while (pcnt > 0) { 3247 #if !defined(__amd64) 3248 /* 3249 * the 32-bit kernel doesn't have seg kpm, so we need 3250 * to map in the driver buffer (if it didn't come down 3251 * with a kernel VA) on the fly. Since this page doesn't 3252 * use the copy buffer, it's not, or will it ever, have 3253 * to be mapped in. 3254 */ 3255 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3256 #endif 3257 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3258 3259 /* 3260 * we need to update pidx and cur_pp or we'll loose 3261 * track of where we are. 3262 */ 3263 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3264 *cur_pp = (*cur_pp)->p_next; 3265 } 3266 pidx++; 3267 pcnt--; 3268 } 3269 } 3270 } 3271 3272 3273 /* 3274 * rootnex_sgllen_window_boundary() 3275 * Called in the bind slow path when the next cookie causes us to exceed (in 3276 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3277 * length supported by the DMA H/W. 3278 */ 3279 static int 3280 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3281 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3282 off_t cur_offset) 3283 { 3284 off_t new_offset; 3285 size_t trim_sz; 3286 off_t coffset; 3287 3288 3289 /* 3290 * if we know we'll never have to trim, it's pretty easy. Just move to 3291 * the next window and init it. We're done. 3292 */ 3293 if (!dma->dp_trim_required) { 3294 (*windowp)++; 3295 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3296 (*windowp)->wd_cookie_cnt++; 3297 (*windowp)->wd_size = cookie->dmac_size; 3298 return (DDI_SUCCESS); 3299 } 3300 3301 /* figure out how much we need to trim from the window */ 3302 ASSERT(attr->dma_attr_granular != 0); 3303 if (dma->dp_granularity_power_2) { 3304 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3305 } else { 3306 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3307 } 3308 3309 /* The window's a whole multiple of granularity. We're done */ 3310 if (trim_sz == 0) { 3311 (*windowp)++; 3312 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3313 (*windowp)->wd_cookie_cnt++; 3314 (*windowp)->wd_size = cookie->dmac_size; 3315 return (DDI_SUCCESS); 3316 } 3317 3318 /* 3319 * The window's not a whole multiple of granularity, since we know this 3320 * is due to the sgllen, we need to go back to the last cookie and trim 3321 * that one, add the left over part of the old cookie into the new 3322 * window, and then add in the new cookie into the new window. 3323 */ 3324 3325 /* 3326 * make sure the driver isn't making us do something bad... Trimming and 3327 * sgllen == 1 don't go together. 3328 */ 3329 if (attr->dma_attr_sgllen == 1) { 3330 return (DDI_DMA_NOMAPPING); 3331 } 3332 3333 /* 3334 * first, setup the current window to account for the trim. Need to go 3335 * back to the last cookie for this. 3336 */ 3337 cookie--; 3338 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3339 (*windowp)->wd_trim.tr_last_cookie = cookie; 3340 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3341 ASSERT(cookie->dmac_size > trim_sz); 3342 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3343 (*windowp)->wd_size -= trim_sz; 3344 3345 /* save the buffer offsets for the next window */ 3346 coffset = cookie->dmac_size - trim_sz; 3347 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3348 3349 /* 3350 * set this now in case this is the first window. all other cases are 3351 * set in dma_win() 3352 */ 3353 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3354 3355 /* 3356 * initialize the next window using what's left over in the previous 3357 * cookie. 3358 */ 3359 (*windowp)++; 3360 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3361 (*windowp)->wd_cookie_cnt++; 3362 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3363 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3364 (*windowp)->wd_trim.tr_first_size = trim_sz; 3365 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3366 (*windowp)->wd_dosync = B_TRUE; 3367 } 3368 3369 /* 3370 * now go back to the current cookie and add it to the new window. set 3371 * the new window size to the what was left over from the previous 3372 * cookie and what's in the current cookie. 3373 */ 3374 cookie++; 3375 (*windowp)->wd_cookie_cnt++; 3376 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3377 3378 /* 3379 * trim plus the next cookie could put us over maxxfer (a cookie can be 3380 * a max size of maxxfer). Handle that case. 3381 */ 3382 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3383 /* 3384 * maxxfer is already a whole multiple of granularity, and this 3385 * trim will be <= the previous trim (since a cookie can't be 3386 * larger than maxxfer). Make things simple here. 3387 */ 3388 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3389 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3390 (*windowp)->wd_trim.tr_last_cookie = cookie; 3391 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3392 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3393 (*windowp)->wd_size -= trim_sz; 3394 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3395 3396 /* save the buffer offsets for the next window */ 3397 coffset = cookie->dmac_size - trim_sz; 3398 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3399 3400 /* setup the next window */ 3401 (*windowp)++; 3402 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3403 (*windowp)->wd_cookie_cnt++; 3404 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3405 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3406 coffset; 3407 (*windowp)->wd_trim.tr_first_size = trim_sz; 3408 } 3409 3410 return (DDI_SUCCESS); 3411 } 3412 3413 3414 /* 3415 * rootnex_copybuf_window_boundary() 3416 * Called in bind slowpath when we get to a window boundary because we used 3417 * up all the copy buffer that we have. 3418 */ 3419 static int 3420 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3421 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3422 size_t *copybuf_used) 3423 { 3424 rootnex_sglinfo_t *sinfo; 3425 off_t new_offset; 3426 size_t trim_sz; 3427 paddr_t paddr; 3428 off_t coffset; 3429 uint_t pidx; 3430 off_t poff; 3431 3432 3433 sinfo = &dma->dp_sglinfo; 3434 3435 /* 3436 * the copy buffer should be a whole multiple of page size. We know that 3437 * this cookie is <= MMU_PAGESIZE. 3438 */ 3439 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3440 3441 /* 3442 * from now on, all new windows in this bind need to be re-mapped during 3443 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3444 * space... 3445 */ 3446 #if !defined(__amd64) 3447 dma->dp_cb_remaping = B_TRUE; 3448 #endif 3449 3450 /* reset copybuf used */ 3451 *copybuf_used = 0; 3452 3453 /* 3454 * if we don't have to trim (since granularity is set to 1), go to the 3455 * next window and add the current cookie to it. We know the current 3456 * cookie uses the copy buffer since we're in this code path. 3457 */ 3458 if (!dma->dp_trim_required) { 3459 (*windowp)++; 3460 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3461 3462 /* Add this cookie to the new window */ 3463 (*windowp)->wd_cookie_cnt++; 3464 (*windowp)->wd_size += cookie->dmac_size; 3465 *copybuf_used += MMU_PAGESIZE; 3466 return (DDI_SUCCESS); 3467 } 3468 3469 /* 3470 * *** may need to trim, figure it out. 3471 */ 3472 3473 /* figure out how much we need to trim from the window */ 3474 if (dma->dp_granularity_power_2) { 3475 trim_sz = (*windowp)->wd_size & 3476 (hp->dmai_attr.dma_attr_granular - 1); 3477 } else { 3478 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3479 } 3480 3481 /* 3482 * if the window's a whole multiple of granularity, go to the next 3483 * window, init it, then add in the current cookie. We know the current 3484 * cookie uses the copy buffer since we're in this code path. 3485 */ 3486 if (trim_sz == 0) { 3487 (*windowp)++; 3488 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3489 3490 /* Add this cookie to the new window */ 3491 (*windowp)->wd_cookie_cnt++; 3492 (*windowp)->wd_size += cookie->dmac_size; 3493 *copybuf_used += MMU_PAGESIZE; 3494 return (DDI_SUCCESS); 3495 } 3496 3497 /* 3498 * *** We figured it out, we definitly need to trim 3499 */ 3500 3501 /* 3502 * make sure the driver isn't making us do something bad... 3503 * Trimming and sgllen == 1 don't go together. 3504 */ 3505 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3506 return (DDI_DMA_NOMAPPING); 3507 } 3508 3509 /* 3510 * first, setup the current window to account for the trim. Need to go 3511 * back to the last cookie for this. Some of the last cookie will be in 3512 * the current window, and some of the last cookie will be in the new 3513 * window. All of the current cookie will be in the new window. 3514 */ 3515 cookie--; 3516 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3517 (*windowp)->wd_trim.tr_last_cookie = cookie; 3518 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3519 ASSERT(cookie->dmac_size > trim_sz); 3520 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3521 (*windowp)->wd_size -= trim_sz; 3522 3523 /* 3524 * we're trimming the last cookie (not the current cookie). So that 3525 * last cookie may have or may not have been using the copy buffer ( 3526 * we know the cookie passed in uses the copy buffer since we're in 3527 * this code path). 3528 * 3529 * If the last cookie doesn't use the copy buffer, nothing special to 3530 * do. However, if it does uses the copy buffer, it will be both the 3531 * last page in the current window and the first page in the next 3532 * window. Since we are reusing the copy buffer (and KVA space on the 3533 * 32-bit kernel), this page will use the end of the copy buffer in the 3534 * current window, and the start of the copy buffer in the next window. 3535 * Track that info... The cookie physical address was already set to 3536 * the copy buffer physical address in setup_cookie.. 3537 */ 3538 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3539 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3540 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3541 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3542 (*windowp)->wd_trim.tr_last_pidx = pidx; 3543 (*windowp)->wd_trim.tr_last_cbaddr = 3544 dma->dp_pgmap[pidx].pm_cbaddr; 3545 #if !defined(__amd64) 3546 (*windowp)->wd_trim.tr_last_kaddr = 3547 dma->dp_pgmap[pidx].pm_kaddr; 3548 #endif 3549 } 3550 3551 /* save the buffer offsets for the next window */ 3552 coffset = cookie->dmac_size - trim_sz; 3553 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3554 3555 /* 3556 * set this now in case this is the first window. all other cases are 3557 * set in dma_win() 3558 */ 3559 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3560 3561 /* 3562 * initialize the next window using what's left over in the previous 3563 * cookie. 3564 */ 3565 (*windowp)++; 3566 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3567 (*windowp)->wd_cookie_cnt++; 3568 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3569 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3570 (*windowp)->wd_trim.tr_first_size = trim_sz; 3571 3572 /* 3573 * again, we're tracking if the last cookie uses the copy buffer. 3574 * read the comment above for more info on why we need to track 3575 * additional state. 3576 * 3577 * For the first cookie in the new window, we need reset the physical 3578 * address to DMA into to the start of the copy buffer plus any 3579 * initial page offset which may be present. 3580 */ 3581 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3582 (*windowp)->wd_dosync = B_TRUE; 3583 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3584 (*windowp)->wd_trim.tr_first_pidx = pidx; 3585 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3586 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3587 3588 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3589 poff; 3590 #ifdef __xpv 3591 /* 3592 * If we're dom0, we're using a real device so we need to load 3593 * the cookies with MAs instead of PAs. 3594 */ 3595 (*windowp)->wd_trim.tr_first_paddr = 3596 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3597 #else 3598 (*windowp)->wd_trim.tr_first_paddr = paddr; 3599 #endif 3600 3601 #if !defined(__amd64) 3602 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3603 #endif 3604 /* account for the cookie copybuf usage in the new window */ 3605 *copybuf_used += MMU_PAGESIZE; 3606 3607 /* 3608 * every piece of code has to have a hack, and here is this 3609 * ones :-) 3610 * 3611 * There is a complex interaction between setup_cookie and the 3612 * copybuf window boundary. The complexity had to be in either 3613 * the maxxfer window, or the copybuf window, and I chose the 3614 * copybuf code. 3615 * 3616 * So in this code path, we have taken the last cookie, 3617 * virtually broken it in half due to the trim, and it happens 3618 * to use the copybuf which further complicates life. At the 3619 * same time, we have already setup the current cookie, which 3620 * is now wrong. More background info: the current cookie uses 3621 * the copybuf, so it is only a page long max. So we need to 3622 * fix the current cookies copy buffer address, physical 3623 * address, and kva for the 32-bit kernel. We due this by 3624 * bumping them by page size (of course, we can't due this on 3625 * the physical address since the copy buffer may not be 3626 * physically contiguous). 3627 */ 3628 cookie++; 3629 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3630 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3631 3632 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3633 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3634 #ifdef __xpv 3635 /* 3636 * If we're dom0, we're using a real device so we need to load 3637 * the cookies with MAs instead of PAs. 3638 */ 3639 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3640 #else 3641 cookie->dmac_laddress = paddr; 3642 #endif 3643 3644 #if !defined(__amd64) 3645 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3646 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3647 #endif 3648 } else { 3649 /* go back to the current cookie */ 3650 cookie++; 3651 } 3652 3653 /* 3654 * add the current cookie to the new window. set the new window size to 3655 * the what was left over from the previous cookie and what's in the 3656 * current cookie. 3657 */ 3658 (*windowp)->wd_cookie_cnt++; 3659 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3660 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3661 3662 /* 3663 * we know that the cookie passed in always uses the copy buffer. We 3664 * wouldn't be here if it didn't. 3665 */ 3666 *copybuf_used += MMU_PAGESIZE; 3667 3668 return (DDI_SUCCESS); 3669 } 3670 3671 3672 /* 3673 * rootnex_maxxfer_window_boundary() 3674 * Called in bind slowpath when we get to a window boundary because we will 3675 * go over maxxfer. 3676 */ 3677 static int 3678 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3679 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3680 { 3681 size_t dmac_size; 3682 off_t new_offset; 3683 size_t trim_sz; 3684 off_t coffset; 3685 3686 3687 /* 3688 * calculate how much we have to trim off of the current cookie to equal 3689 * maxxfer. We don't have to account for granularity here since our 3690 * maxxfer already takes that into account. 3691 */ 3692 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3693 ASSERT(trim_sz <= cookie->dmac_size); 3694 ASSERT(trim_sz <= dma->dp_maxxfer); 3695 3696 /* save cookie size since we need it later and we might change it */ 3697 dmac_size = cookie->dmac_size; 3698 3699 /* 3700 * if we're not trimming the entire cookie, setup the current window to 3701 * account for the trim. 3702 */ 3703 if (trim_sz < cookie->dmac_size) { 3704 (*windowp)->wd_cookie_cnt++; 3705 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3706 (*windowp)->wd_trim.tr_last_cookie = cookie; 3707 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3708 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3709 (*windowp)->wd_size = dma->dp_maxxfer; 3710 3711 /* 3712 * set the adjusted cookie size now in case this is the first 3713 * window. All other windows are taken care of in get win 3714 */ 3715 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3716 } 3717 3718 /* 3719 * coffset is the current offset within the cookie, new_offset is the 3720 * current offset with the entire buffer. 3721 */ 3722 coffset = dmac_size - trim_sz; 3723 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3724 3725 /* initialize the next window */ 3726 (*windowp)++; 3727 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3728 (*windowp)->wd_cookie_cnt++; 3729 (*windowp)->wd_size = trim_sz; 3730 if (trim_sz < dmac_size) { 3731 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3732 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3733 coffset; 3734 (*windowp)->wd_trim.tr_first_size = trim_sz; 3735 } 3736 3737 return (DDI_SUCCESS); 3738 } 3739 3740 3741 /* 3742 * rootnex_dma_sync() 3743 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3744 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3745 * is set, ddi_dma_sync() returns immediately passing back success. 3746 */ 3747 /*ARGSUSED*/ 3748 static int 3749 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3750 off_t off, size_t len, uint_t cache_flags) 3751 { 3752 rootnex_sglinfo_t *sinfo; 3753 rootnex_pgmap_t *cbpage; 3754 rootnex_window_t *win; 3755 ddi_dma_impl_t *hp; 3756 rootnex_dma_t *dma; 3757 caddr_t fromaddr; 3758 caddr_t toaddr; 3759 uint_t psize; 3760 off_t offset; 3761 uint_t pidx; 3762 size_t size; 3763 off_t poff; 3764 int e; 3765 3766 3767 hp = (ddi_dma_impl_t *)handle; 3768 dma = (rootnex_dma_t *)hp->dmai_private; 3769 sinfo = &dma->dp_sglinfo; 3770 3771 /* 3772 * if we don't have any windows, we don't need to sync. A copybuf 3773 * will cause us to have at least one window. 3774 */ 3775 if (dma->dp_window == NULL) { 3776 return (DDI_SUCCESS); 3777 } 3778 3779 /* This window may not need to be sync'd */ 3780 win = &dma->dp_window[dma->dp_current_win]; 3781 if (!win->wd_dosync) { 3782 return (DDI_SUCCESS); 3783 } 3784 3785 /* handle off and len special cases */ 3786 if ((off == 0) || (rootnex_sync_ignore_params)) { 3787 offset = win->wd_offset; 3788 } else { 3789 offset = off; 3790 } 3791 if ((len == 0) || (rootnex_sync_ignore_params)) { 3792 size = win->wd_size; 3793 } else { 3794 size = len; 3795 } 3796 3797 /* check the sync args to make sure they make a little sense */ 3798 if (rootnex_sync_check_parms) { 3799 e = rootnex_valid_sync_parms(hp, win, offset, size, 3800 cache_flags); 3801 if (e != DDI_SUCCESS) { 3802 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3803 return (DDI_FAILURE); 3804 } 3805 } 3806 3807 /* 3808 * special case the first page to handle the offset into the page. The 3809 * offset to the current page for our buffer is the offset into the 3810 * first page of the buffer plus our current offset into the buffer 3811 * itself, masked of course. 3812 */ 3813 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3814 psize = MIN((MMU_PAGESIZE - poff), size); 3815 3816 /* go through all the pages that we want to sync */ 3817 while (size > 0) { 3818 /* 3819 * Calculate the page index relative to the start of the buffer. 3820 * The index to the current page for our buffer is the offset 3821 * into the first page of the buffer plus our current offset 3822 * into the buffer itself, shifted of course... 3823 */ 3824 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3825 ASSERT(pidx < sinfo->si_max_pages); 3826 3827 /* 3828 * if this page uses the copy buffer, we need to sync it, 3829 * otherwise, go on to the next page. 3830 */ 3831 cbpage = &dma->dp_pgmap[pidx]; 3832 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3833 (cbpage->pm_uses_copybuf == B_FALSE)); 3834 if (cbpage->pm_uses_copybuf) { 3835 /* cbaddr and kaddr should be page aligned */ 3836 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3837 MMU_PAGEOFFSET) == 0); 3838 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3839 MMU_PAGEOFFSET) == 0); 3840 3841 /* 3842 * if we're copying for the device, we are going to 3843 * copy from the drivers buffer and to the rootnex 3844 * allocated copy buffer. 3845 */ 3846 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3847 fromaddr = cbpage->pm_kaddr + poff; 3848 toaddr = cbpage->pm_cbaddr + poff; 3849 DTRACE_PROBE2(rootnex__sync__dev, 3850 dev_info_t *, dma->dp_dip, size_t, psize); 3851 3852 /* 3853 * if we're copying for the cpu/kernel, we are going to 3854 * copy from the rootnex allocated copy buffer to the 3855 * drivers buffer. 3856 */ 3857 } else { 3858 fromaddr = cbpage->pm_cbaddr + poff; 3859 toaddr = cbpage->pm_kaddr + poff; 3860 DTRACE_PROBE2(rootnex__sync__cpu, 3861 dev_info_t *, dma->dp_dip, size_t, psize); 3862 } 3863 3864 bcopy(fromaddr, toaddr, psize); 3865 } 3866 3867 /* 3868 * decrement size until we're done, update our offset into the 3869 * buffer, and get the next page size. 3870 */ 3871 size -= psize; 3872 offset += psize; 3873 psize = MIN(MMU_PAGESIZE, size); 3874 3875 /* page offset is zero for the rest of this loop */ 3876 poff = 0; 3877 } 3878 3879 return (DDI_SUCCESS); 3880 } 3881 3882 3883 /* 3884 * rootnex_valid_sync_parms() 3885 * checks the parameters passed to sync to verify they are correct. 3886 */ 3887 static int 3888 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 3889 off_t offset, size_t size, uint_t cache_flags) 3890 { 3891 off_t woffset; 3892 3893 3894 /* 3895 * the first part of the test to make sure the offset passed in is 3896 * within the window. 3897 */ 3898 if (offset < win->wd_offset) { 3899 return (DDI_FAILURE); 3900 } 3901 3902 /* 3903 * second and last part of the test to make sure the offset and length 3904 * passed in is within the window. 3905 */ 3906 woffset = offset - win->wd_offset; 3907 if ((woffset + size) > win->wd_size) { 3908 return (DDI_FAILURE); 3909 } 3910 3911 /* 3912 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 3913 * be set too. 3914 */ 3915 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 3916 (hp->dmai_rflags & DDI_DMA_WRITE)) { 3917 return (DDI_SUCCESS); 3918 } 3919 3920 /* 3921 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 3922 * should be set. Also DDI_DMA_READ should be set in the flags. 3923 */ 3924 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 3925 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 3926 (hp->dmai_rflags & DDI_DMA_READ)) { 3927 return (DDI_SUCCESS); 3928 } 3929 3930 return (DDI_FAILURE); 3931 } 3932 3933 3934 /* 3935 * rootnex_dma_win() 3936 * called from ddi_dma_getwin() 3937 */ 3938 /*ARGSUSED*/ 3939 static int 3940 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3941 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 3942 uint_t *ccountp) 3943 { 3944 rootnex_window_t *window; 3945 rootnex_trim_t *trim; 3946 ddi_dma_impl_t *hp; 3947 rootnex_dma_t *dma; 3948 #if !defined(__amd64) 3949 rootnex_sglinfo_t *sinfo; 3950 rootnex_pgmap_t *pmap; 3951 uint_t pidx; 3952 uint_t pcnt; 3953 off_t poff; 3954 int i; 3955 #endif 3956 3957 3958 hp = (ddi_dma_impl_t *)handle; 3959 dma = (rootnex_dma_t *)hp->dmai_private; 3960 #if !defined(__amd64) 3961 sinfo = &dma->dp_sglinfo; 3962 #endif 3963 3964 /* If we try and get a window which doesn't exist, return failure */ 3965 if (win >= hp->dmai_nwin) { 3966 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3967 return (DDI_FAILURE); 3968 } 3969 3970 /* 3971 * if we don't have any windows, and they're asking for the first 3972 * window, setup the cookie pointer to the first cookie in the bind. 3973 * setup our return values, then increment the cookie since we return 3974 * the first cookie on the stack. 3975 */ 3976 if (dma->dp_window == NULL) { 3977 if (win != 0) { 3978 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3979 return (DDI_FAILURE); 3980 } 3981 hp->dmai_cookie = dma->dp_cookies; 3982 *offp = 0; 3983 *lenp = dma->dp_dma.dmao_size; 3984 *ccountp = dma->dp_sglinfo.si_sgl_size; 3985 *cookiep = hp->dmai_cookie[0]; 3986 hp->dmai_cookie++; 3987 return (DDI_SUCCESS); 3988 } 3989 3990 /* sync the old window before moving on to the new one */ 3991 window = &dma->dp_window[dma->dp_current_win]; 3992 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 3993 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 3994 DDI_DMA_SYNC_FORCPU); 3995 } 3996 3997 #if !defined(__amd64) 3998 /* 3999 * before we move to the next window, if we need to re-map, unmap all 4000 * the pages in this window. 4001 */ 4002 if (dma->dp_cb_remaping) { 4003 /* 4004 * If we switch to this window again, we'll need to map in 4005 * on the fly next time. 4006 */ 4007 window->wd_remap_copybuf = B_TRUE; 4008 4009 /* 4010 * calculate the page index into the buffer where this window 4011 * starts, and the number of pages this window takes up. 4012 */ 4013 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4014 MMU_PAGESHIFT; 4015 poff = (sinfo->si_buf_offset + window->wd_offset) & 4016 MMU_PAGEOFFSET; 4017 pcnt = mmu_btopr(window->wd_size + poff); 4018 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4019 4020 /* unmap pages which are currently mapped in this window */ 4021 for (i = 0; i < pcnt; i++) { 4022 if (dma->dp_pgmap[pidx].pm_mapped) { 4023 hat_unload(kas.a_hat, 4024 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4025 HAT_UNLOAD); 4026 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4027 } 4028 pidx++; 4029 } 4030 } 4031 #endif 4032 4033 /* 4034 * Move to the new window. 4035 * NOTE: current_win must be set for sync to work right 4036 */ 4037 dma->dp_current_win = win; 4038 window = &dma->dp_window[win]; 4039 4040 /* if needed, adjust the first and/or last cookies for trim */ 4041 trim = &window->wd_trim; 4042 if (trim->tr_trim_first) { 4043 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4044 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4045 #if !defined(__amd64) 4046 window->wd_first_cookie->dmac_type = 4047 (window->wd_first_cookie->dmac_type & 4048 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4049 #endif 4050 if (trim->tr_first_copybuf_win) { 4051 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4052 trim->tr_first_cbaddr; 4053 #if !defined(__amd64) 4054 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4055 trim->tr_first_kaddr; 4056 #endif 4057 } 4058 } 4059 if (trim->tr_trim_last) { 4060 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4061 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4062 if (trim->tr_last_copybuf_win) { 4063 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4064 trim->tr_last_cbaddr; 4065 #if !defined(__amd64) 4066 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4067 trim->tr_last_kaddr; 4068 #endif 4069 } 4070 } 4071 4072 /* 4073 * setup the cookie pointer to the first cookie in the window. setup 4074 * our return values, then increment the cookie since we return the 4075 * first cookie on the stack. 4076 */ 4077 hp->dmai_cookie = window->wd_first_cookie; 4078 *offp = window->wd_offset; 4079 *lenp = window->wd_size; 4080 *ccountp = window->wd_cookie_cnt; 4081 *cookiep = hp->dmai_cookie[0]; 4082 hp->dmai_cookie++; 4083 4084 #if !defined(__amd64) 4085 /* re-map copybuf if required for this window */ 4086 if (dma->dp_cb_remaping) { 4087 /* 4088 * calculate the page index into the buffer where this 4089 * window starts. 4090 */ 4091 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4092 MMU_PAGESHIFT; 4093 ASSERT(pidx < sinfo->si_max_pages); 4094 4095 /* 4096 * the first page can get unmapped if it's shared with the 4097 * previous window. Even if the rest of this window is already 4098 * mapped in, we need to still check this one. 4099 */ 4100 pmap = &dma->dp_pgmap[pidx]; 4101 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4102 if (pmap->pm_pp != NULL) { 4103 pmap->pm_mapped = B_TRUE; 4104 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4105 } else if (pmap->pm_vaddr != NULL) { 4106 pmap->pm_mapped = B_TRUE; 4107 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4108 pmap->pm_kaddr); 4109 } 4110 } 4111 pidx++; 4112 4113 /* map in the rest of the pages if required */ 4114 if (window->wd_remap_copybuf) { 4115 window->wd_remap_copybuf = B_FALSE; 4116 4117 /* figure out many pages this window takes up */ 4118 poff = (sinfo->si_buf_offset + window->wd_offset) & 4119 MMU_PAGEOFFSET; 4120 pcnt = mmu_btopr(window->wd_size + poff); 4121 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4122 4123 /* map pages which require it */ 4124 for (i = 1; i < pcnt; i++) { 4125 pmap = &dma->dp_pgmap[pidx]; 4126 if (pmap->pm_uses_copybuf) { 4127 ASSERT(pmap->pm_mapped == B_FALSE); 4128 if (pmap->pm_pp != NULL) { 4129 pmap->pm_mapped = B_TRUE; 4130 i86_pp_map(pmap->pm_pp, 4131 pmap->pm_kaddr); 4132 } else if (pmap->pm_vaddr != NULL) { 4133 pmap->pm_mapped = B_TRUE; 4134 i86_va_map(pmap->pm_vaddr, 4135 sinfo->si_asp, 4136 pmap->pm_kaddr); 4137 } 4138 } 4139 pidx++; 4140 } 4141 } 4142 } 4143 #endif 4144 4145 /* if the new window uses the copy buffer, sync it for the device */ 4146 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4147 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4148 DDI_DMA_SYNC_FORDEV); 4149 } 4150 4151 return (DDI_SUCCESS); 4152 } 4153 4154 4155 4156 /* 4157 * ************************ 4158 * obsoleted dma routines 4159 * ************************ 4160 */ 4161 4162 /* 4163 * rootnex_dma_map() 4164 * called from ddi_dma_setup() 4165 */ 4166 /* ARGSUSED */ 4167 static int 4168 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4169 ddi_dma_handle_t *handlep) 4170 { 4171 #if defined(__amd64) 4172 /* 4173 * this interface is not supported in 64-bit x86 kernel. See comment in 4174 * rootnex_dma_mctl() 4175 */ 4176 return (DDI_DMA_NORESOURCES); 4177 4178 #else /* 32-bit x86 kernel */ 4179 ddi_dma_handle_t *lhandlep; 4180 ddi_dma_handle_t lhandle; 4181 ddi_dma_cookie_t cookie; 4182 ddi_dma_attr_t dma_attr; 4183 ddi_dma_lim_t *dma_lim; 4184 uint_t ccnt; 4185 int e; 4186 4187 4188 /* 4189 * if the driver is just testing to see if it's possible to do the bind, 4190 * we'll use local state. Otherwise, use the handle pointer passed in. 4191 */ 4192 if (handlep == NULL) { 4193 lhandlep = &lhandle; 4194 } else { 4195 lhandlep = handlep; 4196 } 4197 4198 /* convert the limit structure to a dma_attr one */ 4199 dma_lim = dmareq->dmar_limits; 4200 dma_attr.dma_attr_version = DMA_ATTR_V0; 4201 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4202 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4203 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4204 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4205 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4206 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4207 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4208 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4209 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4210 dma_attr.dma_attr_align = MMU_PAGESIZE; 4211 dma_attr.dma_attr_flags = 0; 4212 4213 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4214 dmareq->dmar_arg, lhandlep); 4215 if (e != DDI_SUCCESS) { 4216 return (e); 4217 } 4218 4219 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4220 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4221 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4222 return (e); 4223 } 4224 4225 /* 4226 * if the driver is just testing to see if it's possible to do the bind, 4227 * free up the local state and return the result. 4228 */ 4229 if (handlep == NULL) { 4230 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4231 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4232 if (e == DDI_DMA_MAPPED) { 4233 return (DDI_DMA_MAPOK); 4234 } else { 4235 return (DDI_DMA_NOMAPPING); 4236 } 4237 } 4238 4239 return (e); 4240 #endif /* defined(__amd64) */ 4241 } 4242 4243 4244 /* 4245 * rootnex_dma_mctl() 4246 * 4247 */ 4248 /* ARGSUSED */ 4249 static int 4250 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4251 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4252 uint_t cache_flags) 4253 { 4254 #if defined(__amd64) 4255 /* 4256 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4257 * common implementation in genunix, so they no longer have x86 4258 * specific functionality which called into dma_ctl. 4259 * 4260 * The rest of the obsoleted interfaces were never supported in the 4261 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4262 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4263 * implementation issues. 4264 * 4265 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4266 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4267 * reflect that now too... 4268 * 4269 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4270 * not going to put this functionality into the 64-bit x86 kernel now. 4271 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4272 * that in a future release. 4273 */ 4274 return (DDI_FAILURE); 4275 4276 #else /* 32-bit x86 kernel */ 4277 ddi_dma_cookie_t lcookie; 4278 ddi_dma_cookie_t *cookie; 4279 rootnex_window_t *window; 4280 ddi_dma_impl_t *hp; 4281 rootnex_dma_t *dma; 4282 uint_t nwin; 4283 uint_t ccnt; 4284 size_t len; 4285 off_t off; 4286 int e; 4287 4288 4289 /* 4290 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4291 * hacky since were optimizing for the current interfaces and so we can 4292 * cleanup the mess in genunix. Hopefully we will remove the this 4293 * obsoleted routines someday soon. 4294 */ 4295 4296 switch (request) { 4297 4298 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4299 hp = (ddi_dma_impl_t *)handle; 4300 cookie = (ddi_dma_cookie_t *)objpp; 4301 4302 /* 4303 * convert segment to cookie. We don't distinguish between the 4304 * two :-) 4305 */ 4306 *cookie = *hp->dmai_cookie; 4307 *lenp = cookie->dmac_size; 4308 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4309 return (DDI_SUCCESS); 4310 4311 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4312 hp = (ddi_dma_impl_t *)handle; 4313 dma = (rootnex_dma_t *)hp->dmai_private; 4314 4315 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4316 return (DDI_DMA_STALE); 4317 } 4318 4319 /* handle the case where we don't have any windows */ 4320 if (dma->dp_window == NULL) { 4321 /* 4322 * if seg == NULL, and we don't have any windows, 4323 * return the first cookie in the sgl. 4324 */ 4325 if (*lenp == NULL) { 4326 dma->dp_current_cookie = 0; 4327 hp->dmai_cookie = dma->dp_cookies; 4328 *objpp = (caddr_t)handle; 4329 return (DDI_SUCCESS); 4330 4331 /* if we have more cookies, go to the next cookie */ 4332 } else { 4333 if ((dma->dp_current_cookie + 1) >= 4334 dma->dp_sglinfo.si_sgl_size) { 4335 return (DDI_DMA_DONE); 4336 } 4337 dma->dp_current_cookie++; 4338 hp->dmai_cookie++; 4339 return (DDI_SUCCESS); 4340 } 4341 } 4342 4343 /* We have one or more windows */ 4344 window = &dma->dp_window[dma->dp_current_win]; 4345 4346 /* 4347 * if seg == NULL, return the first cookie in the current 4348 * window 4349 */ 4350 if (*lenp == NULL) { 4351 dma->dp_current_cookie = 0; 4352 hp->dmai_cookie = window->wd_first_cookie; 4353 4354 /* 4355 * go to the next cookie in the window then see if we done with 4356 * this window. 4357 */ 4358 } else { 4359 if ((dma->dp_current_cookie + 1) >= 4360 window->wd_cookie_cnt) { 4361 return (DDI_DMA_DONE); 4362 } 4363 dma->dp_current_cookie++; 4364 hp->dmai_cookie++; 4365 } 4366 *objpp = (caddr_t)handle; 4367 return (DDI_SUCCESS); 4368 4369 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4370 hp = (ddi_dma_impl_t *)handle; 4371 dma = (rootnex_dma_t *)hp->dmai_private; 4372 4373 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4374 return (DDI_DMA_STALE); 4375 } 4376 4377 /* if win == NULL, return the first window in the bind */ 4378 if (*offp == NULL) { 4379 nwin = 0; 4380 4381 /* 4382 * else, go to the next window then see if we're done with all 4383 * the windows. 4384 */ 4385 } else { 4386 nwin = dma->dp_current_win + 1; 4387 if (nwin >= hp->dmai_nwin) { 4388 return (DDI_DMA_DONE); 4389 } 4390 } 4391 4392 /* switch to the next window */ 4393 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4394 &lcookie, &ccnt); 4395 ASSERT(e == DDI_SUCCESS); 4396 if (e != DDI_SUCCESS) { 4397 return (DDI_DMA_STALE); 4398 } 4399 4400 /* reset the cookie back to the first cookie in the window */ 4401 if (dma->dp_window != NULL) { 4402 window = &dma->dp_window[dma->dp_current_win]; 4403 hp->dmai_cookie = window->wd_first_cookie; 4404 } else { 4405 hp->dmai_cookie = dma->dp_cookies; 4406 } 4407 4408 *objpp = (caddr_t)handle; 4409 return (DDI_SUCCESS); 4410 4411 case DDI_DMA_FREE: /* ddi_dma_free() */ 4412 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4413 (void) rootnex_dma_freehdl(dip, rdip, handle); 4414 if (rootnex_state->r_dvma_call_list_id) { 4415 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4416 } 4417 return (DDI_SUCCESS); 4418 4419 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4420 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4421 /* should never get here, handled in genunix */ 4422 ASSERT(0); 4423 return (DDI_FAILURE); 4424 4425 case DDI_DMA_KVADDR: 4426 case DDI_DMA_GETERR: 4427 case DDI_DMA_COFF: 4428 return (DDI_FAILURE); 4429 } 4430 4431 return (DDI_FAILURE); 4432 #endif /* defined(__amd64) */ 4433 } 4434 4435 4436 /* 4437 * ********* 4438 * FMA Code 4439 * ********* 4440 */ 4441 4442 /* 4443 * rootnex_fm_init() 4444 * FMA init busop 4445 */ 4446 /* ARGSUSED */ 4447 static int 4448 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4449 ddi_iblock_cookie_t *ibc) 4450 { 4451 *ibc = rootnex_state->r_err_ibc; 4452 4453 return (ddi_system_fmcap); 4454 } 4455 4456 /* 4457 * rootnex_dma_check() 4458 * Function called after a dma fault occurred to find out whether the 4459 * fault address is associated with a driver that is able to handle faults 4460 * and recover from faults. 4461 */ 4462 /* ARGSUSED */ 4463 static int 4464 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4465 const void *not_used) 4466 { 4467 rootnex_window_t *window; 4468 uint64_t start_addr; 4469 uint64_t fault_addr; 4470 ddi_dma_impl_t *hp; 4471 rootnex_dma_t *dma; 4472 uint64_t end_addr; 4473 size_t csize; 4474 int i; 4475 int j; 4476 4477 4478 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4479 hp = (ddi_dma_impl_t *)handle; 4480 ASSERT(hp); 4481 4482 dma = (rootnex_dma_t *)hp->dmai_private; 4483 4484 /* Get the address that we need to search for */ 4485 fault_addr = *(uint64_t *)addr; 4486 4487 /* 4488 * if we don't have any windows, we can just walk through all the 4489 * cookies. 4490 */ 4491 if (dma->dp_window == NULL) { 4492 /* for each cookie */ 4493 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4494 /* 4495 * if the faulted address is within the physical address 4496 * range of the cookie, return DDI_FM_NONFATAL. 4497 */ 4498 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4499 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4500 dma->dp_cookies[i].dmac_size))) { 4501 return (DDI_FM_NONFATAL); 4502 } 4503 } 4504 4505 /* fault_addr not within this DMA handle */ 4506 return (DDI_FM_UNKNOWN); 4507 } 4508 4509 /* we have mutiple windows, walk through each window */ 4510 for (i = 0; i < hp->dmai_nwin; i++) { 4511 window = &dma->dp_window[i]; 4512 4513 /* Go through all the cookies in the window */ 4514 for (j = 0; j < window->wd_cookie_cnt; j++) { 4515 4516 start_addr = window->wd_first_cookie[j].dmac_laddress; 4517 csize = window->wd_first_cookie[j].dmac_size; 4518 4519 /* 4520 * if we are trimming the first cookie in the window, 4521 * and this is the first cookie, adjust the start 4522 * address and size of the cookie to account for the 4523 * trim. 4524 */ 4525 if (window->wd_trim.tr_trim_first && (j == 0)) { 4526 start_addr = window->wd_trim.tr_first_paddr; 4527 csize = window->wd_trim.tr_first_size; 4528 } 4529 4530 /* 4531 * if we are trimming the last cookie in the window, 4532 * and this is the last cookie, adjust the start 4533 * address and size of the cookie to account for the 4534 * trim. 4535 */ 4536 if (window->wd_trim.tr_trim_last && 4537 (j == (window->wd_cookie_cnt - 1))) { 4538 start_addr = window->wd_trim.tr_last_paddr; 4539 csize = window->wd_trim.tr_last_size; 4540 } 4541 4542 end_addr = start_addr + csize; 4543 4544 /* 4545 * if the faulted address is within the physical address 4546 * range of the cookie, return DDI_FM_NONFATAL. 4547 */ 4548 if ((fault_addr >= start_addr) && 4549 (fault_addr <= end_addr)) { 4550 return (DDI_FM_NONFATAL); 4551 } 4552 } 4553 } 4554 4555 /* fault_addr not within this DMA handle */ 4556 return (DDI_FM_UNKNOWN); 4557 } 4558