1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2007 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 #pragma ident "%Z%%M% %I% %E% SMI" 27 28 /* 29 * x86 root nexus driver 30 */ 31 32 #include <sys/sysmacros.h> 33 #include <sys/conf.h> 34 #include <sys/autoconf.h> 35 #include <sys/sysmacros.h> 36 #include <sys/debug.h> 37 #include <sys/psw.h> 38 #include <sys/ddidmareq.h> 39 #include <sys/promif.h> 40 #include <sys/devops.h> 41 #include <sys/kmem.h> 42 #include <sys/cmn_err.h> 43 #include <vm/seg.h> 44 #include <vm/seg_kmem.h> 45 #include <vm/seg_dev.h> 46 #include <sys/vmem.h> 47 #include <sys/mman.h> 48 #include <vm/hat.h> 49 #include <vm/as.h> 50 #include <vm/page.h> 51 #include <sys/avintr.h> 52 #include <sys/errno.h> 53 #include <sys/modctl.h> 54 #include <sys/ddi_impldefs.h> 55 #include <sys/sunddi.h> 56 #include <sys/sunndi.h> 57 #include <sys/mach_intr.h> 58 #include <sys/psm.h> 59 #include <sys/ontrap.h> 60 #include <sys/atomic.h> 61 #include <sys/sdt.h> 62 #include <sys/rootnex.h> 63 #include <vm/hat_i86.h> 64 #include <sys/ddifm.h> 65 66 #ifdef __xpv 67 #include <sys/bootinfo.h> 68 #include <sys/hypervisor.h> 69 #include <sys/bootconf.h> 70 #include <vm/kboot_mmu.h> 71 #endif 72 73 /* 74 * enable/disable extra checking of function parameters. Useful for debugging 75 * drivers. 76 */ 77 #ifdef DEBUG 78 int rootnex_alloc_check_parms = 1; 79 int rootnex_bind_check_parms = 1; 80 int rootnex_bind_check_inuse = 1; 81 int rootnex_unbind_verify_buffer = 0; 82 int rootnex_sync_check_parms = 1; 83 #else 84 int rootnex_alloc_check_parms = 0; 85 int rootnex_bind_check_parms = 0; 86 int rootnex_bind_check_inuse = 0; 87 int rootnex_unbind_verify_buffer = 0; 88 int rootnex_sync_check_parms = 0; 89 #endif 90 91 /* Master Abort and Target Abort panic flag */ 92 int rootnex_fm_ma_ta_panic_flag = 0; 93 94 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 95 int rootnex_bind_fail = 1; 96 int rootnex_bind_warn = 1; 97 uint8_t *rootnex_warn_list; 98 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 99 #define ROOTNEX_BIND_WARNING (0x1 << 0) 100 101 /* 102 * revert back to old broken behavior of always sync'ing entire copy buffer. 103 * This is useful if be have a buggy driver which doesn't correctly pass in 104 * the offset and size into ddi_dma_sync(). 105 */ 106 int rootnex_sync_ignore_params = 0; 107 108 /* 109 * maximum size that we will allow for a copy buffer. Can be patched on the 110 * fly 111 */ 112 size_t rootnex_max_copybuf_size = 0x100000; 113 114 /* 115 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 116 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 117 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 118 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 119 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 120 * (< 8K). We will still need to allocate the copy buffer during bind though 121 * (if we need one). These can only be modified in /etc/system before rootnex 122 * attach. 123 */ 124 #if defined(__amd64) 125 int rootnex_prealloc_cookies = 65; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #else 129 int rootnex_prealloc_cookies = 33; 130 int rootnex_prealloc_windows = 4; 131 int rootnex_prealloc_copybuf = 2; 132 #endif 133 134 /* driver global state */ 135 static rootnex_state_t *rootnex_state; 136 137 /* shortcut to rootnex counters */ 138 static uint64_t *rootnex_cnt; 139 140 /* 141 * XXX - does x86 even need these or are they left over from the SPARC days? 142 */ 143 /* statically defined integer/boolean properties for the root node */ 144 static rootnex_intprop_t rootnex_intprp[] = { 145 { "PAGESIZE", PAGESIZE }, 146 { "MMU_PAGESIZE", MMU_PAGESIZE }, 147 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 148 { DDI_RELATIVE_ADDRESSING, 1 }, 149 }; 150 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 151 152 #ifdef __xpv 153 typedef maddr_t rootnex_addr_t; 154 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 155 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 156 #else 157 typedef paddr_t rootnex_addr_t; 158 #endif 159 160 161 static struct cb_ops rootnex_cb_ops = { 162 nodev, /* open */ 163 nodev, /* close */ 164 nodev, /* strategy */ 165 nodev, /* print */ 166 nodev, /* dump */ 167 nodev, /* read */ 168 nodev, /* write */ 169 nodev, /* ioctl */ 170 nodev, /* devmap */ 171 nodev, /* mmap */ 172 nodev, /* segmap */ 173 nochpoll, /* chpoll */ 174 ddi_prop_op, /* cb_prop_op */ 175 NULL, /* struct streamtab */ 176 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 177 CB_REV, /* Rev */ 178 nodev, /* cb_aread */ 179 nodev /* cb_awrite */ 180 }; 181 182 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 183 off_t offset, off_t len, caddr_t *vaddrp); 184 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 185 struct hat *hat, struct seg *seg, caddr_t addr, 186 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 187 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 188 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 189 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 190 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 191 ddi_dma_handle_t *handlep); 192 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 193 ddi_dma_handle_t handle); 194 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 195 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 196 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 197 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 198 ddi_dma_handle_t handle); 199 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 201 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 202 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 203 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 204 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 206 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 207 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 208 ddi_ctl_enum_t ctlop, void *arg, void *result); 209 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 210 ddi_iblock_cookie_t *ibc); 211 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 212 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 213 214 215 static struct bus_ops rootnex_bus_ops = { 216 BUSO_REV, 217 rootnex_map, 218 NULL, 219 NULL, 220 NULL, 221 rootnex_map_fault, 222 rootnex_dma_map, 223 rootnex_dma_allochdl, 224 rootnex_dma_freehdl, 225 rootnex_dma_bindhdl, 226 rootnex_dma_unbindhdl, 227 rootnex_dma_sync, 228 rootnex_dma_win, 229 rootnex_dma_mctl, 230 rootnex_ctlops, 231 ddi_bus_prop_op, 232 i_ddi_rootnex_get_eventcookie, 233 i_ddi_rootnex_add_eventcall, 234 i_ddi_rootnex_remove_eventcall, 235 i_ddi_rootnex_post_event, 236 0, /* bus_intr_ctl */ 237 0, /* bus_config */ 238 0, /* bus_unconfig */ 239 rootnex_fm_init, /* bus_fm_init */ 240 NULL, /* bus_fm_fini */ 241 NULL, /* bus_fm_access_enter */ 242 NULL, /* bus_fm_access_exit */ 243 NULL, /* bus_powr */ 244 rootnex_intr_ops /* bus_intr_op */ 245 }; 246 247 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 248 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 249 250 static struct dev_ops rootnex_ops = { 251 DEVO_REV, 252 0, 253 ddi_no_info, 254 nulldev, 255 nulldev, 256 rootnex_attach, 257 rootnex_detach, 258 nulldev, 259 &rootnex_cb_ops, 260 &rootnex_bus_ops 261 }; 262 263 static struct modldrv rootnex_modldrv = { 264 &mod_driverops, 265 "i86pc root nexus %I%", 266 &rootnex_ops 267 }; 268 269 static struct modlinkage rootnex_modlinkage = { 270 MODREV_1, 271 (void *)&rootnex_modldrv, 272 NULL 273 }; 274 275 276 /* 277 * extern hacks 278 */ 279 extern struct seg_ops segdev_ops; 280 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 281 #ifdef DDI_MAP_DEBUG 282 extern int ddi_map_debug_flag; 283 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 284 #endif 285 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 286 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 287 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 288 psm_intr_op_t, int *); 289 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 290 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 291 /* 292 * Use device arena to use for device control register mappings. 293 * Various kernel memory walkers (debugger, dtrace) need to know 294 * to avoid this address range to prevent undesired device activity. 295 */ 296 extern void *device_arena_alloc(size_t size, int vm_flag); 297 extern void device_arena_free(void * vaddr, size_t size); 298 299 300 /* 301 * Internal functions 302 */ 303 static int rootnex_dma_init(); 304 static void rootnex_add_props(dev_info_t *); 305 static int rootnex_ctl_reportdev(dev_info_t *dip); 306 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 307 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 308 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 309 static int rootnex_map_handle(ddi_map_req_t *mp); 310 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 311 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 312 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 313 ddi_dma_attr_t *attr); 314 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 315 rootnex_sglinfo_t *sglinfo); 316 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 317 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 318 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 319 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 320 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 321 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 322 ddi_dma_attr_t *attr, int kmflag); 323 static void rootnex_teardown_windows(rootnex_dma_t *dma); 324 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 325 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 326 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 327 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 328 size_t *copybuf_used, page_t **cur_pp); 329 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 330 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 331 ddi_dma_attr_t *attr, off_t cur_offset); 332 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 333 rootnex_dma_t *dma, rootnex_window_t **windowp, 334 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 335 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 336 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 337 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 338 off_t offset, size_t size, uint_t cache_flags); 339 static int rootnex_verify_buffer(rootnex_dma_t *dma); 340 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 341 const void *comp_addr, const void *not_used); 342 343 /* 344 * _init() 345 * 346 */ 347 int 348 _init(void) 349 { 350 351 rootnex_state = NULL; 352 return (mod_install(&rootnex_modlinkage)); 353 } 354 355 356 /* 357 * _info() 358 * 359 */ 360 int 361 _info(struct modinfo *modinfop) 362 { 363 return (mod_info(&rootnex_modlinkage, modinfop)); 364 } 365 366 367 /* 368 * _fini() 369 * 370 */ 371 int 372 _fini(void) 373 { 374 return (EBUSY); 375 } 376 377 378 /* 379 * rootnex_attach() 380 * 381 */ 382 static int 383 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 384 { 385 int fmcap; 386 int e; 387 388 switch (cmd) { 389 case DDI_ATTACH: 390 break; 391 case DDI_RESUME: 392 return (DDI_SUCCESS); 393 default: 394 return (DDI_FAILURE); 395 } 396 397 /* 398 * We should only have one instance of rootnex. Save it away since we 399 * don't have an easy way to get it back later. 400 */ 401 ASSERT(rootnex_state == NULL); 402 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 403 404 rootnex_state->r_dip = dip; 405 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 406 rootnex_state->r_reserved_msg_printed = B_FALSE; 407 rootnex_cnt = &rootnex_state->r_counters[0]; 408 409 /* 410 * Set minimum fm capability level for i86pc platforms and then 411 * initialize error handling. Since we're the rootnex, we don't 412 * care what's returned in the fmcap field. 413 */ 414 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 415 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 416 fmcap = ddi_system_fmcap; 417 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 418 419 /* initialize DMA related state */ 420 e = rootnex_dma_init(); 421 if (e != DDI_SUCCESS) { 422 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 423 return (DDI_FAILURE); 424 } 425 426 /* Add static root node properties */ 427 rootnex_add_props(dip); 428 429 /* since we can't call ddi_report_dev() */ 430 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 431 432 /* Initialize rootnex event handle */ 433 i_ddi_rootnex_init_events(dip); 434 435 return (DDI_SUCCESS); 436 } 437 438 439 /* 440 * rootnex_detach() 441 * 442 */ 443 /*ARGSUSED*/ 444 static int 445 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 446 { 447 switch (cmd) { 448 case DDI_SUSPEND: 449 break; 450 default: 451 return (DDI_FAILURE); 452 } 453 454 return (DDI_SUCCESS); 455 } 456 457 458 /* 459 * rootnex_dma_init() 460 * 461 */ 462 /*ARGSUSED*/ 463 static int 464 rootnex_dma_init() 465 { 466 size_t bufsize; 467 468 469 /* 470 * size of our cookie/window/copybuf state needed in dma bind that we 471 * pre-alloc in dma_alloc_handle 472 */ 473 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 474 rootnex_state->r_prealloc_size = 475 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 476 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 477 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 478 479 /* 480 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 481 * allocate 16 extra bytes for struct pointer alignment 482 * (p->dmai_private & dma->dp_prealloc_buffer) 483 */ 484 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 485 rootnex_state->r_prealloc_size + 0x10; 486 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 487 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 488 if (rootnex_state->r_dmahdl_cache == NULL) { 489 return (DDI_FAILURE); 490 } 491 492 /* 493 * allocate array to track which major numbers we have printed warnings 494 * for. 495 */ 496 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 497 KM_SLEEP); 498 499 return (DDI_SUCCESS); 500 } 501 502 503 /* 504 * rootnex_add_props() 505 * 506 */ 507 static void 508 rootnex_add_props(dev_info_t *dip) 509 { 510 rootnex_intprop_t *rpp; 511 int i; 512 513 /* Add static integer/boolean properties to the root node */ 514 rpp = rootnex_intprp; 515 for (i = 0; i < NROOT_INTPROPS; i++) { 516 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 517 rpp[i].prop_name, rpp[i].prop_value); 518 } 519 } 520 521 522 523 /* 524 * ************************* 525 * ctlops related routines 526 * ************************* 527 */ 528 529 /* 530 * rootnex_ctlops() 531 * 532 */ 533 /*ARGSUSED*/ 534 static int 535 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 536 void *arg, void *result) 537 { 538 int n, *ptr; 539 struct ddi_parent_private_data *pdp; 540 541 switch (ctlop) { 542 case DDI_CTLOPS_DMAPMAPC: 543 /* 544 * Return 'partial' to indicate that dma mapping 545 * has to be done in the main MMU. 546 */ 547 return (DDI_DMA_PARTIAL); 548 549 case DDI_CTLOPS_BTOP: 550 /* 551 * Convert byte count input to physical page units. 552 * (byte counts that are not a page-size multiple 553 * are rounded down) 554 */ 555 *(ulong_t *)result = btop(*(ulong_t *)arg); 556 return (DDI_SUCCESS); 557 558 case DDI_CTLOPS_PTOB: 559 /* 560 * Convert size in physical pages to bytes 561 */ 562 *(ulong_t *)result = ptob(*(ulong_t *)arg); 563 return (DDI_SUCCESS); 564 565 case DDI_CTLOPS_BTOPR: 566 /* 567 * Convert byte count input to physical page units 568 * (byte counts that are not a page-size multiple 569 * are rounded up) 570 */ 571 *(ulong_t *)result = btopr(*(ulong_t *)arg); 572 return (DDI_SUCCESS); 573 574 case DDI_CTLOPS_INITCHILD: 575 return (impl_ddi_sunbus_initchild(arg)); 576 577 case DDI_CTLOPS_UNINITCHILD: 578 impl_ddi_sunbus_removechild(arg); 579 return (DDI_SUCCESS); 580 581 case DDI_CTLOPS_REPORTDEV: 582 return (rootnex_ctl_reportdev(rdip)); 583 584 case DDI_CTLOPS_IOMIN: 585 /* 586 * Nothing to do here but reflect back.. 587 */ 588 return (DDI_SUCCESS); 589 590 case DDI_CTLOPS_REGSIZE: 591 case DDI_CTLOPS_NREGS: 592 break; 593 594 case DDI_CTLOPS_SIDDEV: 595 if (ndi_dev_is_prom_node(rdip)) 596 return (DDI_SUCCESS); 597 if (ndi_dev_is_persistent_node(rdip)) 598 return (DDI_SUCCESS); 599 return (DDI_FAILURE); 600 601 case DDI_CTLOPS_POWER: 602 return ((*pm_platform_power)((power_req_t *)arg)); 603 604 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 605 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 606 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 607 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 608 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 609 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 610 if (!rootnex_state->r_reserved_msg_printed) { 611 rootnex_state->r_reserved_msg_printed = B_TRUE; 612 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 613 "1 or more reserved/obsolete operations."); 614 } 615 return (DDI_FAILURE); 616 617 default: 618 return (DDI_FAILURE); 619 } 620 /* 621 * The rest are for "hardware" properties 622 */ 623 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 624 return (DDI_FAILURE); 625 626 if (ctlop == DDI_CTLOPS_NREGS) { 627 ptr = (int *)result; 628 *ptr = pdp->par_nreg; 629 } else { 630 off_t *size = (off_t *)result; 631 632 ptr = (int *)arg; 633 n = *ptr; 634 if (n >= pdp->par_nreg) { 635 return (DDI_FAILURE); 636 } 637 *size = (off_t)pdp->par_reg[n].regspec_size; 638 } 639 return (DDI_SUCCESS); 640 } 641 642 643 /* 644 * rootnex_ctl_reportdev() 645 * 646 */ 647 static int 648 rootnex_ctl_reportdev(dev_info_t *dev) 649 { 650 int i, n, len, f_len = 0; 651 char *buf; 652 653 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 654 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 655 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 656 len = strlen(buf); 657 658 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 659 660 struct regspec *rp = sparc_pd_getreg(dev, i); 661 662 if (i == 0) 663 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 664 ": "); 665 else 666 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 667 " and "); 668 len = strlen(buf); 669 670 switch (rp->regspec_bustype) { 671 672 case BTEISA: 673 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 674 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 675 break; 676 677 case BTISA: 678 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 679 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 680 break; 681 682 default: 683 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 684 "space %x offset %x", 685 rp->regspec_bustype, rp->regspec_addr); 686 break; 687 } 688 len = strlen(buf); 689 } 690 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 691 int pri; 692 693 if (i != 0) { 694 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 695 ","); 696 len = strlen(buf); 697 } 698 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 699 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 700 " sparc ipl %d", pri); 701 len = strlen(buf); 702 } 703 #ifdef DEBUG 704 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 705 cmn_err(CE_NOTE, "next message is truncated: " 706 "printed length 1024, real length %d", f_len); 707 } 708 #endif /* DEBUG */ 709 cmn_err(CE_CONT, "?%s\n", buf); 710 kmem_free(buf, REPORTDEV_BUFSIZE); 711 return (DDI_SUCCESS); 712 } 713 714 715 /* 716 * ****************** 717 * map related code 718 * ****************** 719 */ 720 721 /* 722 * rootnex_map() 723 * 724 */ 725 static int 726 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 727 off_t len, caddr_t *vaddrp) 728 { 729 struct regspec *rp, tmp_reg; 730 ddi_map_req_t mr = *mp; /* Get private copy of request */ 731 int error; 732 733 mp = &mr; 734 735 switch (mp->map_op) { 736 case DDI_MO_MAP_LOCKED: 737 case DDI_MO_UNMAP: 738 case DDI_MO_MAP_HANDLE: 739 break; 740 default: 741 #ifdef DDI_MAP_DEBUG 742 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 743 mp->map_op); 744 #endif /* DDI_MAP_DEBUG */ 745 return (DDI_ME_UNIMPLEMENTED); 746 } 747 748 if (mp->map_flags & DDI_MF_USER_MAPPING) { 749 #ifdef DDI_MAP_DEBUG 750 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 751 #endif /* DDI_MAP_DEBUG */ 752 return (DDI_ME_UNIMPLEMENTED); 753 } 754 755 /* 756 * First, if given an rnumber, convert it to a regspec... 757 * (Presumably, this is on behalf of a child of the root node?) 758 */ 759 760 if (mp->map_type == DDI_MT_RNUMBER) { 761 762 int rnumber = mp->map_obj.rnumber; 763 #ifdef DDI_MAP_DEBUG 764 static char *out_of_range = 765 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 766 #endif /* DDI_MAP_DEBUG */ 767 768 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 769 if (rp == NULL) { 770 #ifdef DDI_MAP_DEBUG 771 cmn_err(CE_WARN, out_of_range, rnumber, 772 ddi_get_name(rdip)); 773 #endif /* DDI_MAP_DEBUG */ 774 return (DDI_ME_RNUMBER_RANGE); 775 } 776 777 /* 778 * Convert the given ddi_map_req_t from rnumber to regspec... 779 */ 780 781 mp->map_type = DDI_MT_REGSPEC; 782 mp->map_obj.rp = rp; 783 } 784 785 /* 786 * Adjust offset and length correspnding to called values... 787 * XXX: A non-zero length means override the one in the regspec 788 * XXX: (regardless of what's in the parent's range?) 789 */ 790 791 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 792 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 793 794 #ifdef DDI_MAP_DEBUG 795 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 796 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 797 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 798 len, mp->map_handlep); 799 #endif /* DDI_MAP_DEBUG */ 800 801 /* 802 * I/O or memory mapping: 803 * 804 * <bustype=0, addr=x, len=x>: memory 805 * <bustype=1, addr=x, len=x>: i/o 806 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 807 */ 808 809 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 810 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 811 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 812 ddi_get_name(rdip), rp->regspec_bustype, 813 rp->regspec_addr, rp->regspec_size); 814 return (DDI_ME_INVAL); 815 } 816 817 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 818 /* 819 * compatibility i/o mapping 820 */ 821 rp->regspec_bustype += (uint_t)offset; 822 } else { 823 /* 824 * Normal memory or i/o mapping 825 */ 826 rp->regspec_addr += (uint_t)offset; 827 } 828 829 if (len != 0) 830 rp->regspec_size = (uint_t)len; 831 832 #ifdef DDI_MAP_DEBUG 833 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 834 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 835 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 836 offset, len, mp->map_handlep); 837 #endif /* DDI_MAP_DEBUG */ 838 839 /* 840 * Apply any parent ranges at this level, if applicable. 841 * (This is where nexus specific regspec translation takes place. 842 * Use of this function is implicit agreement that translation is 843 * provided via ddi_apply_range.) 844 */ 845 846 #ifdef DDI_MAP_DEBUG 847 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 848 ddi_get_name(dip), ddi_get_name(rdip)); 849 #endif /* DDI_MAP_DEBUG */ 850 851 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 852 return (error); 853 854 switch (mp->map_op) { 855 case DDI_MO_MAP_LOCKED: 856 857 /* 858 * Set up the locked down kernel mapping to the regspec... 859 */ 860 861 return (rootnex_map_regspec(mp, vaddrp)); 862 863 case DDI_MO_UNMAP: 864 865 /* 866 * Release mapping... 867 */ 868 869 return (rootnex_unmap_regspec(mp, vaddrp)); 870 871 case DDI_MO_MAP_HANDLE: 872 873 return (rootnex_map_handle(mp)); 874 875 default: 876 return (DDI_ME_UNIMPLEMENTED); 877 } 878 } 879 880 881 /* 882 * rootnex_map_fault() 883 * 884 * fault in mappings for requestors 885 */ 886 /*ARGSUSED*/ 887 static int 888 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 889 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 890 uint_t lock) 891 { 892 893 #ifdef DDI_MAP_DEBUG 894 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 895 ddi_map_debug(" Seg <%s>\n", 896 seg->s_ops == &segdev_ops ? "segdev" : 897 seg == &kvseg ? "segkmem" : "NONE!"); 898 #endif /* DDI_MAP_DEBUG */ 899 900 /* 901 * This is all terribly broken, but it is a start 902 * 903 * XXX Note that this test means that segdev_ops 904 * must be exported from seg_dev.c. 905 * XXX What about devices with their own segment drivers? 906 */ 907 if (seg->s_ops == &segdev_ops) { 908 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 909 910 if (hat == NULL) { 911 /* 912 * This is one plausible interpretation of 913 * a null hat i.e. use the first hat on the 914 * address space hat list which by convention is 915 * the hat of the system MMU. At alternative 916 * would be to panic .. this might well be better .. 917 */ 918 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 919 hat = seg->s_as->a_hat; 920 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 921 } 922 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 923 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 924 } else if (seg == &kvseg && dp == NULL) { 925 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 926 HAT_LOAD_LOCK); 927 } else 928 return (DDI_FAILURE); 929 return (DDI_SUCCESS); 930 } 931 932 933 /* 934 * rootnex_map_regspec() 935 * we don't support mapping of I/O cards above 4Gb 936 */ 937 static int 938 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 939 { 940 rootnex_addr_t rbase; 941 void *cvaddr; 942 uint_t npages, pgoffset; 943 struct regspec *rp; 944 ddi_acc_hdl_t *hp; 945 ddi_acc_impl_t *ap; 946 uint_t hat_acc_flags; 947 paddr_t pbase; 948 949 rp = mp->map_obj.rp; 950 hp = mp->map_handlep; 951 952 #ifdef DDI_MAP_DEBUG 953 ddi_map_debug( 954 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 955 rp->regspec_bustype, rp->regspec_addr, 956 rp->regspec_size, mp->map_handlep); 957 #endif /* DDI_MAP_DEBUG */ 958 959 /* 960 * I/O or memory mapping 961 * 962 * <bustype=0, addr=x, len=x>: memory 963 * <bustype=1, addr=x, len=x>: i/o 964 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 965 */ 966 967 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 968 cmn_err(CE_WARN, "rootnex: invalid register spec" 969 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 970 rp->regspec_addr, rp->regspec_size); 971 return (DDI_FAILURE); 972 } 973 974 if (rp->regspec_bustype != 0) { 975 /* 976 * I/O space - needs a handle. 977 */ 978 if (hp == NULL) { 979 return (DDI_FAILURE); 980 } 981 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 982 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 983 impl_acc_hdl_init(hp); 984 985 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 986 #ifdef DDI_MAP_DEBUG 987 ddi_map_debug("rootnex_map_regspec: mmap() " 988 "to I/O space is not supported.\n"); 989 #endif /* DDI_MAP_DEBUG */ 990 return (DDI_ME_INVAL); 991 } else { 992 /* 993 * 1275-compliant vs. compatibility i/o mapping 994 */ 995 *vaddrp = 996 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 997 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 998 ((caddr_t)(uintptr_t)rp->regspec_addr); 999 #ifdef __xpv 1000 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1001 hp->ah_pfn = xen_assign_pfn( 1002 mmu_btop((ulong_t)rp->regspec_addr & 1003 MMU_PAGEMASK)); 1004 } else { 1005 hp->ah_pfn = mmu_btop( 1006 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1007 } 1008 #else 1009 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1010 MMU_PAGEMASK); 1011 #endif 1012 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1013 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1014 } 1015 1016 #ifdef DDI_MAP_DEBUG 1017 ddi_map_debug( 1018 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1019 rp->regspec_size, *vaddrp); 1020 #endif /* DDI_MAP_DEBUG */ 1021 return (DDI_SUCCESS); 1022 } 1023 1024 /* 1025 * Memory space 1026 */ 1027 1028 if (hp != NULL) { 1029 /* 1030 * hat layer ignores 1031 * hp->ah_acc.devacc_attr_endian_flags. 1032 */ 1033 switch (hp->ah_acc.devacc_attr_dataorder) { 1034 case DDI_STRICTORDER_ACC: 1035 hat_acc_flags = HAT_STRICTORDER; 1036 break; 1037 case DDI_UNORDERED_OK_ACC: 1038 hat_acc_flags = HAT_UNORDERED_OK; 1039 break; 1040 case DDI_MERGING_OK_ACC: 1041 hat_acc_flags = HAT_MERGING_OK; 1042 break; 1043 case DDI_LOADCACHING_OK_ACC: 1044 hat_acc_flags = HAT_LOADCACHING_OK; 1045 break; 1046 case DDI_STORECACHING_OK_ACC: 1047 hat_acc_flags = HAT_STORECACHING_OK; 1048 break; 1049 } 1050 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1051 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1052 impl_acc_hdl_init(hp); 1053 hp->ah_hat_flags = hat_acc_flags; 1054 } else { 1055 hat_acc_flags = HAT_STRICTORDER; 1056 } 1057 1058 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1059 #ifdef __xpv 1060 /* 1061 * If we're dom0, we're using a real device so we need to translate 1062 * the MA to a PA. 1063 */ 1064 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1065 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1066 } else { 1067 pbase = rbase; 1068 } 1069 #else 1070 pbase = rbase; 1071 #endif 1072 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1073 1074 if (rp->regspec_size == 0) { 1075 #ifdef DDI_MAP_DEBUG 1076 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1077 #endif /* DDI_MAP_DEBUG */ 1078 return (DDI_ME_INVAL); 1079 } 1080 1081 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1082 /* extra cast to make gcc happy */ 1083 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1084 } else { 1085 npages = mmu_btopr(rp->regspec_size + pgoffset); 1086 1087 #ifdef DDI_MAP_DEBUG 1088 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1089 "physical %llx", npages, pbase); 1090 #endif /* DDI_MAP_DEBUG */ 1091 1092 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1093 if (cvaddr == NULL) 1094 return (DDI_ME_NORESOURCES); 1095 1096 /* 1097 * Now map in the pages we've allocated... 1098 */ 1099 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1100 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1101 HAT_LOAD_LOCK); 1102 *vaddrp = (caddr_t)cvaddr + pgoffset; 1103 1104 /* save away pfn and npages for FMA */ 1105 hp = mp->map_handlep; 1106 if (hp) { 1107 hp->ah_pfn = mmu_btop(pbase); 1108 hp->ah_pnum = npages; 1109 } 1110 } 1111 1112 #ifdef DDI_MAP_DEBUG 1113 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1114 #endif /* DDI_MAP_DEBUG */ 1115 return (DDI_SUCCESS); 1116 } 1117 1118 1119 /* 1120 * rootnex_unmap_regspec() 1121 * 1122 */ 1123 static int 1124 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1125 { 1126 caddr_t addr = (caddr_t)*vaddrp; 1127 uint_t npages, pgoffset; 1128 struct regspec *rp; 1129 1130 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1131 return (0); 1132 1133 rp = mp->map_obj.rp; 1134 1135 if (rp->regspec_size == 0) { 1136 #ifdef DDI_MAP_DEBUG 1137 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1138 #endif /* DDI_MAP_DEBUG */ 1139 return (DDI_ME_INVAL); 1140 } 1141 1142 /* 1143 * I/O or memory mapping: 1144 * 1145 * <bustype=0, addr=x, len=x>: memory 1146 * <bustype=1, addr=x, len=x>: i/o 1147 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1148 */ 1149 if (rp->regspec_bustype != 0) { 1150 /* 1151 * This is I/O space, which requires no particular 1152 * processing on unmap since it isn't mapped in the 1153 * first place. 1154 */ 1155 return (DDI_SUCCESS); 1156 } 1157 1158 /* 1159 * Memory space 1160 */ 1161 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1162 npages = mmu_btopr(rp->regspec_size + pgoffset); 1163 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1164 device_arena_free(addr - pgoffset, ptob(npages)); 1165 1166 /* 1167 * Destroy the pointer - the mapping has logically gone 1168 */ 1169 *vaddrp = NULL; 1170 1171 return (DDI_SUCCESS); 1172 } 1173 1174 1175 /* 1176 * rootnex_map_handle() 1177 * 1178 */ 1179 static int 1180 rootnex_map_handle(ddi_map_req_t *mp) 1181 { 1182 rootnex_addr_t rbase; 1183 ddi_acc_hdl_t *hp; 1184 uint_t pgoffset; 1185 struct regspec *rp; 1186 paddr_t pbase; 1187 1188 rp = mp->map_obj.rp; 1189 1190 #ifdef DDI_MAP_DEBUG 1191 ddi_map_debug( 1192 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1193 rp->regspec_bustype, rp->regspec_addr, 1194 rp->regspec_size, mp->map_handlep); 1195 #endif /* DDI_MAP_DEBUG */ 1196 1197 /* 1198 * I/O or memory mapping: 1199 * 1200 * <bustype=0, addr=x, len=x>: memory 1201 * <bustype=1, addr=x, len=x>: i/o 1202 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1203 */ 1204 if (rp->regspec_bustype != 0) { 1205 /* 1206 * This refers to I/O space, and we don't support "mapping" 1207 * I/O space to a user. 1208 */ 1209 return (DDI_FAILURE); 1210 } 1211 1212 /* 1213 * Set up the hat_flags for the mapping. 1214 */ 1215 hp = mp->map_handlep; 1216 1217 switch (hp->ah_acc.devacc_attr_endian_flags) { 1218 case DDI_NEVERSWAP_ACC: 1219 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1220 break; 1221 case DDI_STRUCTURE_LE_ACC: 1222 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1223 break; 1224 case DDI_STRUCTURE_BE_ACC: 1225 return (DDI_FAILURE); 1226 default: 1227 return (DDI_REGS_ACC_CONFLICT); 1228 } 1229 1230 switch (hp->ah_acc.devacc_attr_dataorder) { 1231 case DDI_STRICTORDER_ACC: 1232 break; 1233 case DDI_UNORDERED_OK_ACC: 1234 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1235 break; 1236 case DDI_MERGING_OK_ACC: 1237 hp->ah_hat_flags |= HAT_MERGING_OK; 1238 break; 1239 case DDI_LOADCACHING_OK_ACC: 1240 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1241 break; 1242 case DDI_STORECACHING_OK_ACC: 1243 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1244 break; 1245 default: 1246 return (DDI_FAILURE); 1247 } 1248 1249 rbase = (rootnex_addr_t)rp->regspec_addr & 1250 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1251 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1252 1253 if (rp->regspec_size == 0) 1254 return (DDI_ME_INVAL); 1255 1256 #ifdef __xpv 1257 /* 1258 * If we're dom0, we're using a real device so we need to translate 1259 * the MA to a PA. 1260 */ 1261 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1262 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1263 (rbase & MMU_PAGEOFFSET); 1264 } else { 1265 pbase = rbase; 1266 } 1267 #else 1268 pbase = rbase; 1269 #endif 1270 1271 hp->ah_pfn = mmu_btop(pbase); 1272 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1273 1274 return (DDI_SUCCESS); 1275 } 1276 1277 1278 1279 /* 1280 * ************************ 1281 * interrupt related code 1282 * ************************ 1283 */ 1284 1285 /* 1286 * rootnex_intr_ops() 1287 * bus_intr_op() function for interrupt support 1288 */ 1289 /* ARGSUSED */ 1290 static int 1291 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1292 ddi_intr_handle_impl_t *hdlp, void *result) 1293 { 1294 struct intrspec *ispec; 1295 struct ddi_parent_private_data *pdp; 1296 1297 DDI_INTR_NEXDBG((CE_CONT, 1298 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1299 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1300 1301 /* Process the interrupt operation */ 1302 switch (intr_op) { 1303 case DDI_INTROP_GETCAP: 1304 /* First check with pcplusmp */ 1305 if (psm_intr_ops == NULL) 1306 return (DDI_FAILURE); 1307 1308 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1309 *(int *)result = 0; 1310 return (DDI_FAILURE); 1311 } 1312 break; 1313 case DDI_INTROP_SETCAP: 1314 if (psm_intr_ops == NULL) 1315 return (DDI_FAILURE); 1316 1317 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1318 return (DDI_FAILURE); 1319 break; 1320 case DDI_INTROP_ALLOC: 1321 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1322 return (DDI_FAILURE); 1323 hdlp->ih_pri = ispec->intrspec_pri; 1324 *(int *)result = hdlp->ih_scratch1; 1325 break; 1326 case DDI_INTROP_FREE: 1327 pdp = ddi_get_parent_data(rdip); 1328 /* 1329 * Special case for 'pcic' driver' only. 1330 * If an intrspec was created for it, clean it up here 1331 * See detailed comments on this in the function 1332 * rootnex_get_ispec(). 1333 */ 1334 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1335 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1336 pdp->par_nintr); 1337 /* 1338 * Set it to zero; so that 1339 * DDI framework doesn't free it again 1340 */ 1341 pdp->par_intr = NULL; 1342 pdp->par_nintr = 0; 1343 } 1344 break; 1345 case DDI_INTROP_GETPRI: 1346 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1347 return (DDI_FAILURE); 1348 *(int *)result = ispec->intrspec_pri; 1349 break; 1350 case DDI_INTROP_SETPRI: 1351 /* Validate the interrupt priority passed to us */ 1352 if (*(int *)result > LOCK_LEVEL) 1353 return (DDI_FAILURE); 1354 1355 /* Ensure that PSM is all initialized and ispec is ok */ 1356 if ((psm_intr_ops == NULL) || 1357 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1358 return (DDI_FAILURE); 1359 1360 /* Change the priority */ 1361 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1362 PSM_FAILURE) 1363 return (DDI_FAILURE); 1364 1365 /* update the ispec with the new priority */ 1366 ispec->intrspec_pri = *(int *)result; 1367 break; 1368 case DDI_INTROP_ADDISR: 1369 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1370 return (DDI_FAILURE); 1371 ispec->intrspec_func = hdlp->ih_cb_func; 1372 break; 1373 case DDI_INTROP_REMISR: 1374 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1375 return (DDI_FAILURE); 1376 ispec->intrspec_func = (uint_t (*)()) 0; 1377 break; 1378 case DDI_INTROP_ENABLE: 1379 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1380 return (DDI_FAILURE); 1381 1382 /* Call psmi to translate irq with the dip */ 1383 if (psm_intr_ops == NULL) 1384 return (DDI_FAILURE); 1385 1386 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1387 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1388 (int *)&hdlp->ih_vector); 1389 1390 /* Add the interrupt handler */ 1391 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1392 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1393 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1394 return (DDI_FAILURE); 1395 break; 1396 case DDI_INTROP_DISABLE: 1397 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1398 return (DDI_FAILURE); 1399 1400 /* Call psm_ops() to translate irq with the dip */ 1401 if (psm_intr_ops == NULL) 1402 return (DDI_FAILURE); 1403 1404 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1405 (void) (*psm_intr_ops)(rdip, hdlp, 1406 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1407 1408 /* Remove the interrupt handler */ 1409 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1410 hdlp->ih_cb_func, hdlp->ih_vector); 1411 break; 1412 case DDI_INTROP_SETMASK: 1413 if (psm_intr_ops == NULL) 1414 return (DDI_FAILURE); 1415 1416 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1417 return (DDI_FAILURE); 1418 break; 1419 case DDI_INTROP_CLRMASK: 1420 if (psm_intr_ops == NULL) 1421 return (DDI_FAILURE); 1422 1423 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1424 return (DDI_FAILURE); 1425 break; 1426 case DDI_INTROP_GETPENDING: 1427 if (psm_intr_ops == NULL) 1428 return (DDI_FAILURE); 1429 1430 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1431 result)) { 1432 *(int *)result = 0; 1433 return (DDI_FAILURE); 1434 } 1435 break; 1436 case DDI_INTROP_NAVAIL: 1437 case DDI_INTROP_NINTRS: 1438 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1439 if (*(int *)result == 0) { 1440 /* 1441 * Special case for 'pcic' driver' only. This driver 1442 * driver is a child of 'isa' and 'rootnex' drivers. 1443 * 1444 * See detailed comments on this in the function 1445 * rootnex_get_ispec(). 1446 * 1447 * Children of 'pcic' send 'NINITR' request all the 1448 * way to rootnex driver. But, the 'pdp->par_nintr' 1449 * field may not initialized. So, we fake it here 1450 * to return 1 (a la what PCMCIA nexus does). 1451 */ 1452 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1453 *(int *)result = 1; 1454 else 1455 return (DDI_FAILURE); 1456 } 1457 break; 1458 case DDI_INTROP_SUPPORTED_TYPES: 1459 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1460 break; 1461 default: 1462 return (DDI_FAILURE); 1463 } 1464 1465 return (DDI_SUCCESS); 1466 } 1467 1468 1469 /* 1470 * rootnex_get_ispec() 1471 * convert an interrupt number to an interrupt specification. 1472 * The interrupt number determines which interrupt spec will be 1473 * returned if more than one exists. 1474 * 1475 * Look into the parent private data area of the 'rdip' to find out 1476 * the interrupt specification. First check to make sure there is 1477 * one that matchs "inumber" and then return a pointer to it. 1478 * 1479 * Return NULL if one could not be found. 1480 * 1481 * NOTE: This is needed for rootnex_intr_ops() 1482 */ 1483 static struct intrspec * 1484 rootnex_get_ispec(dev_info_t *rdip, int inum) 1485 { 1486 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1487 1488 /* 1489 * Special case handling for drivers that provide their own 1490 * intrspec structures instead of relying on the DDI framework. 1491 * 1492 * A broken hardware driver in ON could potentially provide its 1493 * own intrspec structure, instead of relying on the hardware. 1494 * If these drivers are children of 'rootnex' then we need to 1495 * continue to provide backward compatibility to them here. 1496 * 1497 * Following check is a special case for 'pcic' driver which 1498 * was found to have broken hardwre andby provides its own intrspec. 1499 * 1500 * Verbatim comments from this driver are shown here: 1501 * "Don't use the ddi_add_intr since we don't have a 1502 * default intrspec in all cases." 1503 * 1504 * Since an 'ispec' may not be always created for it, 1505 * check for that and create one if so. 1506 * 1507 * NOTE: Currently 'pcic' is the only driver found to do this. 1508 */ 1509 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1510 pdp->par_nintr = 1; 1511 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1512 pdp->par_nintr, KM_SLEEP); 1513 } 1514 1515 /* Validate the interrupt number */ 1516 if (inum >= pdp->par_nintr) 1517 return (NULL); 1518 1519 /* Get the interrupt structure pointer and return that */ 1520 return ((struct intrspec *)&pdp->par_intr[inum]); 1521 } 1522 1523 1524 /* 1525 * ****************** 1526 * dma related code 1527 * ****************** 1528 */ 1529 1530 /* 1531 * rootnex_dma_allochdl() 1532 * called from ddi_dma_alloc_handle(). 1533 */ 1534 /*ARGSUSED*/ 1535 static int 1536 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1537 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1538 { 1539 uint64_t maxsegmentsize_ll; 1540 uint_t maxsegmentsize; 1541 ddi_dma_impl_t *hp; 1542 rootnex_dma_t *dma; 1543 uint64_t count_max; 1544 uint64_t seg; 1545 int kmflag; 1546 int e; 1547 1548 1549 /* convert our sleep flags */ 1550 if (waitfp == DDI_DMA_SLEEP) { 1551 kmflag = KM_SLEEP; 1552 } else { 1553 kmflag = KM_NOSLEEP; 1554 } 1555 1556 /* 1557 * We try to do only one memory allocation here. We'll do a little 1558 * pointer manipulation later. If the bind ends up taking more than 1559 * our prealloc's space, we'll have to allocate more memory in the 1560 * bind operation. Not great, but much better than before and the 1561 * best we can do with the current bind interfaces. 1562 */ 1563 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1564 if (hp == NULL) { 1565 if (waitfp != DDI_DMA_DONTWAIT) { 1566 ddi_set_callback(waitfp, arg, 1567 &rootnex_state->r_dvma_call_list_id); 1568 } 1569 return (DDI_DMA_NORESOURCES); 1570 } 1571 1572 /* Do our pointer manipulation now, align the structures */ 1573 hp->dmai_private = (void *)(((uintptr_t)hp + 1574 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1575 dma = (rootnex_dma_t *)hp->dmai_private; 1576 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1577 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1578 1579 /* setup the handle */ 1580 rootnex_clean_dmahdl(hp); 1581 dma->dp_dip = rdip; 1582 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1583 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1584 hp->dmai_minxfer = attr->dma_attr_minxfer; 1585 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1586 hp->dmai_rdip = rdip; 1587 hp->dmai_attr = *attr; 1588 1589 /* we don't need to worry about the SPL since we do a tryenter */ 1590 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1591 1592 /* 1593 * Figure out our maximum segment size. If the segment size is greater 1594 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1595 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1596 * dma_attr_count_max are size-1 type values. 1597 * 1598 * Maximum segment size is the largest physically contiguous chunk of 1599 * memory that we can return from a bind (i.e. the maximum size of a 1600 * single cookie). 1601 */ 1602 1603 /* handle the rollover cases */ 1604 seg = attr->dma_attr_seg + 1; 1605 if (seg < attr->dma_attr_seg) { 1606 seg = attr->dma_attr_seg; 1607 } 1608 count_max = attr->dma_attr_count_max + 1; 1609 if (count_max < attr->dma_attr_count_max) { 1610 count_max = attr->dma_attr_count_max; 1611 } 1612 1613 /* 1614 * granularity may or may not be a power of two. If it isn't, we can't 1615 * use a simple mask. 1616 */ 1617 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1618 dma->dp_granularity_power_2 = B_FALSE; 1619 } else { 1620 dma->dp_granularity_power_2 = B_TRUE; 1621 } 1622 1623 /* 1624 * maxxfer should be a whole multiple of granularity. If we're going to 1625 * break up a window because we're greater than maxxfer, we might as 1626 * well make sure it's maxxfer is a whole multiple so we don't have to 1627 * worry about triming the window later on for this case. 1628 */ 1629 if (attr->dma_attr_granular > 1) { 1630 if (dma->dp_granularity_power_2) { 1631 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1632 (attr->dma_attr_maxxfer & 1633 (attr->dma_attr_granular - 1)); 1634 } else { 1635 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1636 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1637 } 1638 } else { 1639 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1640 } 1641 1642 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1643 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1644 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1645 maxsegmentsize = 0xFFFFFFFF; 1646 } else { 1647 maxsegmentsize = maxsegmentsize_ll; 1648 } 1649 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1650 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1651 1652 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1653 if (rootnex_alloc_check_parms) { 1654 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1655 if (e != DDI_SUCCESS) { 1656 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1657 (void) rootnex_dma_freehdl(dip, rdip, 1658 (ddi_dma_handle_t)hp); 1659 return (e); 1660 } 1661 } 1662 1663 *handlep = (ddi_dma_handle_t)hp; 1664 1665 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1666 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1667 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1668 1669 return (DDI_SUCCESS); 1670 } 1671 1672 1673 /* 1674 * rootnex_dma_freehdl() 1675 * called from ddi_dma_free_handle(). 1676 */ 1677 /*ARGSUSED*/ 1678 static int 1679 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1680 { 1681 ddi_dma_impl_t *hp; 1682 rootnex_dma_t *dma; 1683 1684 1685 hp = (ddi_dma_impl_t *)handle; 1686 dma = (rootnex_dma_t *)hp->dmai_private; 1687 1688 /* unbind should have been called first */ 1689 ASSERT(!dma->dp_inuse); 1690 1691 mutex_destroy(&dma->dp_mutex); 1692 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1693 1694 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1695 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1696 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1697 1698 if (rootnex_state->r_dvma_call_list_id) 1699 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1700 1701 return (DDI_SUCCESS); 1702 } 1703 1704 1705 /* 1706 * rootnex_dma_bindhdl() 1707 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1708 */ 1709 /*ARGSUSED*/ 1710 static int 1711 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1712 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1713 { 1714 rootnex_sglinfo_t *sinfo; 1715 ddi_dma_attr_t *attr; 1716 ddi_dma_impl_t *hp; 1717 rootnex_dma_t *dma; 1718 int kmflag; 1719 int e; 1720 1721 1722 hp = (ddi_dma_impl_t *)handle; 1723 dma = (rootnex_dma_t *)hp->dmai_private; 1724 sinfo = &dma->dp_sglinfo; 1725 attr = &hp->dmai_attr; 1726 1727 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1728 1729 /* 1730 * This is useful for debugging a driver. Not as useful in a production 1731 * system. The only time this will fail is if you have a driver bug. 1732 */ 1733 if (rootnex_bind_check_inuse) { 1734 /* 1735 * No one else should ever have this lock unless someone else 1736 * is trying to use this handle. So contention on the lock 1737 * is the same as inuse being set. 1738 */ 1739 e = mutex_tryenter(&dma->dp_mutex); 1740 if (e == 0) { 1741 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1742 return (DDI_DMA_INUSE); 1743 } 1744 if (dma->dp_inuse) { 1745 mutex_exit(&dma->dp_mutex); 1746 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1747 return (DDI_DMA_INUSE); 1748 } 1749 dma->dp_inuse = B_TRUE; 1750 mutex_exit(&dma->dp_mutex); 1751 } 1752 1753 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1754 if (rootnex_bind_check_parms) { 1755 e = rootnex_valid_bind_parms(dmareq, attr); 1756 if (e != DDI_SUCCESS) { 1757 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1758 rootnex_clean_dmahdl(hp); 1759 return (e); 1760 } 1761 } 1762 1763 /* save away the original bind info */ 1764 dma->dp_dma = dmareq->dmar_object; 1765 1766 /* 1767 * Figure out a rough estimate of what maximum number of pages this 1768 * buffer could use (a high estimate of course). 1769 */ 1770 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1771 1772 /* 1773 * We'll use the pre-allocated cookies for any bind that will *always* 1774 * fit (more important to be consistent, we don't want to create 1775 * additional degenerate cases). 1776 */ 1777 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1778 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1779 dma->dp_need_to_free_cookie = B_FALSE; 1780 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1781 uint_t, sinfo->si_max_pages); 1782 1783 /* 1784 * For anything larger than that, we'll go ahead and allocate the 1785 * maximum number of pages we expect to see. Hopefuly, we won't be 1786 * seeing this path in the fast path for high performance devices very 1787 * frequently. 1788 * 1789 * a ddi bind interface that allowed the driver to provide storage to 1790 * the bind interface would speed this case up. 1791 */ 1792 } else { 1793 /* convert the sleep flags */ 1794 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1795 kmflag = KM_SLEEP; 1796 } else { 1797 kmflag = KM_NOSLEEP; 1798 } 1799 1800 /* 1801 * Save away how much memory we allocated. If we're doing a 1802 * nosleep, the alloc could fail... 1803 */ 1804 dma->dp_cookie_size = sinfo->si_max_pages * 1805 sizeof (ddi_dma_cookie_t); 1806 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1807 if (dma->dp_cookies == NULL) { 1808 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1809 rootnex_clean_dmahdl(hp); 1810 return (DDI_DMA_NORESOURCES); 1811 } 1812 dma->dp_need_to_free_cookie = B_TRUE; 1813 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1814 sinfo->si_max_pages); 1815 } 1816 hp->dmai_cookie = dma->dp_cookies; 1817 1818 /* 1819 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1820 * looking at the contraints in the dma structure. It will then put some 1821 * additional state about the sgl in the dma struct (i.e. is the sgl 1822 * clean, or do we need to do some munging; how many pages need to be 1823 * copied, etc.) 1824 */ 1825 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1826 &dma->dp_sglinfo); 1827 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1828 1829 /* if we don't need a copy buffer, we don't need to sync */ 1830 if (sinfo->si_copybuf_req == 0) { 1831 hp->dmai_rflags |= DMP_NOSYNC; 1832 } 1833 1834 /* 1835 * If the driver supports FMA, insert the handle in the FMA DMA handle 1836 * cache. 1837 */ 1838 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1839 hp->dmai_error.err_cf = rootnex_dma_check; 1840 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1841 } 1842 1843 /* 1844 * if we don't need the copybuf and we don't need to do a partial, we 1845 * hit the fast path. All the high performance devices should be trying 1846 * to hit this path. To hit this path, a device should be able to reach 1847 * all of memory, shouldn't try to bind more than it can transfer, and 1848 * the buffer shouldn't require more cookies than the driver/device can 1849 * handle [sgllen]). 1850 */ 1851 if ((sinfo->si_copybuf_req == 0) && 1852 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1853 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1854 /* 1855 * copy out the first cookie and ccountp, set the cookie 1856 * pointer to the second cookie. The first cookie is passed 1857 * back on the stack. Additional cookies are accessed via 1858 * ddi_dma_nextcookie() 1859 */ 1860 *cookiep = dma->dp_cookies[0]; 1861 *ccountp = sinfo->si_sgl_size; 1862 hp->dmai_cookie++; 1863 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1864 hp->dmai_nwin = 1; 1865 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1866 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1867 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1868 dma->dp_dma.dmao_size); 1869 return (DDI_DMA_MAPPED); 1870 } 1871 1872 /* 1873 * go to the slow path, we may need to alloc more memory, create 1874 * multiple windows, and munge up a sgl to make the device happy. 1875 */ 1876 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 1877 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 1878 if (dma->dp_need_to_free_cookie) { 1879 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1880 } 1881 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1882 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 1883 return (e); 1884 } 1885 1886 /* if the first window uses the copy buffer, sync it for the device */ 1887 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 1888 (hp->dmai_rflags & DDI_DMA_WRITE)) { 1889 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1890 DDI_DMA_SYNC_FORDEV); 1891 } 1892 1893 /* 1894 * copy out the first cookie and ccountp, set the cookie pointer to the 1895 * second cookie. Make sure the partial flag is set/cleared correctly. 1896 * If we have a partial map (i.e. multiple windows), the number of 1897 * cookies we return is the number of cookies in the first window. 1898 */ 1899 if (e == DDI_DMA_MAPPED) { 1900 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1901 *ccountp = sinfo->si_sgl_size; 1902 } else { 1903 hp->dmai_rflags |= DDI_DMA_PARTIAL; 1904 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 1905 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 1906 } 1907 *cookiep = dma->dp_cookies[0]; 1908 hp->dmai_cookie++; 1909 1910 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1911 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 1912 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1913 dma->dp_dma.dmao_size); 1914 return (e); 1915 } 1916 1917 1918 /* 1919 * rootnex_dma_unbindhdl() 1920 * called from ddi_dma_unbind_handle() 1921 */ 1922 /*ARGSUSED*/ 1923 static int 1924 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 1925 ddi_dma_handle_t handle) 1926 { 1927 ddi_dma_impl_t *hp; 1928 rootnex_dma_t *dma; 1929 int e; 1930 1931 1932 hp = (ddi_dma_impl_t *)handle; 1933 dma = (rootnex_dma_t *)hp->dmai_private; 1934 1935 /* make sure the buffer wasn't free'd before calling unbind */ 1936 if (rootnex_unbind_verify_buffer) { 1937 e = rootnex_verify_buffer(dma); 1938 if (e != DDI_SUCCESS) { 1939 ASSERT(0); 1940 return (DDI_FAILURE); 1941 } 1942 } 1943 1944 /* sync the current window before unbinding the buffer */ 1945 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 1946 (hp->dmai_rflags & DDI_DMA_READ)) { 1947 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1948 DDI_DMA_SYNC_FORCPU); 1949 } 1950 1951 /* 1952 * If the driver supports FMA, remove the handle in the FMA DMA handle 1953 * cache. 1954 */ 1955 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 1956 if ((DEVI(rdip)->devi_fmhdl != NULL) && 1957 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 1958 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 1959 } 1960 } 1961 1962 /* 1963 * cleanup and copy buffer or window state. if we didn't use the copy 1964 * buffer or windows, there won't be much to do :-) 1965 */ 1966 rootnex_teardown_copybuf(dma); 1967 rootnex_teardown_windows(dma); 1968 1969 /* 1970 * If we had to allocate space to for the worse case sgl (it didn't 1971 * fit into our pre-allocate buffer), free that up now 1972 */ 1973 if (dma->dp_need_to_free_cookie) { 1974 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1975 } 1976 1977 /* 1978 * clean up the handle so it's ready for the next bind (i.e. if the 1979 * handle is reused). 1980 */ 1981 rootnex_clean_dmahdl(hp); 1982 1983 if (rootnex_state->r_dvma_call_list_id) 1984 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1985 1986 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1987 DTRACE_PROBE1(rootnex__unbind, uint64_t, 1988 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1989 1990 return (DDI_SUCCESS); 1991 } 1992 1993 1994 /* 1995 * rootnex_verify_buffer() 1996 * verify buffer wasn't free'd 1997 */ 1998 static int 1999 rootnex_verify_buffer(rootnex_dma_t *dma) 2000 { 2001 page_t **pplist; 2002 caddr_t vaddr; 2003 uint_t pcnt; 2004 uint_t poff; 2005 page_t *pp; 2006 char b; 2007 int i; 2008 2009 /* Figure out how many pages this buffer occupies */ 2010 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2011 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2012 } else { 2013 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2014 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2015 } 2016 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2017 2018 switch (dma->dp_dma.dmao_type) { 2019 case DMA_OTYP_PAGES: 2020 /* 2021 * for a linked list of pp's walk through them to make sure 2022 * they're locked and not free. 2023 */ 2024 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2025 for (i = 0; i < pcnt; i++) { 2026 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2027 return (DDI_FAILURE); 2028 } 2029 pp = pp->p_next; 2030 } 2031 break; 2032 2033 case DMA_OTYP_VADDR: 2034 case DMA_OTYP_BUFVADDR: 2035 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2036 /* 2037 * for an array of pp's walk through them to make sure they're 2038 * not free. It's possible that they may not be locked. 2039 */ 2040 if (pplist) { 2041 for (i = 0; i < pcnt; i++) { 2042 if (PP_ISFREE(pplist[i])) { 2043 return (DDI_FAILURE); 2044 } 2045 } 2046 2047 /* For a virtual address, try to peek at each page */ 2048 } else { 2049 if (dma->dp_sglinfo.si_asp == &kas) { 2050 for (i = 0; i < pcnt; i++) { 2051 if (ddi_peek8(NULL, vaddr, &b) == 2052 DDI_FAILURE) 2053 return (DDI_FAILURE); 2054 vaddr += MMU_PAGESIZE; 2055 } 2056 } 2057 } 2058 break; 2059 2060 default: 2061 ASSERT(0); 2062 break; 2063 } 2064 2065 return (DDI_SUCCESS); 2066 } 2067 2068 2069 /* 2070 * rootnex_clean_dmahdl() 2071 * Clean the dma handle. This should be called on a handle alloc and an 2072 * unbind handle. Set the handle state to the default settings. 2073 */ 2074 static void 2075 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2076 { 2077 rootnex_dma_t *dma; 2078 2079 2080 dma = (rootnex_dma_t *)hp->dmai_private; 2081 2082 hp->dmai_nwin = 0; 2083 dma->dp_current_cookie = 0; 2084 dma->dp_copybuf_size = 0; 2085 dma->dp_window = NULL; 2086 dma->dp_cbaddr = NULL; 2087 dma->dp_inuse = B_FALSE; 2088 dma->dp_need_to_free_cookie = B_FALSE; 2089 dma->dp_need_to_free_window = B_FALSE; 2090 dma->dp_partial_required = B_FALSE; 2091 dma->dp_trim_required = B_FALSE; 2092 dma->dp_sglinfo.si_copybuf_req = 0; 2093 #if !defined(__amd64) 2094 dma->dp_cb_remaping = B_FALSE; 2095 dma->dp_kva = NULL; 2096 #endif 2097 2098 /* FMA related initialization */ 2099 hp->dmai_fault = 0; 2100 hp->dmai_fault_check = NULL; 2101 hp->dmai_fault_notify = NULL; 2102 hp->dmai_error.err_ena = 0; 2103 hp->dmai_error.err_status = DDI_FM_OK; 2104 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2105 hp->dmai_error.err_ontrap = NULL; 2106 hp->dmai_error.err_fep = NULL; 2107 hp->dmai_error.err_cf = NULL; 2108 } 2109 2110 2111 /* 2112 * rootnex_valid_alloc_parms() 2113 * Called in ddi_dma_alloc_handle path to validate its parameters. 2114 */ 2115 static int 2116 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2117 { 2118 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2119 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2120 (attr->dma_attr_granular > MMU_PAGESIZE) || 2121 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2122 return (DDI_DMA_BADATTR); 2123 } 2124 2125 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2126 return (DDI_DMA_BADATTR); 2127 } 2128 2129 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2130 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2131 attr->dma_attr_sgllen <= 0) { 2132 return (DDI_DMA_BADATTR); 2133 } 2134 2135 /* We should be able to DMA into every byte offset in a page */ 2136 if (maxsegmentsize < MMU_PAGESIZE) { 2137 return (DDI_DMA_BADATTR); 2138 } 2139 2140 return (DDI_SUCCESS); 2141 } 2142 2143 2144 /* 2145 * rootnex_valid_bind_parms() 2146 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2147 */ 2148 /* ARGSUSED */ 2149 static int 2150 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2151 { 2152 #if !defined(__amd64) 2153 /* 2154 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2155 * we can track the offset for the obsoleted interfaces. 2156 */ 2157 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2158 return (DDI_DMA_TOOBIG); 2159 } 2160 #endif 2161 2162 return (DDI_SUCCESS); 2163 } 2164 2165 2166 /* 2167 * rootnex_get_sgl() 2168 * Called in bind fastpath to get the sgl. Most of this will be replaced 2169 * with a call to the vm layer when vm2.0 comes around... 2170 */ 2171 static void 2172 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2173 rootnex_sglinfo_t *sglinfo) 2174 { 2175 ddi_dma_atyp_t buftype; 2176 rootnex_addr_t raddr; 2177 uint64_t last_page; 2178 uint64_t offset; 2179 uint64_t addrhi; 2180 uint64_t addrlo; 2181 uint64_t maxseg; 2182 page_t **pplist; 2183 uint64_t paddr; 2184 uint32_t psize; 2185 uint32_t size; 2186 caddr_t vaddr; 2187 uint_t pcnt; 2188 page_t *pp; 2189 uint_t cnt; 2190 2191 2192 /* shortcuts */ 2193 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2194 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2195 maxseg = sglinfo->si_max_cookie_size; 2196 buftype = dmar_object->dmao_type; 2197 addrhi = sglinfo->si_max_addr; 2198 addrlo = sglinfo->si_min_addr; 2199 size = dmar_object->dmao_size; 2200 2201 pcnt = 0; 2202 cnt = 0; 2203 2204 /* 2205 * if we were passed down a linked list of pages, i.e. pointer to 2206 * page_t, use this to get our physical address and buf offset. 2207 */ 2208 if (buftype == DMA_OTYP_PAGES) { 2209 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2210 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2211 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2212 MMU_PAGEOFFSET; 2213 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2214 psize = MIN(size, (MMU_PAGESIZE - offset)); 2215 pp = pp->p_next; 2216 sglinfo->si_asp = NULL; 2217 2218 /* 2219 * We weren't passed down a linked list of pages, but if we were passed 2220 * down an array of pages, use this to get our physical address and buf 2221 * offset. 2222 */ 2223 } else if (pplist != NULL) { 2224 ASSERT((buftype == DMA_OTYP_VADDR) || 2225 (buftype == DMA_OTYP_BUFVADDR)); 2226 2227 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2228 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2229 if (sglinfo->si_asp == NULL) { 2230 sglinfo->si_asp = &kas; 2231 } 2232 2233 ASSERT(!PP_ISFREE(pplist[pcnt])); 2234 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2235 paddr += offset; 2236 psize = MIN(size, (MMU_PAGESIZE - offset)); 2237 pcnt++; 2238 2239 /* 2240 * All we have is a virtual address, we'll need to call into the VM 2241 * to get the physical address. 2242 */ 2243 } else { 2244 ASSERT((buftype == DMA_OTYP_VADDR) || 2245 (buftype == DMA_OTYP_BUFVADDR)); 2246 2247 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2248 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2249 if (sglinfo->si_asp == NULL) { 2250 sglinfo->si_asp = &kas; 2251 } 2252 2253 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2254 paddr += offset; 2255 psize = MIN(size, (MMU_PAGESIZE - offset)); 2256 vaddr += psize; 2257 } 2258 2259 #ifdef __xpv 2260 /* 2261 * If we're dom0, we're using a real device so we need to load 2262 * the cookies with MFNs instead of PFNs. 2263 */ 2264 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2265 #else 2266 raddr = paddr; 2267 #endif 2268 2269 /* 2270 * Setup the first cookie with the physical address of the page and the 2271 * size of the page (which takes into account the initial offset into 2272 * the page. 2273 */ 2274 sgl[cnt].dmac_laddress = raddr; 2275 sgl[cnt].dmac_size = psize; 2276 sgl[cnt].dmac_type = 0; 2277 2278 /* 2279 * Save away the buffer offset into the page. We'll need this later in 2280 * the copy buffer code to help figure out the page index within the 2281 * buffer and the offset into the current page. 2282 */ 2283 sglinfo->si_buf_offset = offset; 2284 2285 /* 2286 * If the DMA engine can't reach the physical address, increase how 2287 * much copy buffer we need. We always increase by pagesize so we don't 2288 * have to worry about converting offsets. Set a flag in the cookies 2289 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2290 * last cookie, go to the next cookie (since we separate each page which 2291 * uses the copy buffer in case the copy buffer is not physically 2292 * contiguous. 2293 */ 2294 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2295 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2296 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2297 if ((cnt + 1) < sglinfo->si_max_pages) { 2298 cnt++; 2299 sgl[cnt].dmac_laddress = 0; 2300 sgl[cnt].dmac_size = 0; 2301 sgl[cnt].dmac_type = 0; 2302 } 2303 } 2304 2305 /* 2306 * save this page's physical address so we can figure out if the next 2307 * page is physically contiguous. Keep decrementing size until we are 2308 * done with the buffer. 2309 */ 2310 last_page = raddr & MMU_PAGEMASK; 2311 size -= psize; 2312 2313 while (size > 0) { 2314 /* Get the size for this page (i.e. partial or full page) */ 2315 psize = MIN(size, MMU_PAGESIZE); 2316 2317 if (buftype == DMA_OTYP_PAGES) { 2318 /* get the paddr from the page_t */ 2319 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2320 paddr = pfn_to_pa(pp->p_pagenum); 2321 pp = pp->p_next; 2322 } else if (pplist != NULL) { 2323 /* index into the array of page_t's to get the paddr */ 2324 ASSERT(!PP_ISFREE(pplist[pcnt])); 2325 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2326 pcnt++; 2327 } else { 2328 /* call into the VM to get the paddr */ 2329 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2330 vaddr)); 2331 vaddr += psize; 2332 } 2333 2334 #ifdef __xpv 2335 /* 2336 * If we're dom0, we're using a real device so we need to load 2337 * the cookies with MFNs instead of PFNs. 2338 */ 2339 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2340 #else 2341 raddr = paddr; 2342 #endif 2343 2344 /* check to see if this page needs the copy buffer */ 2345 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2346 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2347 2348 /* 2349 * if there is something in the current cookie, go to 2350 * the next one. We only want one page in a cookie which 2351 * uses the copybuf since the copybuf doesn't have to 2352 * be physically contiguous. 2353 */ 2354 if (sgl[cnt].dmac_size != 0) { 2355 cnt++; 2356 } 2357 sgl[cnt].dmac_laddress = raddr; 2358 sgl[cnt].dmac_size = psize; 2359 #if defined(__amd64) 2360 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2361 #else 2362 /* 2363 * save the buf offset for 32-bit kernel. used in the 2364 * obsoleted interfaces. 2365 */ 2366 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2367 (dmar_object->dmao_size - size); 2368 #endif 2369 /* if this isn't the last cookie, go to the next one */ 2370 if ((cnt + 1) < sglinfo->si_max_pages) { 2371 cnt++; 2372 sgl[cnt].dmac_laddress = 0; 2373 sgl[cnt].dmac_size = 0; 2374 sgl[cnt].dmac_type = 0; 2375 } 2376 2377 /* 2378 * this page didn't need the copy buffer, if it's not physically 2379 * contiguous, or it would put us over a segment boundary, or it 2380 * puts us over the max cookie size, or the current sgl doesn't 2381 * have anything in it. 2382 */ 2383 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2384 !(raddr & sglinfo->si_segmask) || 2385 ((sgl[cnt].dmac_size + psize) > maxseg) || 2386 (sgl[cnt].dmac_size == 0)) { 2387 /* 2388 * if we're not already in a new cookie, go to the next 2389 * cookie. 2390 */ 2391 if (sgl[cnt].dmac_size != 0) { 2392 cnt++; 2393 } 2394 2395 /* save the cookie information */ 2396 sgl[cnt].dmac_laddress = raddr; 2397 sgl[cnt].dmac_size = psize; 2398 #if defined(__amd64) 2399 sgl[cnt].dmac_type = 0; 2400 #else 2401 /* 2402 * save the buf offset for 32-bit kernel. used in the 2403 * obsoleted interfaces. 2404 */ 2405 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2406 #endif 2407 2408 /* 2409 * this page didn't need the copy buffer, it is physically 2410 * contiguous with the last page, and it's <= the max cookie 2411 * size. 2412 */ 2413 } else { 2414 sgl[cnt].dmac_size += psize; 2415 2416 /* 2417 * if this exactly == the maximum cookie size, and 2418 * it isn't the last cookie, go to the next cookie. 2419 */ 2420 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2421 ((cnt + 1) < sglinfo->si_max_pages)) { 2422 cnt++; 2423 sgl[cnt].dmac_laddress = 0; 2424 sgl[cnt].dmac_size = 0; 2425 sgl[cnt].dmac_type = 0; 2426 } 2427 } 2428 2429 /* 2430 * save this page's physical address so we can figure out if the 2431 * next page is physically contiguous. Keep decrementing size 2432 * until we are done with the buffer. 2433 */ 2434 last_page = raddr; 2435 size -= psize; 2436 } 2437 2438 /* we're done, save away how many cookies the sgl has */ 2439 if (sgl[cnt].dmac_size == 0) { 2440 ASSERT(cnt < sglinfo->si_max_pages); 2441 sglinfo->si_sgl_size = cnt; 2442 } else { 2443 sglinfo->si_sgl_size = cnt + 1; 2444 } 2445 } 2446 2447 2448 /* 2449 * rootnex_bind_slowpath() 2450 * Call in the bind path if the calling driver can't use the sgl without 2451 * modifying it. We either need to use the copy buffer and/or we will end up 2452 * with a partial bind. 2453 */ 2454 static int 2455 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2456 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2457 { 2458 rootnex_sglinfo_t *sinfo; 2459 rootnex_window_t *window; 2460 ddi_dma_cookie_t *cookie; 2461 size_t copybuf_used; 2462 size_t dmac_size; 2463 boolean_t partial; 2464 off_t cur_offset; 2465 page_t *cur_pp; 2466 major_t mnum; 2467 int e; 2468 int i; 2469 2470 2471 sinfo = &dma->dp_sglinfo; 2472 copybuf_used = 0; 2473 partial = B_FALSE; 2474 2475 /* 2476 * If we're using the copybuf, set the copybuf state in dma struct. 2477 * Needs to be first since it sets the copy buffer size. 2478 */ 2479 if (sinfo->si_copybuf_req != 0) { 2480 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2481 if (e != DDI_SUCCESS) { 2482 return (e); 2483 } 2484 } else { 2485 dma->dp_copybuf_size = 0; 2486 } 2487 2488 /* 2489 * Figure out if we need to do a partial mapping. If so, figure out 2490 * if we need to trim the buffers when we munge the sgl. 2491 */ 2492 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2493 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2494 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2495 dma->dp_partial_required = B_TRUE; 2496 if (attr->dma_attr_granular != 1) { 2497 dma->dp_trim_required = B_TRUE; 2498 } 2499 } else { 2500 dma->dp_partial_required = B_FALSE; 2501 dma->dp_trim_required = B_FALSE; 2502 } 2503 2504 /* If we need to do a partial bind, make sure the driver supports it */ 2505 if (dma->dp_partial_required && 2506 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2507 2508 mnum = ddi_driver_major(dma->dp_dip); 2509 /* 2510 * patchable which allows us to print one warning per major 2511 * number. 2512 */ 2513 if ((rootnex_bind_warn) && 2514 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2515 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2516 cmn_err(CE_WARN, "!%s: coding error detected, the " 2517 "driver is using ddi_dma_attr(9S) incorrectly. " 2518 "There is a small risk of data corruption in " 2519 "particular with large I/Os. The driver should be " 2520 "replaced with a corrected version for proper " 2521 "system operation. To disable this warning, add " 2522 "'set rootnex:rootnex_bind_warn=0' to " 2523 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2524 } 2525 return (DDI_DMA_TOOBIG); 2526 } 2527 2528 /* 2529 * we might need multiple windows, setup state to handle them. In this 2530 * code path, we will have at least one window. 2531 */ 2532 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2533 if (e != DDI_SUCCESS) { 2534 rootnex_teardown_copybuf(dma); 2535 return (e); 2536 } 2537 2538 window = &dma->dp_window[0]; 2539 cookie = &dma->dp_cookies[0]; 2540 cur_offset = 0; 2541 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2542 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2543 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2544 } 2545 2546 /* loop though all the cookies we got back from get_sgl() */ 2547 for (i = 0; i < sinfo->si_sgl_size; i++) { 2548 /* 2549 * If we're using the copy buffer, check this cookie and setup 2550 * its associated copy buffer state. If this cookie uses the 2551 * copy buffer, make sure we sync this window during dma_sync. 2552 */ 2553 if (dma->dp_copybuf_size > 0) { 2554 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2555 cur_offset, ©buf_used, &cur_pp); 2556 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2557 window->wd_dosync = B_TRUE; 2558 } 2559 } 2560 2561 /* 2562 * save away the cookie size, since it could be modified in 2563 * the windowing code. 2564 */ 2565 dmac_size = cookie->dmac_size; 2566 2567 /* if we went over max copybuf size */ 2568 if (dma->dp_copybuf_size && 2569 (copybuf_used > dma->dp_copybuf_size)) { 2570 partial = B_TRUE; 2571 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2572 cookie, cur_offset, ©buf_used); 2573 if (e != DDI_SUCCESS) { 2574 rootnex_teardown_copybuf(dma); 2575 rootnex_teardown_windows(dma); 2576 return (e); 2577 } 2578 2579 /* 2580 * if the coookie uses the copy buffer, make sure the 2581 * new window we just moved to is set to sync. 2582 */ 2583 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2584 window->wd_dosync = B_TRUE; 2585 } 2586 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2587 dma->dp_dip); 2588 2589 /* if the cookie cnt == max sgllen, move to the next window */ 2590 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2591 partial = B_TRUE; 2592 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2593 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2594 cookie, attr, cur_offset); 2595 if (e != DDI_SUCCESS) { 2596 rootnex_teardown_copybuf(dma); 2597 rootnex_teardown_windows(dma); 2598 return (e); 2599 } 2600 2601 /* 2602 * if the coookie uses the copy buffer, make sure the 2603 * new window we just moved to is set to sync. 2604 */ 2605 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2606 window->wd_dosync = B_TRUE; 2607 } 2608 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2609 dma->dp_dip); 2610 2611 /* else if we will be over maxxfer */ 2612 } else if ((window->wd_size + dmac_size) > 2613 dma->dp_maxxfer) { 2614 partial = B_TRUE; 2615 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2616 cookie); 2617 if (e != DDI_SUCCESS) { 2618 rootnex_teardown_copybuf(dma); 2619 rootnex_teardown_windows(dma); 2620 return (e); 2621 } 2622 2623 /* 2624 * if the coookie uses the copy buffer, make sure the 2625 * new window we just moved to is set to sync. 2626 */ 2627 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2628 window->wd_dosync = B_TRUE; 2629 } 2630 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2631 dma->dp_dip); 2632 2633 /* else this cookie fits in the current window */ 2634 } else { 2635 window->wd_cookie_cnt++; 2636 window->wd_size += dmac_size; 2637 } 2638 2639 /* track our offset into the buffer, go to the next cookie */ 2640 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2641 ASSERT(cookie->dmac_size <= dmac_size); 2642 cur_offset += dmac_size; 2643 cookie++; 2644 } 2645 2646 /* if we ended up with a zero sized window in the end, clean it up */ 2647 if (window->wd_size == 0) { 2648 hp->dmai_nwin--; 2649 window--; 2650 } 2651 2652 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2653 2654 if (!partial) { 2655 return (DDI_DMA_MAPPED); 2656 } 2657 2658 ASSERT(dma->dp_partial_required); 2659 return (DDI_DMA_PARTIAL_MAP); 2660 } 2661 2662 2663 /* 2664 * rootnex_setup_copybuf() 2665 * Called in bind slowpath. Figures out if we're going to use the copy 2666 * buffer, and if we do, sets up the basic state to handle it. 2667 */ 2668 static int 2669 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2670 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2671 { 2672 rootnex_sglinfo_t *sinfo; 2673 ddi_dma_attr_t lattr; 2674 size_t max_copybuf; 2675 int cansleep; 2676 int e; 2677 #if !defined(__amd64) 2678 int vmflag; 2679 #endif 2680 2681 2682 sinfo = &dma->dp_sglinfo; 2683 2684 /* 2685 * read this first so it's consistent through the routine so we can 2686 * patch it on the fly. 2687 */ 2688 max_copybuf = rootnex_max_copybuf_size & MMU_PAGEMASK; 2689 2690 /* We need to call into the rootnex on ddi_dma_sync() */ 2691 hp->dmai_rflags &= ~DMP_NOSYNC; 2692 2693 /* make sure the copybuf size <= the max size */ 2694 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2695 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2696 2697 #if !defined(__amd64) 2698 /* 2699 * if we don't have kva space to copy to/from, allocate the KVA space 2700 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2701 * the 64-bit kernel. 2702 */ 2703 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2704 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2705 2706 /* convert the sleep flags */ 2707 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2708 vmflag = VM_SLEEP; 2709 } else { 2710 vmflag = VM_NOSLEEP; 2711 } 2712 2713 /* allocate Kernel VA space that we can bcopy to/from */ 2714 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2715 vmflag); 2716 if (dma->dp_kva == NULL) { 2717 return (DDI_DMA_NORESOURCES); 2718 } 2719 } 2720 #endif 2721 2722 /* convert the sleep flags */ 2723 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2724 cansleep = 1; 2725 } else { 2726 cansleep = 0; 2727 } 2728 2729 /* 2730 * Allocated the actual copy buffer. This needs to fit within the DMA 2731 * engines limits, so we can't use kmem_alloc... 2732 */ 2733 lattr = *attr; 2734 lattr.dma_attr_align = MMU_PAGESIZE; 2735 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2736 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2737 if (e != DDI_SUCCESS) { 2738 #if !defined(__amd64) 2739 if (dma->dp_kva != NULL) { 2740 vmem_free(heap_arena, dma->dp_kva, 2741 dma->dp_copybuf_size); 2742 } 2743 #endif 2744 return (DDI_DMA_NORESOURCES); 2745 } 2746 2747 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2748 size_t, dma->dp_copybuf_size); 2749 2750 return (DDI_SUCCESS); 2751 } 2752 2753 2754 /* 2755 * rootnex_setup_windows() 2756 * Called in bind slowpath to setup the window state. We always have windows 2757 * in the slowpath. Even if the window count = 1. 2758 */ 2759 static int 2760 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2761 ddi_dma_attr_t *attr, int kmflag) 2762 { 2763 rootnex_window_t *windowp; 2764 rootnex_sglinfo_t *sinfo; 2765 size_t copy_state_size; 2766 size_t win_state_size; 2767 size_t state_available; 2768 size_t space_needed; 2769 uint_t copybuf_win; 2770 uint_t maxxfer_win; 2771 size_t space_used; 2772 uint_t sglwin; 2773 2774 2775 sinfo = &dma->dp_sglinfo; 2776 2777 dma->dp_current_win = 0; 2778 hp->dmai_nwin = 0; 2779 2780 /* If we don't need to do a partial, we only have one window */ 2781 if (!dma->dp_partial_required) { 2782 dma->dp_max_win = 1; 2783 2784 /* 2785 * we need multiple windows, need to figure out the worse case number 2786 * of windows. 2787 */ 2788 } else { 2789 /* 2790 * if we need windows because we need more copy buffer that 2791 * we allow, the worse case number of windows we could need 2792 * here would be (copybuf space required / copybuf space that 2793 * we have) plus one for remainder, and plus 2 to handle the 2794 * extra pages on the trim for the first and last pages of the 2795 * buffer (a page is the minimum window size so under the right 2796 * attr settings, you could have a window for each page). 2797 * The last page will only be hit here if the size is not a 2798 * multiple of the granularity (which theoretically shouldn't 2799 * be the case but never has been enforced, so we could have 2800 * broken things without it). 2801 */ 2802 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2803 ASSERT(dma->dp_copybuf_size > 0); 2804 copybuf_win = (sinfo->si_copybuf_req / 2805 dma->dp_copybuf_size) + 1 + 2; 2806 } else { 2807 copybuf_win = 0; 2808 } 2809 2810 /* 2811 * if we need windows because we have more cookies than the H/W 2812 * can handle, the number of windows we would need here would 2813 * be (cookie count / cookies count H/W supports) plus one for 2814 * remainder, and plus 2 to handle the extra pages on the trim 2815 * (see above comment about trim) 2816 */ 2817 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2818 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2819 + 1) + 2; 2820 } else { 2821 sglwin = 0; 2822 } 2823 2824 /* 2825 * if we need windows because we're binding more memory than the 2826 * H/W can transfer at once, the number of windows we would need 2827 * here would be (xfer count / max xfer H/W supports) plus one 2828 * for remainder, and plus 2 to handle the extra pages on the 2829 * trim (see above comment about trim) 2830 */ 2831 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 2832 maxxfer_win = (dma->dp_dma.dmao_size / 2833 dma->dp_maxxfer) + 1 + 2; 2834 } else { 2835 maxxfer_win = 0; 2836 } 2837 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 2838 ASSERT(dma->dp_max_win > 0); 2839 } 2840 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 2841 2842 /* 2843 * Get space for window and potential copy buffer state. Before we 2844 * go and allocate memory, see if we can get away with using what's 2845 * left in the pre-allocted state or the dynamically allocated sgl. 2846 */ 2847 space_used = (uintptr_t)(sinfo->si_sgl_size * 2848 sizeof (ddi_dma_cookie_t)); 2849 2850 /* if we dynamically allocated space for the cookies */ 2851 if (dma->dp_need_to_free_cookie) { 2852 /* if we have more space in the pre-allocted buffer, use it */ 2853 ASSERT(space_used <= dma->dp_cookie_size); 2854 if ((dma->dp_cookie_size - space_used) <= 2855 rootnex_state->r_prealloc_size) { 2856 state_available = rootnex_state->r_prealloc_size; 2857 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 2858 2859 /* 2860 * else, we have more free space in the dynamically allocated 2861 * buffer, i.e. the buffer wasn't worse case fragmented so we 2862 * didn't need a lot of cookies. 2863 */ 2864 } else { 2865 state_available = dma->dp_cookie_size - space_used; 2866 windowp = (rootnex_window_t *) 2867 &dma->dp_cookies[sinfo->si_sgl_size]; 2868 } 2869 2870 /* we used the pre-alloced buffer */ 2871 } else { 2872 ASSERT(space_used <= rootnex_state->r_prealloc_size); 2873 state_available = rootnex_state->r_prealloc_size - space_used; 2874 windowp = (rootnex_window_t *) 2875 &dma->dp_cookies[sinfo->si_sgl_size]; 2876 } 2877 2878 /* 2879 * figure out how much state we need to track the copy buffer. Add an 2880 * addition 8 bytes for pointer alignemnt later. 2881 */ 2882 if (dma->dp_copybuf_size > 0) { 2883 copy_state_size = sinfo->si_max_pages * 2884 sizeof (rootnex_pgmap_t); 2885 } else { 2886 copy_state_size = 0; 2887 } 2888 /* add an additional 8 bytes for pointer alignment */ 2889 space_needed = win_state_size + copy_state_size + 0x8; 2890 2891 /* if we have enough space already, use it */ 2892 if (state_available >= space_needed) { 2893 dma->dp_window = windowp; 2894 dma->dp_need_to_free_window = B_FALSE; 2895 2896 /* not enough space, need to allocate more. */ 2897 } else { 2898 dma->dp_window = kmem_alloc(space_needed, kmflag); 2899 if (dma->dp_window == NULL) { 2900 return (DDI_DMA_NORESOURCES); 2901 } 2902 dma->dp_need_to_free_window = B_TRUE; 2903 dma->dp_window_size = space_needed; 2904 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 2905 dma->dp_dip, size_t, space_needed); 2906 } 2907 2908 /* 2909 * we allocate copy buffer state and window state at the same time. 2910 * setup our copy buffer state pointers. Make sure it's aligned. 2911 */ 2912 if (dma->dp_copybuf_size > 0) { 2913 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 2914 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 2915 2916 #if !defined(__amd64) 2917 /* 2918 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 2919 * false/NULL. Should be quicker to bzero vs loop and set. 2920 */ 2921 bzero(dma->dp_pgmap, copy_state_size); 2922 #endif 2923 } else { 2924 dma->dp_pgmap = NULL; 2925 } 2926 2927 return (DDI_SUCCESS); 2928 } 2929 2930 2931 /* 2932 * rootnex_teardown_copybuf() 2933 * cleans up after rootnex_setup_copybuf() 2934 */ 2935 static void 2936 rootnex_teardown_copybuf(rootnex_dma_t *dma) 2937 { 2938 #if !defined(__amd64) 2939 int i; 2940 2941 /* 2942 * if we allocated kernel heap VMEM space, go through all the pages and 2943 * map out any of the ones that we're mapped into the kernel heap VMEM 2944 * arena. Then free the VMEM space. 2945 */ 2946 if (dma->dp_kva != NULL) { 2947 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 2948 if (dma->dp_pgmap[i].pm_mapped) { 2949 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 2950 MMU_PAGESIZE, HAT_UNLOAD); 2951 dma->dp_pgmap[i].pm_mapped = B_FALSE; 2952 } 2953 } 2954 2955 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 2956 } 2957 2958 #endif 2959 2960 /* if we allocated a copy buffer, free it */ 2961 if (dma->dp_cbaddr != NULL) { 2962 i_ddi_mem_free(dma->dp_cbaddr, NULL); 2963 } 2964 } 2965 2966 2967 /* 2968 * rootnex_teardown_windows() 2969 * cleans up after rootnex_setup_windows() 2970 */ 2971 static void 2972 rootnex_teardown_windows(rootnex_dma_t *dma) 2973 { 2974 /* 2975 * if we had to allocate window state on the last bind (because we 2976 * didn't have enough pre-allocated space in the handle), free it. 2977 */ 2978 if (dma->dp_need_to_free_window) { 2979 kmem_free(dma->dp_window, dma->dp_window_size); 2980 } 2981 } 2982 2983 2984 /* 2985 * rootnex_init_win() 2986 * Called in bind slow path during creation of a new window. Initializes 2987 * window state to default values. 2988 */ 2989 /*ARGSUSED*/ 2990 static void 2991 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2992 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 2993 { 2994 hp->dmai_nwin++; 2995 window->wd_dosync = B_FALSE; 2996 window->wd_offset = cur_offset; 2997 window->wd_size = 0; 2998 window->wd_first_cookie = cookie; 2999 window->wd_cookie_cnt = 0; 3000 window->wd_trim.tr_trim_first = B_FALSE; 3001 window->wd_trim.tr_trim_last = B_FALSE; 3002 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3003 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3004 #if !defined(__amd64) 3005 window->wd_remap_copybuf = dma->dp_cb_remaping; 3006 #endif 3007 } 3008 3009 3010 /* 3011 * rootnex_setup_cookie() 3012 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3013 * the sgl uses the copy buffer, we need to go through each cookie, figure 3014 * out if it uses the copy buffer, and if it does, save away everything we'll 3015 * need during sync. 3016 */ 3017 static void 3018 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3019 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3020 page_t **cur_pp) 3021 { 3022 boolean_t copybuf_sz_power_2; 3023 rootnex_sglinfo_t *sinfo; 3024 paddr_t paddr; 3025 uint_t pidx; 3026 uint_t pcnt; 3027 off_t poff; 3028 #if defined(__amd64) 3029 pfn_t pfn; 3030 #else 3031 page_t **pplist; 3032 #endif 3033 3034 sinfo = &dma->dp_sglinfo; 3035 3036 /* 3037 * Calculate the page index relative to the start of the buffer. The 3038 * index to the current page for our buffer is the offset into the 3039 * first page of the buffer plus our current offset into the buffer 3040 * itself, shifted of course... 3041 */ 3042 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3043 ASSERT(pidx < sinfo->si_max_pages); 3044 3045 /* if this cookie uses the copy buffer */ 3046 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3047 /* 3048 * NOTE: we know that since this cookie uses the copy buffer, it 3049 * is <= MMU_PAGESIZE. 3050 */ 3051 3052 /* 3053 * get the offset into the page. For the 64-bit kernel, get the 3054 * pfn which we'll use with seg kpm. 3055 */ 3056 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3057 #if defined(__amd64) 3058 /* mfn_to_pfn() is a NOP on i86pc */ 3059 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3060 #endif /* __amd64 */ 3061 3062 /* figure out if the copybuf size is a power of 2 */ 3063 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3064 copybuf_sz_power_2 = B_FALSE; 3065 } else { 3066 copybuf_sz_power_2 = B_TRUE; 3067 } 3068 3069 /* This page uses the copy buffer */ 3070 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3071 3072 /* 3073 * save the copy buffer KVA that we'll use with this page. 3074 * if we still fit within the copybuf, it's a simple add. 3075 * otherwise, we need to wrap over using & or % accordingly. 3076 */ 3077 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3078 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3079 *copybuf_used; 3080 } else { 3081 if (copybuf_sz_power_2) { 3082 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3083 (uintptr_t)dma->dp_cbaddr + 3084 (*copybuf_used & 3085 (dma->dp_copybuf_size - 1))); 3086 } else { 3087 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3088 (uintptr_t)dma->dp_cbaddr + 3089 (*copybuf_used % dma->dp_copybuf_size)); 3090 } 3091 } 3092 3093 /* 3094 * over write the cookie physical address with the address of 3095 * the physical address of the copy buffer page that we will 3096 * use. 3097 */ 3098 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3099 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3100 3101 #ifdef __xpv 3102 /* 3103 * If we're dom0, we're using a real device so we need to load 3104 * the cookies with MAs instead of PAs. 3105 */ 3106 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3107 #else 3108 cookie->dmac_laddress = paddr; 3109 #endif 3110 3111 /* if we have a kernel VA, it's easy, just save that address */ 3112 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3113 (sinfo->si_asp == &kas)) { 3114 /* 3115 * save away the page aligned virtual address of the 3116 * driver buffer. Offsets are handled in the sync code. 3117 */ 3118 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3119 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3120 & MMU_PAGEMASK); 3121 #if !defined(__amd64) 3122 /* 3123 * we didn't need to, and will never need to map this 3124 * page. 3125 */ 3126 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3127 #endif 3128 3129 /* we don't have a kernel VA. We need one for the bcopy. */ 3130 } else { 3131 #if defined(__amd64) 3132 /* 3133 * for the 64-bit kernel, it's easy. We use seg kpm to 3134 * get a Kernel VA for the corresponding pfn. 3135 */ 3136 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3137 #else 3138 /* 3139 * for the 32-bit kernel, this is a pain. First we'll 3140 * save away the page_t or user VA for this page. This 3141 * is needed in rootnex_dma_win() when we switch to a 3142 * new window which requires us to re-map the copy 3143 * buffer. 3144 */ 3145 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3146 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3147 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3148 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3149 } else if (pplist != NULL) { 3150 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3151 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3152 } else { 3153 dma->dp_pgmap[pidx].pm_pp = NULL; 3154 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3155 (((uintptr_t) 3156 dmar_object->dmao_obj.virt_obj.v_addr + 3157 cur_offset) & MMU_PAGEMASK); 3158 } 3159 3160 /* 3161 * save away the page aligned virtual address which was 3162 * allocated from the kernel heap arena (taking into 3163 * account if we need more copy buffer than we alloced 3164 * and use multiple windows to handle this, i.e. &,%). 3165 * NOTE: there isn't and physical memory backing up this 3166 * virtual address space currently. 3167 */ 3168 if ((*copybuf_used + MMU_PAGESIZE) <= 3169 dma->dp_copybuf_size) { 3170 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3171 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3172 MMU_PAGEMASK); 3173 } else { 3174 if (copybuf_sz_power_2) { 3175 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3176 (((uintptr_t)dma->dp_kva + 3177 (*copybuf_used & 3178 (dma->dp_copybuf_size - 1))) & 3179 MMU_PAGEMASK); 3180 } else { 3181 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3182 (((uintptr_t)dma->dp_kva + 3183 (*copybuf_used % 3184 dma->dp_copybuf_size)) & 3185 MMU_PAGEMASK); 3186 } 3187 } 3188 3189 /* 3190 * if we haven't used up the available copy buffer yet, 3191 * map the kva to the physical page. 3192 */ 3193 if (!dma->dp_cb_remaping && ((*copybuf_used + 3194 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3195 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3196 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3197 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3198 dma->dp_pgmap[pidx].pm_kaddr); 3199 } else { 3200 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3201 sinfo->si_asp, 3202 dma->dp_pgmap[pidx].pm_kaddr); 3203 } 3204 3205 /* 3206 * we've used up the available copy buffer, this page 3207 * will have to be mapped during rootnex_dma_win() when 3208 * we switch to a new window which requires a re-map 3209 * the copy buffer. (32-bit kernel only) 3210 */ 3211 } else { 3212 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3213 } 3214 #endif 3215 /* go to the next page_t */ 3216 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3217 *cur_pp = (*cur_pp)->p_next; 3218 } 3219 } 3220 3221 /* add to the copy buffer count */ 3222 *copybuf_used += MMU_PAGESIZE; 3223 3224 /* 3225 * This cookie doesn't use the copy buffer. Walk through the pages this 3226 * cookie occupies to reflect this. 3227 */ 3228 } else { 3229 /* 3230 * figure out how many pages the cookie occupies. We need to 3231 * use the original page offset of the buffer and the cookies 3232 * offset in the buffer to do this. 3233 */ 3234 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3235 pcnt = mmu_btopr(cookie->dmac_size + poff); 3236 3237 while (pcnt > 0) { 3238 #if !defined(__amd64) 3239 /* 3240 * the 32-bit kernel doesn't have seg kpm, so we need 3241 * to map in the driver buffer (if it didn't come down 3242 * with a kernel VA) on the fly. Since this page doesn't 3243 * use the copy buffer, it's not, or will it ever, have 3244 * to be mapped in. 3245 */ 3246 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3247 #endif 3248 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3249 3250 /* 3251 * we need to update pidx and cur_pp or we'll loose 3252 * track of where we are. 3253 */ 3254 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3255 *cur_pp = (*cur_pp)->p_next; 3256 } 3257 pidx++; 3258 pcnt--; 3259 } 3260 } 3261 } 3262 3263 3264 /* 3265 * rootnex_sgllen_window_boundary() 3266 * Called in the bind slow path when the next cookie causes us to exceed (in 3267 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3268 * length supported by the DMA H/W. 3269 */ 3270 static int 3271 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3272 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3273 off_t cur_offset) 3274 { 3275 off_t new_offset; 3276 size_t trim_sz; 3277 off_t coffset; 3278 3279 3280 /* 3281 * if we know we'll never have to trim, it's pretty easy. Just move to 3282 * the next window and init it. We're done. 3283 */ 3284 if (!dma->dp_trim_required) { 3285 (*windowp)++; 3286 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3287 (*windowp)->wd_cookie_cnt++; 3288 (*windowp)->wd_size = cookie->dmac_size; 3289 return (DDI_SUCCESS); 3290 } 3291 3292 /* figure out how much we need to trim from the window */ 3293 ASSERT(attr->dma_attr_granular != 0); 3294 if (dma->dp_granularity_power_2) { 3295 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3296 } else { 3297 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3298 } 3299 3300 /* The window's a whole multiple of granularity. We're done */ 3301 if (trim_sz == 0) { 3302 (*windowp)++; 3303 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3304 (*windowp)->wd_cookie_cnt++; 3305 (*windowp)->wd_size = cookie->dmac_size; 3306 return (DDI_SUCCESS); 3307 } 3308 3309 /* 3310 * The window's not a whole multiple of granularity, since we know this 3311 * is due to the sgllen, we need to go back to the last cookie and trim 3312 * that one, add the left over part of the old cookie into the new 3313 * window, and then add in the new cookie into the new window. 3314 */ 3315 3316 /* 3317 * make sure the driver isn't making us do something bad... Trimming and 3318 * sgllen == 1 don't go together. 3319 */ 3320 if (attr->dma_attr_sgllen == 1) { 3321 return (DDI_DMA_NOMAPPING); 3322 } 3323 3324 /* 3325 * first, setup the current window to account for the trim. Need to go 3326 * back to the last cookie for this. 3327 */ 3328 cookie--; 3329 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3330 (*windowp)->wd_trim.tr_last_cookie = cookie; 3331 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3332 ASSERT(cookie->dmac_size > trim_sz); 3333 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3334 (*windowp)->wd_size -= trim_sz; 3335 3336 /* save the buffer offsets for the next window */ 3337 coffset = cookie->dmac_size - trim_sz; 3338 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3339 3340 /* 3341 * set this now in case this is the first window. all other cases are 3342 * set in dma_win() 3343 */ 3344 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3345 3346 /* 3347 * initialize the next window using what's left over in the previous 3348 * cookie. 3349 */ 3350 (*windowp)++; 3351 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3352 (*windowp)->wd_cookie_cnt++; 3353 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3354 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3355 (*windowp)->wd_trim.tr_first_size = trim_sz; 3356 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3357 (*windowp)->wd_dosync = B_TRUE; 3358 } 3359 3360 /* 3361 * now go back to the current cookie and add it to the new window. set 3362 * the new window size to the what was left over from the previous 3363 * cookie and what's in the current cookie. 3364 */ 3365 cookie++; 3366 (*windowp)->wd_cookie_cnt++; 3367 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3368 3369 /* 3370 * trim plus the next cookie could put us over maxxfer (a cookie can be 3371 * a max size of maxxfer). Handle that case. 3372 */ 3373 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3374 /* 3375 * maxxfer is already a whole multiple of granularity, and this 3376 * trim will be <= the previous trim (since a cookie can't be 3377 * larger than maxxfer). Make things simple here. 3378 */ 3379 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3380 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3381 (*windowp)->wd_trim.tr_last_cookie = cookie; 3382 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3383 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3384 (*windowp)->wd_size -= trim_sz; 3385 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3386 3387 /* save the buffer offsets for the next window */ 3388 coffset = cookie->dmac_size - trim_sz; 3389 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3390 3391 /* setup the next window */ 3392 (*windowp)++; 3393 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3394 (*windowp)->wd_cookie_cnt++; 3395 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3396 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3397 coffset; 3398 (*windowp)->wd_trim.tr_first_size = trim_sz; 3399 } 3400 3401 return (DDI_SUCCESS); 3402 } 3403 3404 3405 /* 3406 * rootnex_copybuf_window_boundary() 3407 * Called in bind slowpath when we get to a window boundary because we used 3408 * up all the copy buffer that we have. 3409 */ 3410 static int 3411 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3412 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3413 size_t *copybuf_used) 3414 { 3415 rootnex_sglinfo_t *sinfo; 3416 off_t new_offset; 3417 size_t trim_sz; 3418 paddr_t paddr; 3419 off_t coffset; 3420 uint_t pidx; 3421 off_t poff; 3422 3423 3424 sinfo = &dma->dp_sglinfo; 3425 3426 /* 3427 * the copy buffer should be a whole multiple of page size. We know that 3428 * this cookie is <= MMU_PAGESIZE. 3429 */ 3430 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3431 3432 /* 3433 * from now on, all new windows in this bind need to be re-mapped during 3434 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3435 * space... 3436 */ 3437 #if !defined(__amd64) 3438 dma->dp_cb_remaping = B_TRUE; 3439 #endif 3440 3441 /* reset copybuf used */ 3442 *copybuf_used = 0; 3443 3444 /* 3445 * if we don't have to trim (since granularity is set to 1), go to the 3446 * next window and add the current cookie to it. We know the current 3447 * cookie uses the copy buffer since we're in this code path. 3448 */ 3449 if (!dma->dp_trim_required) { 3450 (*windowp)++; 3451 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3452 3453 /* Add this cookie to the new window */ 3454 (*windowp)->wd_cookie_cnt++; 3455 (*windowp)->wd_size += cookie->dmac_size; 3456 *copybuf_used += MMU_PAGESIZE; 3457 return (DDI_SUCCESS); 3458 } 3459 3460 /* 3461 * *** may need to trim, figure it out. 3462 */ 3463 3464 /* figure out how much we need to trim from the window */ 3465 if (dma->dp_granularity_power_2) { 3466 trim_sz = (*windowp)->wd_size & 3467 (hp->dmai_attr.dma_attr_granular - 1); 3468 } else { 3469 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3470 } 3471 3472 /* 3473 * if the window's a whole multiple of granularity, go to the next 3474 * window, init it, then add in the current cookie. We know the current 3475 * cookie uses the copy buffer since we're in this code path. 3476 */ 3477 if (trim_sz == 0) { 3478 (*windowp)++; 3479 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3480 3481 /* Add this cookie to the new window */ 3482 (*windowp)->wd_cookie_cnt++; 3483 (*windowp)->wd_size += cookie->dmac_size; 3484 *copybuf_used += MMU_PAGESIZE; 3485 return (DDI_SUCCESS); 3486 } 3487 3488 /* 3489 * *** We figured it out, we definitly need to trim 3490 */ 3491 3492 /* 3493 * make sure the driver isn't making us do something bad... 3494 * Trimming and sgllen == 1 don't go together. 3495 */ 3496 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3497 return (DDI_DMA_NOMAPPING); 3498 } 3499 3500 /* 3501 * first, setup the current window to account for the trim. Need to go 3502 * back to the last cookie for this. Some of the last cookie will be in 3503 * the current window, and some of the last cookie will be in the new 3504 * window. All of the current cookie will be in the new window. 3505 */ 3506 cookie--; 3507 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3508 (*windowp)->wd_trim.tr_last_cookie = cookie; 3509 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3510 ASSERT(cookie->dmac_size > trim_sz); 3511 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3512 (*windowp)->wd_size -= trim_sz; 3513 3514 /* 3515 * we're trimming the last cookie (not the current cookie). So that 3516 * last cookie may have or may not have been using the copy buffer ( 3517 * we know the cookie passed in uses the copy buffer since we're in 3518 * this code path). 3519 * 3520 * If the last cookie doesn't use the copy buffer, nothing special to 3521 * do. However, if it does uses the copy buffer, it will be both the 3522 * last page in the current window and the first page in the next 3523 * window. Since we are reusing the copy buffer (and KVA space on the 3524 * 32-bit kernel), this page will use the end of the copy buffer in the 3525 * current window, and the start of the copy buffer in the next window. 3526 * Track that info... The cookie physical address was already set to 3527 * the copy buffer physical address in setup_cookie.. 3528 */ 3529 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3530 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3531 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3532 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3533 (*windowp)->wd_trim.tr_last_pidx = pidx; 3534 (*windowp)->wd_trim.tr_last_cbaddr = 3535 dma->dp_pgmap[pidx].pm_cbaddr; 3536 #if !defined(__amd64) 3537 (*windowp)->wd_trim.tr_last_kaddr = 3538 dma->dp_pgmap[pidx].pm_kaddr; 3539 #endif 3540 } 3541 3542 /* save the buffer offsets for the next window */ 3543 coffset = cookie->dmac_size - trim_sz; 3544 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3545 3546 /* 3547 * set this now in case this is the first window. all other cases are 3548 * set in dma_win() 3549 */ 3550 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3551 3552 /* 3553 * initialize the next window using what's left over in the previous 3554 * cookie. 3555 */ 3556 (*windowp)++; 3557 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3558 (*windowp)->wd_cookie_cnt++; 3559 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3560 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3561 (*windowp)->wd_trim.tr_first_size = trim_sz; 3562 3563 /* 3564 * again, we're tracking if the last cookie uses the copy buffer. 3565 * read the comment above for more info on why we need to track 3566 * additional state. 3567 * 3568 * For the first cookie in the new window, we need reset the physical 3569 * address to DMA into to the start of the copy buffer plus any 3570 * initial page offset which may be present. 3571 */ 3572 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3573 (*windowp)->wd_dosync = B_TRUE; 3574 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3575 (*windowp)->wd_trim.tr_first_pidx = pidx; 3576 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3577 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3578 3579 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3580 poff; 3581 #ifdef __xpv 3582 /* 3583 * If we're dom0, we're using a real device so we need to load 3584 * the cookies with MAs instead of PAs. 3585 */ 3586 (*windowp)->wd_trim.tr_first_paddr = 3587 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3588 #else 3589 (*windowp)->wd_trim.tr_first_paddr = paddr; 3590 #endif 3591 3592 #if !defined(__amd64) 3593 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3594 #endif 3595 /* account for the cookie copybuf usage in the new window */ 3596 *copybuf_used += MMU_PAGESIZE; 3597 3598 /* 3599 * every piece of code has to have a hack, and here is this 3600 * ones :-) 3601 * 3602 * There is a complex interaction between setup_cookie and the 3603 * copybuf window boundary. The complexity had to be in either 3604 * the maxxfer window, or the copybuf window, and I chose the 3605 * copybuf code. 3606 * 3607 * So in this code path, we have taken the last cookie, 3608 * virtually broken it in half due to the trim, and it happens 3609 * to use the copybuf which further complicates life. At the 3610 * same time, we have already setup the current cookie, which 3611 * is now wrong. More background info: the current cookie uses 3612 * the copybuf, so it is only a page long max. So we need to 3613 * fix the current cookies copy buffer address, physical 3614 * address, and kva for the 32-bit kernel. We due this by 3615 * bumping them by page size (of course, we can't due this on 3616 * the physical address since the copy buffer may not be 3617 * physically contiguous). 3618 */ 3619 cookie++; 3620 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3621 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3622 3623 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3624 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3625 #ifdef __xpv 3626 /* 3627 * If we're dom0, we're using a real device so we need to load 3628 * the cookies with MAs instead of PAs. 3629 */ 3630 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3631 #else 3632 cookie->dmac_laddress = paddr; 3633 #endif 3634 3635 #if !defined(__amd64) 3636 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3637 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3638 #endif 3639 } else { 3640 /* go back to the current cookie */ 3641 cookie++; 3642 } 3643 3644 /* 3645 * add the current cookie to the new window. set the new window size to 3646 * the what was left over from the previous cookie and what's in the 3647 * current cookie. 3648 */ 3649 (*windowp)->wd_cookie_cnt++; 3650 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3651 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3652 3653 /* 3654 * we know that the cookie passed in always uses the copy buffer. We 3655 * wouldn't be here if it didn't. 3656 */ 3657 *copybuf_used += MMU_PAGESIZE; 3658 3659 return (DDI_SUCCESS); 3660 } 3661 3662 3663 /* 3664 * rootnex_maxxfer_window_boundary() 3665 * Called in bind slowpath when we get to a window boundary because we will 3666 * go over maxxfer. 3667 */ 3668 static int 3669 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3670 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3671 { 3672 size_t dmac_size; 3673 off_t new_offset; 3674 size_t trim_sz; 3675 off_t coffset; 3676 3677 3678 /* 3679 * calculate how much we have to trim off of the current cookie to equal 3680 * maxxfer. We don't have to account for granularity here since our 3681 * maxxfer already takes that into account. 3682 */ 3683 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3684 ASSERT(trim_sz <= cookie->dmac_size); 3685 ASSERT(trim_sz <= dma->dp_maxxfer); 3686 3687 /* save cookie size since we need it later and we might change it */ 3688 dmac_size = cookie->dmac_size; 3689 3690 /* 3691 * if we're not trimming the entire cookie, setup the current window to 3692 * account for the trim. 3693 */ 3694 if (trim_sz < cookie->dmac_size) { 3695 (*windowp)->wd_cookie_cnt++; 3696 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3697 (*windowp)->wd_trim.tr_last_cookie = cookie; 3698 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3699 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3700 (*windowp)->wd_size = dma->dp_maxxfer; 3701 3702 /* 3703 * set the adjusted cookie size now in case this is the first 3704 * window. All other windows are taken care of in get win 3705 */ 3706 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3707 } 3708 3709 /* 3710 * coffset is the current offset within the cookie, new_offset is the 3711 * current offset with the entire buffer. 3712 */ 3713 coffset = dmac_size - trim_sz; 3714 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3715 3716 /* initialize the next window */ 3717 (*windowp)++; 3718 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3719 (*windowp)->wd_cookie_cnt++; 3720 (*windowp)->wd_size = trim_sz; 3721 if (trim_sz < dmac_size) { 3722 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3723 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3724 coffset; 3725 (*windowp)->wd_trim.tr_first_size = trim_sz; 3726 } 3727 3728 return (DDI_SUCCESS); 3729 } 3730 3731 3732 /* 3733 * rootnex_dma_sync() 3734 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3735 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3736 * is set, ddi_dma_sync() returns immediately passing back success. 3737 */ 3738 /*ARGSUSED*/ 3739 static int 3740 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3741 off_t off, size_t len, uint_t cache_flags) 3742 { 3743 rootnex_sglinfo_t *sinfo; 3744 rootnex_pgmap_t *cbpage; 3745 rootnex_window_t *win; 3746 ddi_dma_impl_t *hp; 3747 rootnex_dma_t *dma; 3748 caddr_t fromaddr; 3749 caddr_t toaddr; 3750 uint_t psize; 3751 off_t offset; 3752 uint_t pidx; 3753 size_t size; 3754 off_t poff; 3755 int e; 3756 3757 3758 hp = (ddi_dma_impl_t *)handle; 3759 dma = (rootnex_dma_t *)hp->dmai_private; 3760 sinfo = &dma->dp_sglinfo; 3761 3762 /* 3763 * if we don't have any windows, we don't need to sync. A copybuf 3764 * will cause us to have at least one window. 3765 */ 3766 if (dma->dp_window == NULL) { 3767 return (DDI_SUCCESS); 3768 } 3769 3770 /* This window may not need to be sync'd */ 3771 win = &dma->dp_window[dma->dp_current_win]; 3772 if (!win->wd_dosync) { 3773 return (DDI_SUCCESS); 3774 } 3775 3776 /* handle off and len special cases */ 3777 if ((off == 0) || (rootnex_sync_ignore_params)) { 3778 offset = win->wd_offset; 3779 } else { 3780 offset = off; 3781 } 3782 if ((len == 0) || (rootnex_sync_ignore_params)) { 3783 size = win->wd_size; 3784 } else { 3785 size = len; 3786 } 3787 3788 /* check the sync args to make sure they make a little sense */ 3789 if (rootnex_sync_check_parms) { 3790 e = rootnex_valid_sync_parms(hp, win, offset, size, 3791 cache_flags); 3792 if (e != DDI_SUCCESS) { 3793 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3794 return (DDI_FAILURE); 3795 } 3796 } 3797 3798 /* 3799 * special case the first page to handle the offset into the page. The 3800 * offset to the current page for our buffer is the offset into the 3801 * first page of the buffer plus our current offset into the buffer 3802 * itself, masked of course. 3803 */ 3804 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3805 psize = MIN((MMU_PAGESIZE - poff), size); 3806 3807 /* go through all the pages that we want to sync */ 3808 while (size > 0) { 3809 /* 3810 * Calculate the page index relative to the start of the buffer. 3811 * The index to the current page for our buffer is the offset 3812 * into the first page of the buffer plus our current offset 3813 * into the buffer itself, shifted of course... 3814 */ 3815 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3816 ASSERT(pidx < sinfo->si_max_pages); 3817 3818 /* 3819 * if this page uses the copy buffer, we need to sync it, 3820 * otherwise, go on to the next page. 3821 */ 3822 cbpage = &dma->dp_pgmap[pidx]; 3823 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3824 (cbpage->pm_uses_copybuf == B_FALSE)); 3825 if (cbpage->pm_uses_copybuf) { 3826 /* cbaddr and kaddr should be page aligned */ 3827 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3828 MMU_PAGEOFFSET) == 0); 3829 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3830 MMU_PAGEOFFSET) == 0); 3831 3832 /* 3833 * if we're copying for the device, we are going to 3834 * copy from the drivers buffer and to the rootnex 3835 * allocated copy buffer. 3836 */ 3837 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3838 fromaddr = cbpage->pm_kaddr + poff; 3839 toaddr = cbpage->pm_cbaddr + poff; 3840 DTRACE_PROBE2(rootnex__sync__dev, 3841 dev_info_t *, dma->dp_dip, size_t, psize); 3842 3843 /* 3844 * if we're copying for the cpu/kernel, we are going to 3845 * copy from the rootnex allocated copy buffer to the 3846 * drivers buffer. 3847 */ 3848 } else { 3849 fromaddr = cbpage->pm_cbaddr + poff; 3850 toaddr = cbpage->pm_kaddr + poff; 3851 DTRACE_PROBE2(rootnex__sync__cpu, 3852 dev_info_t *, dma->dp_dip, size_t, psize); 3853 } 3854 3855 bcopy(fromaddr, toaddr, psize); 3856 } 3857 3858 /* 3859 * decrement size until we're done, update our offset into the 3860 * buffer, and get the next page size. 3861 */ 3862 size -= psize; 3863 offset += psize; 3864 psize = MIN(MMU_PAGESIZE, size); 3865 3866 /* page offset is zero for the rest of this loop */ 3867 poff = 0; 3868 } 3869 3870 return (DDI_SUCCESS); 3871 } 3872 3873 3874 /* 3875 * rootnex_valid_sync_parms() 3876 * checks the parameters passed to sync to verify they are correct. 3877 */ 3878 static int 3879 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 3880 off_t offset, size_t size, uint_t cache_flags) 3881 { 3882 off_t woffset; 3883 3884 3885 /* 3886 * the first part of the test to make sure the offset passed in is 3887 * within the window. 3888 */ 3889 if (offset < win->wd_offset) { 3890 return (DDI_FAILURE); 3891 } 3892 3893 /* 3894 * second and last part of the test to make sure the offset and length 3895 * passed in is within the window. 3896 */ 3897 woffset = offset - win->wd_offset; 3898 if ((woffset + size) > win->wd_size) { 3899 return (DDI_FAILURE); 3900 } 3901 3902 /* 3903 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 3904 * be set too. 3905 */ 3906 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 3907 (hp->dmai_rflags & DDI_DMA_WRITE)) { 3908 return (DDI_SUCCESS); 3909 } 3910 3911 /* 3912 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 3913 * should be set. Also DDI_DMA_READ should be set in the flags. 3914 */ 3915 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 3916 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 3917 (hp->dmai_rflags & DDI_DMA_READ)) { 3918 return (DDI_SUCCESS); 3919 } 3920 3921 return (DDI_FAILURE); 3922 } 3923 3924 3925 /* 3926 * rootnex_dma_win() 3927 * called from ddi_dma_getwin() 3928 */ 3929 /*ARGSUSED*/ 3930 static int 3931 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3932 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 3933 uint_t *ccountp) 3934 { 3935 rootnex_window_t *window; 3936 rootnex_trim_t *trim; 3937 ddi_dma_impl_t *hp; 3938 rootnex_dma_t *dma; 3939 #if !defined(__amd64) 3940 rootnex_sglinfo_t *sinfo; 3941 rootnex_pgmap_t *pmap; 3942 uint_t pidx; 3943 uint_t pcnt; 3944 off_t poff; 3945 int i; 3946 #endif 3947 3948 3949 hp = (ddi_dma_impl_t *)handle; 3950 dma = (rootnex_dma_t *)hp->dmai_private; 3951 #if !defined(__amd64) 3952 sinfo = &dma->dp_sglinfo; 3953 #endif 3954 3955 /* If we try and get a window which doesn't exist, return failure */ 3956 if (win >= hp->dmai_nwin) { 3957 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3958 return (DDI_FAILURE); 3959 } 3960 3961 /* 3962 * if we don't have any windows, and they're asking for the first 3963 * window, setup the cookie pointer to the first cookie in the bind. 3964 * setup our return values, then increment the cookie since we return 3965 * the first cookie on the stack. 3966 */ 3967 if (dma->dp_window == NULL) { 3968 if (win != 0) { 3969 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3970 return (DDI_FAILURE); 3971 } 3972 hp->dmai_cookie = dma->dp_cookies; 3973 *offp = 0; 3974 *lenp = dma->dp_dma.dmao_size; 3975 *ccountp = dma->dp_sglinfo.si_sgl_size; 3976 *cookiep = hp->dmai_cookie[0]; 3977 hp->dmai_cookie++; 3978 return (DDI_SUCCESS); 3979 } 3980 3981 /* sync the old window before moving on to the new one */ 3982 window = &dma->dp_window[dma->dp_current_win]; 3983 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 3984 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 3985 DDI_DMA_SYNC_FORCPU); 3986 } 3987 3988 #if !defined(__amd64) 3989 /* 3990 * before we move to the next window, if we need to re-map, unmap all 3991 * the pages in this window. 3992 */ 3993 if (dma->dp_cb_remaping) { 3994 /* 3995 * If we switch to this window again, we'll need to map in 3996 * on the fly next time. 3997 */ 3998 window->wd_remap_copybuf = B_TRUE; 3999 4000 /* 4001 * calculate the page index into the buffer where this window 4002 * starts, and the number of pages this window takes up. 4003 */ 4004 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4005 MMU_PAGESHIFT; 4006 poff = (sinfo->si_buf_offset + window->wd_offset) & 4007 MMU_PAGEOFFSET; 4008 pcnt = mmu_btopr(window->wd_size + poff); 4009 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4010 4011 /* unmap pages which are currently mapped in this window */ 4012 for (i = 0; i < pcnt; i++) { 4013 if (dma->dp_pgmap[pidx].pm_mapped) { 4014 hat_unload(kas.a_hat, 4015 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4016 HAT_UNLOAD); 4017 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4018 } 4019 pidx++; 4020 } 4021 } 4022 #endif 4023 4024 /* 4025 * Move to the new window. 4026 * NOTE: current_win must be set for sync to work right 4027 */ 4028 dma->dp_current_win = win; 4029 window = &dma->dp_window[win]; 4030 4031 /* if needed, adjust the first and/or last cookies for trim */ 4032 trim = &window->wd_trim; 4033 if (trim->tr_trim_first) { 4034 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4035 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4036 #if !defined(__amd64) 4037 window->wd_first_cookie->dmac_type = 4038 (window->wd_first_cookie->dmac_type & 4039 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4040 #endif 4041 if (trim->tr_first_copybuf_win) { 4042 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4043 trim->tr_first_cbaddr; 4044 #if !defined(__amd64) 4045 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4046 trim->tr_first_kaddr; 4047 #endif 4048 } 4049 } 4050 if (trim->tr_trim_last) { 4051 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4052 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4053 if (trim->tr_last_copybuf_win) { 4054 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4055 trim->tr_last_cbaddr; 4056 #if !defined(__amd64) 4057 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4058 trim->tr_last_kaddr; 4059 #endif 4060 } 4061 } 4062 4063 /* 4064 * setup the cookie pointer to the first cookie in the window. setup 4065 * our return values, then increment the cookie since we return the 4066 * first cookie on the stack. 4067 */ 4068 hp->dmai_cookie = window->wd_first_cookie; 4069 *offp = window->wd_offset; 4070 *lenp = window->wd_size; 4071 *ccountp = window->wd_cookie_cnt; 4072 *cookiep = hp->dmai_cookie[0]; 4073 hp->dmai_cookie++; 4074 4075 #if !defined(__amd64) 4076 /* re-map copybuf if required for this window */ 4077 if (dma->dp_cb_remaping) { 4078 /* 4079 * calculate the page index into the buffer where this 4080 * window starts. 4081 */ 4082 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4083 MMU_PAGESHIFT; 4084 ASSERT(pidx < sinfo->si_max_pages); 4085 4086 /* 4087 * the first page can get unmapped if it's shared with the 4088 * previous window. Even if the rest of this window is already 4089 * mapped in, we need to still check this one. 4090 */ 4091 pmap = &dma->dp_pgmap[pidx]; 4092 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4093 if (pmap->pm_pp != NULL) { 4094 pmap->pm_mapped = B_TRUE; 4095 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4096 } else if (pmap->pm_vaddr != NULL) { 4097 pmap->pm_mapped = B_TRUE; 4098 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4099 pmap->pm_kaddr); 4100 } 4101 } 4102 pidx++; 4103 4104 /* map in the rest of the pages if required */ 4105 if (window->wd_remap_copybuf) { 4106 window->wd_remap_copybuf = B_FALSE; 4107 4108 /* figure out many pages this window takes up */ 4109 poff = (sinfo->si_buf_offset + window->wd_offset) & 4110 MMU_PAGEOFFSET; 4111 pcnt = mmu_btopr(window->wd_size + poff); 4112 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4113 4114 /* map pages which require it */ 4115 for (i = 1; i < pcnt; i++) { 4116 pmap = &dma->dp_pgmap[pidx]; 4117 if (pmap->pm_uses_copybuf) { 4118 ASSERT(pmap->pm_mapped == B_FALSE); 4119 if (pmap->pm_pp != NULL) { 4120 pmap->pm_mapped = B_TRUE; 4121 i86_pp_map(pmap->pm_pp, 4122 pmap->pm_kaddr); 4123 } else if (pmap->pm_vaddr != NULL) { 4124 pmap->pm_mapped = B_TRUE; 4125 i86_va_map(pmap->pm_vaddr, 4126 sinfo->si_asp, 4127 pmap->pm_kaddr); 4128 } 4129 } 4130 pidx++; 4131 } 4132 } 4133 } 4134 #endif 4135 4136 /* if the new window uses the copy buffer, sync it for the device */ 4137 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4138 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4139 DDI_DMA_SYNC_FORDEV); 4140 } 4141 4142 return (DDI_SUCCESS); 4143 } 4144 4145 4146 4147 /* 4148 * ************************ 4149 * obsoleted dma routines 4150 * ************************ 4151 */ 4152 4153 /* 4154 * rootnex_dma_map() 4155 * called from ddi_dma_setup() 4156 */ 4157 /* ARGSUSED */ 4158 static int 4159 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4160 ddi_dma_handle_t *handlep) 4161 { 4162 #if defined(__amd64) 4163 /* 4164 * this interface is not supported in 64-bit x86 kernel. See comment in 4165 * rootnex_dma_mctl() 4166 */ 4167 return (DDI_DMA_NORESOURCES); 4168 4169 #else /* 32-bit x86 kernel */ 4170 ddi_dma_handle_t *lhandlep; 4171 ddi_dma_handle_t lhandle; 4172 ddi_dma_cookie_t cookie; 4173 ddi_dma_attr_t dma_attr; 4174 ddi_dma_lim_t *dma_lim; 4175 uint_t ccnt; 4176 int e; 4177 4178 4179 /* 4180 * if the driver is just testing to see if it's possible to do the bind, 4181 * we'll use local state. Otherwise, use the handle pointer passed in. 4182 */ 4183 if (handlep == NULL) { 4184 lhandlep = &lhandle; 4185 } else { 4186 lhandlep = handlep; 4187 } 4188 4189 /* convert the limit structure to a dma_attr one */ 4190 dma_lim = dmareq->dmar_limits; 4191 dma_attr.dma_attr_version = DMA_ATTR_V0; 4192 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4193 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4194 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4195 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4196 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4197 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4198 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4199 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4200 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4201 dma_attr.dma_attr_align = MMU_PAGESIZE; 4202 dma_attr.dma_attr_flags = 0; 4203 4204 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4205 dmareq->dmar_arg, lhandlep); 4206 if (e != DDI_SUCCESS) { 4207 return (e); 4208 } 4209 4210 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4211 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4212 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4213 return (e); 4214 } 4215 4216 /* 4217 * if the driver is just testing to see if it's possible to do the bind, 4218 * free up the local state and return the result. 4219 */ 4220 if (handlep == NULL) { 4221 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4222 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4223 if (e == DDI_DMA_MAPPED) { 4224 return (DDI_DMA_MAPOK); 4225 } else { 4226 return (DDI_DMA_NOMAPPING); 4227 } 4228 } 4229 4230 return (e); 4231 #endif /* defined(__amd64) */ 4232 } 4233 4234 4235 /* 4236 * rootnex_dma_mctl() 4237 * 4238 */ 4239 /* ARGSUSED */ 4240 static int 4241 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4242 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4243 uint_t cache_flags) 4244 { 4245 #if defined(__amd64) 4246 /* 4247 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4248 * common implementation in genunix, so they no longer have x86 4249 * specific functionality which called into dma_ctl. 4250 * 4251 * The rest of the obsoleted interfaces were never supported in the 4252 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4253 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4254 * implementation issues. 4255 * 4256 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4257 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4258 * reflect that now too... 4259 * 4260 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4261 * not going to put this functionality into the 64-bit x86 kernel now. 4262 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4263 * that in a future release. 4264 */ 4265 return (DDI_FAILURE); 4266 4267 #else /* 32-bit x86 kernel */ 4268 ddi_dma_cookie_t lcookie; 4269 ddi_dma_cookie_t *cookie; 4270 rootnex_window_t *window; 4271 ddi_dma_impl_t *hp; 4272 rootnex_dma_t *dma; 4273 uint_t nwin; 4274 uint_t ccnt; 4275 size_t len; 4276 off_t off; 4277 int e; 4278 4279 4280 /* 4281 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4282 * hacky since were optimizing for the current interfaces and so we can 4283 * cleanup the mess in genunix. Hopefully we will remove the this 4284 * obsoleted routines someday soon. 4285 */ 4286 4287 switch (request) { 4288 4289 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4290 hp = (ddi_dma_impl_t *)handle; 4291 cookie = (ddi_dma_cookie_t *)objpp; 4292 4293 /* 4294 * convert segment to cookie. We don't distinguish between the 4295 * two :-) 4296 */ 4297 *cookie = *hp->dmai_cookie; 4298 *lenp = cookie->dmac_size; 4299 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4300 return (DDI_SUCCESS); 4301 4302 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4303 hp = (ddi_dma_impl_t *)handle; 4304 dma = (rootnex_dma_t *)hp->dmai_private; 4305 4306 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4307 return (DDI_DMA_STALE); 4308 } 4309 4310 /* handle the case where we don't have any windows */ 4311 if (dma->dp_window == NULL) { 4312 /* 4313 * if seg == NULL, and we don't have any windows, 4314 * return the first cookie in the sgl. 4315 */ 4316 if (*lenp == NULL) { 4317 dma->dp_current_cookie = 0; 4318 hp->dmai_cookie = dma->dp_cookies; 4319 *objpp = (caddr_t)handle; 4320 return (DDI_SUCCESS); 4321 4322 /* if we have more cookies, go to the next cookie */ 4323 } else { 4324 if ((dma->dp_current_cookie + 1) >= 4325 dma->dp_sglinfo.si_sgl_size) { 4326 return (DDI_DMA_DONE); 4327 } 4328 dma->dp_current_cookie++; 4329 hp->dmai_cookie++; 4330 return (DDI_SUCCESS); 4331 } 4332 } 4333 4334 /* We have one or more windows */ 4335 window = &dma->dp_window[dma->dp_current_win]; 4336 4337 /* 4338 * if seg == NULL, return the first cookie in the current 4339 * window 4340 */ 4341 if (*lenp == NULL) { 4342 dma->dp_current_cookie = 0; 4343 hp->dmai_cookie = window->wd_first_cookie; 4344 4345 /* 4346 * go to the next cookie in the window then see if we done with 4347 * this window. 4348 */ 4349 } else { 4350 if ((dma->dp_current_cookie + 1) >= 4351 window->wd_cookie_cnt) { 4352 return (DDI_DMA_DONE); 4353 } 4354 dma->dp_current_cookie++; 4355 hp->dmai_cookie++; 4356 } 4357 *objpp = (caddr_t)handle; 4358 return (DDI_SUCCESS); 4359 4360 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4361 hp = (ddi_dma_impl_t *)handle; 4362 dma = (rootnex_dma_t *)hp->dmai_private; 4363 4364 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4365 return (DDI_DMA_STALE); 4366 } 4367 4368 /* if win == NULL, return the first window in the bind */ 4369 if (*offp == NULL) { 4370 nwin = 0; 4371 4372 /* 4373 * else, go to the next window then see if we're done with all 4374 * the windows. 4375 */ 4376 } else { 4377 nwin = dma->dp_current_win + 1; 4378 if (nwin >= hp->dmai_nwin) { 4379 return (DDI_DMA_DONE); 4380 } 4381 } 4382 4383 /* switch to the next window */ 4384 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4385 &lcookie, &ccnt); 4386 ASSERT(e == DDI_SUCCESS); 4387 if (e != DDI_SUCCESS) { 4388 return (DDI_DMA_STALE); 4389 } 4390 4391 /* reset the cookie back to the first cookie in the window */ 4392 if (dma->dp_window != NULL) { 4393 window = &dma->dp_window[dma->dp_current_win]; 4394 hp->dmai_cookie = window->wd_first_cookie; 4395 } else { 4396 hp->dmai_cookie = dma->dp_cookies; 4397 } 4398 4399 *objpp = (caddr_t)handle; 4400 return (DDI_SUCCESS); 4401 4402 case DDI_DMA_FREE: /* ddi_dma_free() */ 4403 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4404 (void) rootnex_dma_freehdl(dip, rdip, handle); 4405 if (rootnex_state->r_dvma_call_list_id) { 4406 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4407 } 4408 return (DDI_SUCCESS); 4409 4410 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4411 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4412 /* should never get here, handled in genunix */ 4413 ASSERT(0); 4414 return (DDI_FAILURE); 4415 4416 case DDI_DMA_KVADDR: 4417 case DDI_DMA_GETERR: 4418 case DDI_DMA_COFF: 4419 return (DDI_FAILURE); 4420 } 4421 4422 return (DDI_FAILURE); 4423 #endif /* defined(__amd64) */ 4424 } 4425 4426 4427 /* 4428 * ********* 4429 * FMA Code 4430 * ********* 4431 */ 4432 4433 /* 4434 * rootnex_fm_init() 4435 * FMA init busop 4436 */ 4437 /* ARGSUSED */ 4438 static int 4439 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4440 ddi_iblock_cookie_t *ibc) 4441 { 4442 *ibc = rootnex_state->r_err_ibc; 4443 4444 return (ddi_system_fmcap); 4445 } 4446 4447 /* 4448 * rootnex_dma_check() 4449 * Function called after a dma fault occurred to find out whether the 4450 * fault address is associated with a driver that is able to handle faults 4451 * and recover from faults. 4452 */ 4453 /* ARGSUSED */ 4454 static int 4455 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4456 const void *not_used) 4457 { 4458 rootnex_window_t *window; 4459 uint64_t start_addr; 4460 uint64_t fault_addr; 4461 ddi_dma_impl_t *hp; 4462 rootnex_dma_t *dma; 4463 uint64_t end_addr; 4464 size_t csize; 4465 int i; 4466 int j; 4467 4468 4469 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4470 hp = (ddi_dma_impl_t *)handle; 4471 ASSERT(hp); 4472 4473 dma = (rootnex_dma_t *)hp->dmai_private; 4474 4475 /* Get the address that we need to search for */ 4476 fault_addr = *(uint64_t *)addr; 4477 4478 /* 4479 * if we don't have any windows, we can just walk through all the 4480 * cookies. 4481 */ 4482 if (dma->dp_window == NULL) { 4483 /* for each cookie */ 4484 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4485 /* 4486 * if the faulted address is within the physical address 4487 * range of the cookie, return DDI_FM_NONFATAL. 4488 */ 4489 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4490 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4491 dma->dp_cookies[i].dmac_size))) { 4492 return (DDI_FM_NONFATAL); 4493 } 4494 } 4495 4496 /* fault_addr not within this DMA handle */ 4497 return (DDI_FM_UNKNOWN); 4498 } 4499 4500 /* we have mutiple windows, walk through each window */ 4501 for (i = 0; i < hp->dmai_nwin; i++) { 4502 window = &dma->dp_window[i]; 4503 4504 /* Go through all the cookies in the window */ 4505 for (j = 0; j < window->wd_cookie_cnt; j++) { 4506 4507 start_addr = window->wd_first_cookie[j].dmac_laddress; 4508 csize = window->wd_first_cookie[j].dmac_size; 4509 4510 /* 4511 * if we are trimming the first cookie in the window, 4512 * and this is the first cookie, adjust the start 4513 * address and size of the cookie to account for the 4514 * trim. 4515 */ 4516 if (window->wd_trim.tr_trim_first && (j == 0)) { 4517 start_addr = window->wd_trim.tr_first_paddr; 4518 csize = window->wd_trim.tr_first_size; 4519 } 4520 4521 /* 4522 * if we are trimming the last cookie in the window, 4523 * and this is the last cookie, adjust the start 4524 * address and size of the cookie to account for the 4525 * trim. 4526 */ 4527 if (window->wd_trim.tr_trim_last && 4528 (j == (window->wd_cookie_cnt - 1))) { 4529 start_addr = window->wd_trim.tr_last_paddr; 4530 csize = window->wd_trim.tr_last_size; 4531 } 4532 4533 end_addr = start_addr + csize; 4534 4535 /* 4536 * if the faulted address is within the physical address 4537 * range of the cookie, return DDI_FM_NONFATAL. 4538 */ 4539 if ((fault_addr >= start_addr) && 4540 (fault_addr <= end_addr)) { 4541 return (DDI_FM_NONFATAL); 4542 } 4543 } 4544 } 4545 4546 /* fault_addr not within this DMA handle */ 4547 return (DDI_FM_UNKNOWN); 4548 } 4549