1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #endif 71 72 /* 73 * enable/disable extra checking of function parameters. Useful for debugging 74 * drivers. 75 */ 76 #ifdef DEBUG 77 int rootnex_alloc_check_parms = 1; 78 int rootnex_bind_check_parms = 1; 79 int rootnex_bind_check_inuse = 1; 80 int rootnex_unbind_verify_buffer = 0; 81 int rootnex_sync_check_parms = 1; 82 #else 83 int rootnex_alloc_check_parms = 0; 84 int rootnex_bind_check_parms = 0; 85 int rootnex_bind_check_inuse = 0; 86 int rootnex_unbind_verify_buffer = 0; 87 int rootnex_sync_check_parms = 0; 88 #endif 89 90 /* Master Abort and Target Abort panic flag */ 91 int rootnex_fm_ma_ta_panic_flag = 0; 92 93 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 94 int rootnex_bind_fail = 1; 95 int rootnex_bind_warn = 1; 96 uint8_t *rootnex_warn_list; 97 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 98 #define ROOTNEX_BIND_WARNING (0x1 << 0) 99 100 /* 101 * revert back to old broken behavior of always sync'ing entire copy buffer. 102 * This is useful if be have a buggy driver which doesn't correctly pass in 103 * the offset and size into ddi_dma_sync(). 104 */ 105 int rootnex_sync_ignore_params = 0; 106 107 /* 108 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 109 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 110 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 111 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 112 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 113 * (< 8K). We will still need to allocate the copy buffer during bind though 114 * (if we need one). These can only be modified in /etc/system before rootnex 115 * attach. 116 */ 117 #if defined(__amd64) 118 int rootnex_prealloc_cookies = 65; 119 int rootnex_prealloc_windows = 4; 120 int rootnex_prealloc_copybuf = 2; 121 #else 122 int rootnex_prealloc_cookies = 33; 123 int rootnex_prealloc_windows = 4; 124 int rootnex_prealloc_copybuf = 2; 125 #endif 126 127 /* driver global state */ 128 static rootnex_state_t *rootnex_state; 129 130 /* shortcut to rootnex counters */ 131 static uint64_t *rootnex_cnt; 132 133 /* 134 * XXX - does x86 even need these or are they left over from the SPARC days? 135 */ 136 /* statically defined integer/boolean properties for the root node */ 137 static rootnex_intprop_t rootnex_intprp[] = { 138 { "PAGESIZE", PAGESIZE }, 139 { "MMU_PAGESIZE", MMU_PAGESIZE }, 140 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 141 { DDI_RELATIVE_ADDRESSING, 1 }, 142 }; 143 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 144 145 #ifdef __xpv 146 typedef maddr_t rootnex_addr_t; 147 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 148 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 149 #else 150 typedef paddr_t rootnex_addr_t; 151 #endif 152 153 154 static struct cb_ops rootnex_cb_ops = { 155 nodev, /* open */ 156 nodev, /* close */ 157 nodev, /* strategy */ 158 nodev, /* print */ 159 nodev, /* dump */ 160 nodev, /* read */ 161 nodev, /* write */ 162 nodev, /* ioctl */ 163 nodev, /* devmap */ 164 nodev, /* mmap */ 165 nodev, /* segmap */ 166 nochpoll, /* chpoll */ 167 ddi_prop_op, /* cb_prop_op */ 168 NULL, /* struct streamtab */ 169 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 170 CB_REV, /* Rev */ 171 nodev, /* cb_aread */ 172 nodev /* cb_awrite */ 173 }; 174 175 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 176 off_t offset, off_t len, caddr_t *vaddrp); 177 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 178 struct hat *hat, struct seg *seg, caddr_t addr, 179 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 180 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 181 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 182 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 183 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 184 ddi_dma_handle_t *handlep); 185 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 186 ddi_dma_handle_t handle); 187 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 188 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 189 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 190 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 191 ddi_dma_handle_t handle); 192 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 193 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 194 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 195 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 196 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 197 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 198 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 199 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 200 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 201 ddi_ctl_enum_t ctlop, void *arg, void *result); 202 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 203 ddi_iblock_cookie_t *ibc); 204 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 205 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 206 207 208 static struct bus_ops rootnex_bus_ops = { 209 BUSO_REV, 210 rootnex_map, 211 NULL, 212 NULL, 213 NULL, 214 rootnex_map_fault, 215 rootnex_dma_map, 216 rootnex_dma_allochdl, 217 rootnex_dma_freehdl, 218 rootnex_dma_bindhdl, 219 rootnex_dma_unbindhdl, 220 rootnex_dma_sync, 221 rootnex_dma_win, 222 rootnex_dma_mctl, 223 rootnex_ctlops, 224 ddi_bus_prop_op, 225 i_ddi_rootnex_get_eventcookie, 226 i_ddi_rootnex_add_eventcall, 227 i_ddi_rootnex_remove_eventcall, 228 i_ddi_rootnex_post_event, 229 0, /* bus_intr_ctl */ 230 0, /* bus_config */ 231 0, /* bus_unconfig */ 232 rootnex_fm_init, /* bus_fm_init */ 233 NULL, /* bus_fm_fini */ 234 NULL, /* bus_fm_access_enter */ 235 NULL, /* bus_fm_access_exit */ 236 NULL, /* bus_powr */ 237 rootnex_intr_ops /* bus_intr_op */ 238 }; 239 240 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 241 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 242 243 static struct dev_ops rootnex_ops = { 244 DEVO_REV, 245 0, 246 ddi_no_info, 247 nulldev, 248 nulldev, 249 rootnex_attach, 250 rootnex_detach, 251 nulldev, 252 &rootnex_cb_ops, 253 &rootnex_bus_ops 254 }; 255 256 static struct modldrv rootnex_modldrv = { 257 &mod_driverops, 258 "i86pc root nexus", 259 &rootnex_ops 260 }; 261 262 static struct modlinkage rootnex_modlinkage = { 263 MODREV_1, 264 (void *)&rootnex_modldrv, 265 NULL 266 }; 267 268 269 /* 270 * extern hacks 271 */ 272 extern struct seg_ops segdev_ops; 273 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 274 #ifdef DDI_MAP_DEBUG 275 extern int ddi_map_debug_flag; 276 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 277 #endif 278 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 279 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 280 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 281 psm_intr_op_t, int *); 282 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 283 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 284 285 /* 286 * Use device arena to use for device control register mappings. 287 * Various kernel memory walkers (debugger, dtrace) need to know 288 * to avoid this address range to prevent undesired device activity. 289 */ 290 extern void *device_arena_alloc(size_t size, int vm_flag); 291 extern void device_arena_free(void * vaddr, size_t size); 292 293 294 /* 295 * Internal functions 296 */ 297 static int rootnex_dma_init(); 298 static void rootnex_add_props(dev_info_t *); 299 static int rootnex_ctl_reportdev(dev_info_t *dip); 300 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 301 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 302 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 303 static int rootnex_map_handle(ddi_map_req_t *mp); 304 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 305 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 306 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 307 ddi_dma_attr_t *attr); 308 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 309 rootnex_sglinfo_t *sglinfo); 310 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 311 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 312 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 313 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 314 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 315 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 316 ddi_dma_attr_t *attr, int kmflag); 317 static void rootnex_teardown_windows(rootnex_dma_t *dma); 318 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 319 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 320 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 321 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 322 size_t *copybuf_used, page_t **cur_pp); 323 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 324 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 325 ddi_dma_attr_t *attr, off_t cur_offset); 326 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 327 rootnex_dma_t *dma, rootnex_window_t **windowp, 328 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 329 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 330 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 331 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 332 off_t offset, size_t size, uint_t cache_flags); 333 static int rootnex_verify_buffer(rootnex_dma_t *dma); 334 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 335 const void *comp_addr, const void *not_used); 336 337 /* 338 * _init() 339 * 340 */ 341 int 342 _init(void) 343 { 344 345 rootnex_state = NULL; 346 return (mod_install(&rootnex_modlinkage)); 347 } 348 349 350 /* 351 * _info() 352 * 353 */ 354 int 355 _info(struct modinfo *modinfop) 356 { 357 return (mod_info(&rootnex_modlinkage, modinfop)); 358 } 359 360 361 /* 362 * _fini() 363 * 364 */ 365 int 366 _fini(void) 367 { 368 return (EBUSY); 369 } 370 371 372 /* 373 * rootnex_attach() 374 * 375 */ 376 static int 377 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 378 { 379 int fmcap; 380 int e; 381 382 switch (cmd) { 383 case DDI_ATTACH: 384 break; 385 case DDI_RESUME: 386 return (DDI_SUCCESS); 387 default: 388 return (DDI_FAILURE); 389 } 390 391 /* 392 * We should only have one instance of rootnex. Save it away since we 393 * don't have an easy way to get it back later. 394 */ 395 ASSERT(rootnex_state == NULL); 396 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 397 398 rootnex_state->r_dip = dip; 399 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 400 rootnex_state->r_reserved_msg_printed = B_FALSE; 401 rootnex_cnt = &rootnex_state->r_counters[0]; 402 403 /* 404 * Set minimum fm capability level for i86pc platforms and then 405 * initialize error handling. Since we're the rootnex, we don't 406 * care what's returned in the fmcap field. 407 */ 408 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 409 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 410 fmcap = ddi_system_fmcap; 411 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 412 413 /* initialize DMA related state */ 414 e = rootnex_dma_init(); 415 if (e != DDI_SUCCESS) { 416 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 417 return (DDI_FAILURE); 418 } 419 420 /* Add static root node properties */ 421 rootnex_add_props(dip); 422 423 /* since we can't call ddi_report_dev() */ 424 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 425 426 /* Initialize rootnex event handle */ 427 i_ddi_rootnex_init_events(dip); 428 429 return (DDI_SUCCESS); 430 } 431 432 433 /* 434 * rootnex_detach() 435 * 436 */ 437 /*ARGSUSED*/ 438 static int 439 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 440 { 441 switch (cmd) { 442 case DDI_SUSPEND: 443 break; 444 default: 445 return (DDI_FAILURE); 446 } 447 448 return (DDI_SUCCESS); 449 } 450 451 452 /* 453 * rootnex_dma_init() 454 * 455 */ 456 /*ARGSUSED*/ 457 static int 458 rootnex_dma_init() 459 { 460 size_t bufsize; 461 462 463 /* 464 * size of our cookie/window/copybuf state needed in dma bind that we 465 * pre-alloc in dma_alloc_handle 466 */ 467 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 468 rootnex_state->r_prealloc_size = 469 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 470 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 471 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 472 473 /* 474 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 475 * allocate 16 extra bytes for struct pointer alignment 476 * (p->dmai_private & dma->dp_prealloc_buffer) 477 */ 478 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 479 rootnex_state->r_prealloc_size + 0x10; 480 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 481 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 482 if (rootnex_state->r_dmahdl_cache == NULL) { 483 return (DDI_FAILURE); 484 } 485 486 /* 487 * allocate array to track which major numbers we have printed warnings 488 * for. 489 */ 490 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 491 KM_SLEEP); 492 493 return (DDI_SUCCESS); 494 } 495 496 497 /* 498 * rootnex_add_props() 499 * 500 */ 501 static void 502 rootnex_add_props(dev_info_t *dip) 503 { 504 rootnex_intprop_t *rpp; 505 int i; 506 507 /* Add static integer/boolean properties to the root node */ 508 rpp = rootnex_intprp; 509 for (i = 0; i < NROOT_INTPROPS; i++) { 510 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 511 rpp[i].prop_name, rpp[i].prop_value); 512 } 513 } 514 515 516 517 /* 518 * ************************* 519 * ctlops related routines 520 * ************************* 521 */ 522 523 /* 524 * rootnex_ctlops() 525 * 526 */ 527 /*ARGSUSED*/ 528 static int 529 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 530 void *arg, void *result) 531 { 532 int n, *ptr; 533 struct ddi_parent_private_data *pdp; 534 535 switch (ctlop) { 536 case DDI_CTLOPS_DMAPMAPC: 537 /* 538 * Return 'partial' to indicate that dma mapping 539 * has to be done in the main MMU. 540 */ 541 return (DDI_DMA_PARTIAL); 542 543 case DDI_CTLOPS_BTOP: 544 /* 545 * Convert byte count input to physical page units. 546 * (byte counts that are not a page-size multiple 547 * are rounded down) 548 */ 549 *(ulong_t *)result = btop(*(ulong_t *)arg); 550 return (DDI_SUCCESS); 551 552 case DDI_CTLOPS_PTOB: 553 /* 554 * Convert size in physical pages to bytes 555 */ 556 *(ulong_t *)result = ptob(*(ulong_t *)arg); 557 return (DDI_SUCCESS); 558 559 case DDI_CTLOPS_BTOPR: 560 /* 561 * Convert byte count input to physical page units 562 * (byte counts that are not a page-size multiple 563 * are rounded up) 564 */ 565 *(ulong_t *)result = btopr(*(ulong_t *)arg); 566 return (DDI_SUCCESS); 567 568 case DDI_CTLOPS_INITCHILD: 569 return (impl_ddi_sunbus_initchild(arg)); 570 571 case DDI_CTLOPS_UNINITCHILD: 572 impl_ddi_sunbus_removechild(arg); 573 return (DDI_SUCCESS); 574 575 case DDI_CTLOPS_REPORTDEV: 576 return (rootnex_ctl_reportdev(rdip)); 577 578 case DDI_CTLOPS_IOMIN: 579 /* 580 * Nothing to do here but reflect back.. 581 */ 582 return (DDI_SUCCESS); 583 584 case DDI_CTLOPS_REGSIZE: 585 case DDI_CTLOPS_NREGS: 586 break; 587 588 case DDI_CTLOPS_SIDDEV: 589 if (ndi_dev_is_prom_node(rdip)) 590 return (DDI_SUCCESS); 591 if (ndi_dev_is_persistent_node(rdip)) 592 return (DDI_SUCCESS); 593 return (DDI_FAILURE); 594 595 case DDI_CTLOPS_POWER: 596 return ((*pm_platform_power)((power_req_t *)arg)); 597 598 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 599 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 600 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 601 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 602 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 603 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 604 if (!rootnex_state->r_reserved_msg_printed) { 605 rootnex_state->r_reserved_msg_printed = B_TRUE; 606 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 607 "1 or more reserved/obsolete operations."); 608 } 609 return (DDI_FAILURE); 610 611 default: 612 return (DDI_FAILURE); 613 } 614 /* 615 * The rest are for "hardware" properties 616 */ 617 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 618 return (DDI_FAILURE); 619 620 if (ctlop == DDI_CTLOPS_NREGS) { 621 ptr = (int *)result; 622 *ptr = pdp->par_nreg; 623 } else { 624 off_t *size = (off_t *)result; 625 626 ptr = (int *)arg; 627 n = *ptr; 628 if (n >= pdp->par_nreg) { 629 return (DDI_FAILURE); 630 } 631 *size = (off_t)pdp->par_reg[n].regspec_size; 632 } 633 return (DDI_SUCCESS); 634 } 635 636 637 /* 638 * rootnex_ctl_reportdev() 639 * 640 */ 641 static int 642 rootnex_ctl_reportdev(dev_info_t *dev) 643 { 644 int i, n, len, f_len = 0; 645 char *buf; 646 647 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 648 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 649 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 650 len = strlen(buf); 651 652 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 653 654 struct regspec *rp = sparc_pd_getreg(dev, i); 655 656 if (i == 0) 657 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 658 ": "); 659 else 660 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 661 " and "); 662 len = strlen(buf); 663 664 switch (rp->regspec_bustype) { 665 666 case BTEISA: 667 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 668 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 669 break; 670 671 case BTISA: 672 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 673 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 674 break; 675 676 default: 677 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 678 "space %x offset %x", 679 rp->regspec_bustype, rp->regspec_addr); 680 break; 681 } 682 len = strlen(buf); 683 } 684 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 685 int pri; 686 687 if (i != 0) { 688 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 689 ","); 690 len = strlen(buf); 691 } 692 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 693 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 694 " sparc ipl %d", pri); 695 len = strlen(buf); 696 } 697 #ifdef DEBUG 698 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 699 cmn_err(CE_NOTE, "next message is truncated: " 700 "printed length 1024, real length %d", f_len); 701 } 702 #endif /* DEBUG */ 703 cmn_err(CE_CONT, "?%s\n", buf); 704 kmem_free(buf, REPORTDEV_BUFSIZE); 705 return (DDI_SUCCESS); 706 } 707 708 709 /* 710 * ****************** 711 * map related code 712 * ****************** 713 */ 714 715 /* 716 * rootnex_map() 717 * 718 */ 719 static int 720 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 721 off_t len, caddr_t *vaddrp) 722 { 723 struct regspec *rp, tmp_reg; 724 ddi_map_req_t mr = *mp; /* Get private copy of request */ 725 int error; 726 727 mp = &mr; 728 729 switch (mp->map_op) { 730 case DDI_MO_MAP_LOCKED: 731 case DDI_MO_UNMAP: 732 case DDI_MO_MAP_HANDLE: 733 break; 734 default: 735 #ifdef DDI_MAP_DEBUG 736 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 737 mp->map_op); 738 #endif /* DDI_MAP_DEBUG */ 739 return (DDI_ME_UNIMPLEMENTED); 740 } 741 742 if (mp->map_flags & DDI_MF_USER_MAPPING) { 743 #ifdef DDI_MAP_DEBUG 744 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 745 #endif /* DDI_MAP_DEBUG */ 746 return (DDI_ME_UNIMPLEMENTED); 747 } 748 749 /* 750 * First, if given an rnumber, convert it to a regspec... 751 * (Presumably, this is on behalf of a child of the root node?) 752 */ 753 754 if (mp->map_type == DDI_MT_RNUMBER) { 755 756 int rnumber = mp->map_obj.rnumber; 757 #ifdef DDI_MAP_DEBUG 758 static char *out_of_range = 759 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 760 #endif /* DDI_MAP_DEBUG */ 761 762 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 763 if (rp == NULL) { 764 #ifdef DDI_MAP_DEBUG 765 cmn_err(CE_WARN, out_of_range, rnumber, 766 ddi_get_name(rdip)); 767 #endif /* DDI_MAP_DEBUG */ 768 return (DDI_ME_RNUMBER_RANGE); 769 } 770 771 /* 772 * Convert the given ddi_map_req_t from rnumber to regspec... 773 */ 774 775 mp->map_type = DDI_MT_REGSPEC; 776 mp->map_obj.rp = rp; 777 } 778 779 /* 780 * Adjust offset and length correspnding to called values... 781 * XXX: A non-zero length means override the one in the regspec 782 * XXX: (regardless of what's in the parent's range?) 783 */ 784 785 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 786 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 787 788 #ifdef DDI_MAP_DEBUG 789 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 790 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 791 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 792 len, mp->map_handlep); 793 #endif /* DDI_MAP_DEBUG */ 794 795 /* 796 * I/O or memory mapping: 797 * 798 * <bustype=0, addr=x, len=x>: memory 799 * <bustype=1, addr=x, len=x>: i/o 800 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 801 */ 802 803 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 804 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 805 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 806 ddi_get_name(rdip), rp->regspec_bustype, 807 rp->regspec_addr, rp->regspec_size); 808 return (DDI_ME_INVAL); 809 } 810 811 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 812 /* 813 * compatibility i/o mapping 814 */ 815 rp->regspec_bustype += (uint_t)offset; 816 } else { 817 /* 818 * Normal memory or i/o mapping 819 */ 820 rp->regspec_addr += (uint_t)offset; 821 } 822 823 if (len != 0) 824 rp->regspec_size = (uint_t)len; 825 826 #ifdef DDI_MAP_DEBUG 827 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 828 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 829 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 830 offset, len, mp->map_handlep); 831 #endif /* DDI_MAP_DEBUG */ 832 833 /* 834 * Apply any parent ranges at this level, if applicable. 835 * (This is where nexus specific regspec translation takes place. 836 * Use of this function is implicit agreement that translation is 837 * provided via ddi_apply_range.) 838 */ 839 840 #ifdef DDI_MAP_DEBUG 841 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 842 ddi_get_name(dip), ddi_get_name(rdip)); 843 #endif /* DDI_MAP_DEBUG */ 844 845 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 846 return (error); 847 848 switch (mp->map_op) { 849 case DDI_MO_MAP_LOCKED: 850 851 /* 852 * Set up the locked down kernel mapping to the regspec... 853 */ 854 855 return (rootnex_map_regspec(mp, vaddrp)); 856 857 case DDI_MO_UNMAP: 858 859 /* 860 * Release mapping... 861 */ 862 863 return (rootnex_unmap_regspec(mp, vaddrp)); 864 865 case DDI_MO_MAP_HANDLE: 866 867 return (rootnex_map_handle(mp)); 868 869 default: 870 return (DDI_ME_UNIMPLEMENTED); 871 } 872 } 873 874 875 /* 876 * rootnex_map_fault() 877 * 878 * fault in mappings for requestors 879 */ 880 /*ARGSUSED*/ 881 static int 882 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 883 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 884 uint_t lock) 885 { 886 887 #ifdef DDI_MAP_DEBUG 888 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 889 ddi_map_debug(" Seg <%s>\n", 890 seg->s_ops == &segdev_ops ? "segdev" : 891 seg == &kvseg ? "segkmem" : "NONE!"); 892 #endif /* DDI_MAP_DEBUG */ 893 894 /* 895 * This is all terribly broken, but it is a start 896 * 897 * XXX Note that this test means that segdev_ops 898 * must be exported from seg_dev.c. 899 * XXX What about devices with their own segment drivers? 900 */ 901 if (seg->s_ops == &segdev_ops) { 902 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 903 904 if (hat == NULL) { 905 /* 906 * This is one plausible interpretation of 907 * a null hat i.e. use the first hat on the 908 * address space hat list which by convention is 909 * the hat of the system MMU. At alternative 910 * would be to panic .. this might well be better .. 911 */ 912 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 913 hat = seg->s_as->a_hat; 914 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 915 } 916 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 917 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 918 } else if (seg == &kvseg && dp == NULL) { 919 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 920 HAT_LOAD_LOCK); 921 } else 922 return (DDI_FAILURE); 923 return (DDI_SUCCESS); 924 } 925 926 927 /* 928 * rootnex_map_regspec() 929 * we don't support mapping of I/O cards above 4Gb 930 */ 931 static int 932 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 933 { 934 rootnex_addr_t rbase; 935 void *cvaddr; 936 uint_t npages, pgoffset; 937 struct regspec *rp; 938 ddi_acc_hdl_t *hp; 939 ddi_acc_impl_t *ap; 940 uint_t hat_acc_flags; 941 paddr_t pbase; 942 943 rp = mp->map_obj.rp; 944 hp = mp->map_handlep; 945 946 #ifdef DDI_MAP_DEBUG 947 ddi_map_debug( 948 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 949 rp->regspec_bustype, rp->regspec_addr, 950 rp->regspec_size, mp->map_handlep); 951 #endif /* DDI_MAP_DEBUG */ 952 953 /* 954 * I/O or memory mapping 955 * 956 * <bustype=0, addr=x, len=x>: memory 957 * <bustype=1, addr=x, len=x>: i/o 958 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 959 */ 960 961 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 962 cmn_err(CE_WARN, "rootnex: invalid register spec" 963 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 964 rp->regspec_addr, rp->regspec_size); 965 return (DDI_FAILURE); 966 } 967 968 if (rp->regspec_bustype != 0) { 969 /* 970 * I/O space - needs a handle. 971 */ 972 if (hp == NULL) { 973 return (DDI_FAILURE); 974 } 975 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 976 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 977 impl_acc_hdl_init(hp); 978 979 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 980 #ifdef DDI_MAP_DEBUG 981 ddi_map_debug("rootnex_map_regspec: mmap() " 982 "to I/O space is not supported.\n"); 983 #endif /* DDI_MAP_DEBUG */ 984 return (DDI_ME_INVAL); 985 } else { 986 /* 987 * 1275-compliant vs. compatibility i/o mapping 988 */ 989 *vaddrp = 990 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 991 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 992 ((caddr_t)(uintptr_t)rp->regspec_addr); 993 #ifdef __xpv 994 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 995 hp->ah_pfn = xen_assign_pfn( 996 mmu_btop((ulong_t)rp->regspec_addr & 997 MMU_PAGEMASK)); 998 } else { 999 hp->ah_pfn = mmu_btop( 1000 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1001 } 1002 #else 1003 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1004 MMU_PAGEMASK); 1005 #endif 1006 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1007 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1008 } 1009 1010 #ifdef DDI_MAP_DEBUG 1011 ddi_map_debug( 1012 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1013 rp->regspec_size, *vaddrp); 1014 #endif /* DDI_MAP_DEBUG */ 1015 return (DDI_SUCCESS); 1016 } 1017 1018 /* 1019 * Memory space 1020 */ 1021 1022 if (hp != NULL) { 1023 /* 1024 * hat layer ignores 1025 * hp->ah_acc.devacc_attr_endian_flags. 1026 */ 1027 switch (hp->ah_acc.devacc_attr_dataorder) { 1028 case DDI_STRICTORDER_ACC: 1029 hat_acc_flags = HAT_STRICTORDER; 1030 break; 1031 case DDI_UNORDERED_OK_ACC: 1032 hat_acc_flags = HAT_UNORDERED_OK; 1033 break; 1034 case DDI_MERGING_OK_ACC: 1035 hat_acc_flags = HAT_MERGING_OK; 1036 break; 1037 case DDI_LOADCACHING_OK_ACC: 1038 hat_acc_flags = HAT_LOADCACHING_OK; 1039 break; 1040 case DDI_STORECACHING_OK_ACC: 1041 hat_acc_flags = HAT_STORECACHING_OK; 1042 break; 1043 } 1044 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1045 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1046 impl_acc_hdl_init(hp); 1047 hp->ah_hat_flags = hat_acc_flags; 1048 } else { 1049 hat_acc_flags = HAT_STRICTORDER; 1050 } 1051 1052 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1053 #ifdef __xpv 1054 /* 1055 * If we're dom0, we're using a real device so we need to translate 1056 * the MA to a PA. 1057 */ 1058 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1059 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1060 } else { 1061 pbase = rbase; 1062 } 1063 #else 1064 pbase = rbase; 1065 #endif 1066 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1067 1068 if (rp->regspec_size == 0) { 1069 #ifdef DDI_MAP_DEBUG 1070 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1071 #endif /* DDI_MAP_DEBUG */ 1072 return (DDI_ME_INVAL); 1073 } 1074 1075 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1076 /* extra cast to make gcc happy */ 1077 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1078 } else { 1079 npages = mmu_btopr(rp->regspec_size + pgoffset); 1080 1081 #ifdef DDI_MAP_DEBUG 1082 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1083 "physical %llx", npages, pbase); 1084 #endif /* DDI_MAP_DEBUG */ 1085 1086 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1087 if (cvaddr == NULL) 1088 return (DDI_ME_NORESOURCES); 1089 1090 /* 1091 * Now map in the pages we've allocated... 1092 */ 1093 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1094 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1095 HAT_LOAD_LOCK); 1096 *vaddrp = (caddr_t)cvaddr + pgoffset; 1097 1098 /* save away pfn and npages for FMA */ 1099 hp = mp->map_handlep; 1100 if (hp) { 1101 hp->ah_pfn = mmu_btop(pbase); 1102 hp->ah_pnum = npages; 1103 } 1104 } 1105 1106 #ifdef DDI_MAP_DEBUG 1107 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1108 #endif /* DDI_MAP_DEBUG */ 1109 return (DDI_SUCCESS); 1110 } 1111 1112 1113 /* 1114 * rootnex_unmap_regspec() 1115 * 1116 */ 1117 static int 1118 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1119 { 1120 caddr_t addr = (caddr_t)*vaddrp; 1121 uint_t npages, pgoffset; 1122 struct regspec *rp; 1123 1124 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1125 return (0); 1126 1127 rp = mp->map_obj.rp; 1128 1129 if (rp->regspec_size == 0) { 1130 #ifdef DDI_MAP_DEBUG 1131 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1132 #endif /* DDI_MAP_DEBUG */ 1133 return (DDI_ME_INVAL); 1134 } 1135 1136 /* 1137 * I/O or memory mapping: 1138 * 1139 * <bustype=0, addr=x, len=x>: memory 1140 * <bustype=1, addr=x, len=x>: i/o 1141 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1142 */ 1143 if (rp->regspec_bustype != 0) { 1144 /* 1145 * This is I/O space, which requires no particular 1146 * processing on unmap since it isn't mapped in the 1147 * first place. 1148 */ 1149 return (DDI_SUCCESS); 1150 } 1151 1152 /* 1153 * Memory space 1154 */ 1155 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1156 npages = mmu_btopr(rp->regspec_size + pgoffset); 1157 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1158 device_arena_free(addr - pgoffset, ptob(npages)); 1159 1160 /* 1161 * Destroy the pointer - the mapping has logically gone 1162 */ 1163 *vaddrp = NULL; 1164 1165 return (DDI_SUCCESS); 1166 } 1167 1168 1169 /* 1170 * rootnex_map_handle() 1171 * 1172 */ 1173 static int 1174 rootnex_map_handle(ddi_map_req_t *mp) 1175 { 1176 rootnex_addr_t rbase; 1177 ddi_acc_hdl_t *hp; 1178 uint_t pgoffset; 1179 struct regspec *rp; 1180 paddr_t pbase; 1181 1182 rp = mp->map_obj.rp; 1183 1184 #ifdef DDI_MAP_DEBUG 1185 ddi_map_debug( 1186 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1187 rp->regspec_bustype, rp->regspec_addr, 1188 rp->regspec_size, mp->map_handlep); 1189 #endif /* DDI_MAP_DEBUG */ 1190 1191 /* 1192 * I/O or memory mapping: 1193 * 1194 * <bustype=0, addr=x, len=x>: memory 1195 * <bustype=1, addr=x, len=x>: i/o 1196 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1197 */ 1198 if (rp->regspec_bustype != 0) { 1199 /* 1200 * This refers to I/O space, and we don't support "mapping" 1201 * I/O space to a user. 1202 */ 1203 return (DDI_FAILURE); 1204 } 1205 1206 /* 1207 * Set up the hat_flags for the mapping. 1208 */ 1209 hp = mp->map_handlep; 1210 1211 switch (hp->ah_acc.devacc_attr_endian_flags) { 1212 case DDI_NEVERSWAP_ACC: 1213 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1214 break; 1215 case DDI_STRUCTURE_LE_ACC: 1216 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1217 break; 1218 case DDI_STRUCTURE_BE_ACC: 1219 return (DDI_FAILURE); 1220 default: 1221 return (DDI_REGS_ACC_CONFLICT); 1222 } 1223 1224 switch (hp->ah_acc.devacc_attr_dataorder) { 1225 case DDI_STRICTORDER_ACC: 1226 break; 1227 case DDI_UNORDERED_OK_ACC: 1228 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1229 break; 1230 case DDI_MERGING_OK_ACC: 1231 hp->ah_hat_flags |= HAT_MERGING_OK; 1232 break; 1233 case DDI_LOADCACHING_OK_ACC: 1234 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1235 break; 1236 case DDI_STORECACHING_OK_ACC: 1237 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1238 break; 1239 default: 1240 return (DDI_FAILURE); 1241 } 1242 1243 rbase = (rootnex_addr_t)rp->regspec_addr & 1244 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1245 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1246 1247 if (rp->regspec_size == 0) 1248 return (DDI_ME_INVAL); 1249 1250 #ifdef __xpv 1251 /* 1252 * If we're dom0, we're using a real device so we need to translate 1253 * the MA to a PA. 1254 */ 1255 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1256 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1257 (rbase & MMU_PAGEOFFSET); 1258 } else { 1259 pbase = rbase; 1260 } 1261 #else 1262 pbase = rbase; 1263 #endif 1264 1265 hp->ah_pfn = mmu_btop(pbase); 1266 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1267 1268 return (DDI_SUCCESS); 1269 } 1270 1271 1272 1273 /* 1274 * ************************ 1275 * interrupt related code 1276 * ************************ 1277 */ 1278 1279 /* 1280 * rootnex_intr_ops() 1281 * bus_intr_op() function for interrupt support 1282 */ 1283 /* ARGSUSED */ 1284 static int 1285 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1286 ddi_intr_handle_impl_t *hdlp, void *result) 1287 { 1288 struct intrspec *ispec; 1289 struct ddi_parent_private_data *pdp; 1290 1291 DDI_INTR_NEXDBG((CE_CONT, 1292 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1293 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1294 1295 /* Process the interrupt operation */ 1296 switch (intr_op) { 1297 case DDI_INTROP_GETCAP: 1298 /* First check with pcplusmp */ 1299 if (psm_intr_ops == NULL) 1300 return (DDI_FAILURE); 1301 1302 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1303 *(int *)result = 0; 1304 return (DDI_FAILURE); 1305 } 1306 break; 1307 case DDI_INTROP_SETCAP: 1308 if (psm_intr_ops == NULL) 1309 return (DDI_FAILURE); 1310 1311 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1312 return (DDI_FAILURE); 1313 break; 1314 case DDI_INTROP_ALLOC: 1315 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1316 return (DDI_FAILURE); 1317 hdlp->ih_pri = ispec->intrspec_pri; 1318 *(int *)result = hdlp->ih_scratch1; 1319 break; 1320 case DDI_INTROP_FREE: 1321 pdp = ddi_get_parent_data(rdip); 1322 /* 1323 * Special case for 'pcic' driver' only. 1324 * If an intrspec was created for it, clean it up here 1325 * See detailed comments on this in the function 1326 * rootnex_get_ispec(). 1327 */ 1328 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1329 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1330 pdp->par_nintr); 1331 /* 1332 * Set it to zero; so that 1333 * DDI framework doesn't free it again 1334 */ 1335 pdp->par_intr = NULL; 1336 pdp->par_nintr = 0; 1337 } 1338 break; 1339 case DDI_INTROP_GETPRI: 1340 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1341 return (DDI_FAILURE); 1342 *(int *)result = ispec->intrspec_pri; 1343 break; 1344 case DDI_INTROP_SETPRI: 1345 /* Validate the interrupt priority passed to us */ 1346 if (*(int *)result > LOCK_LEVEL) 1347 return (DDI_FAILURE); 1348 1349 /* Ensure that PSM is all initialized and ispec is ok */ 1350 if ((psm_intr_ops == NULL) || 1351 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1352 return (DDI_FAILURE); 1353 1354 /* Change the priority */ 1355 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1356 PSM_FAILURE) 1357 return (DDI_FAILURE); 1358 1359 /* update the ispec with the new priority */ 1360 ispec->intrspec_pri = *(int *)result; 1361 break; 1362 case DDI_INTROP_ADDISR: 1363 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1364 return (DDI_FAILURE); 1365 ispec->intrspec_func = hdlp->ih_cb_func; 1366 break; 1367 case DDI_INTROP_REMISR: 1368 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1369 return (DDI_FAILURE); 1370 ispec->intrspec_func = (uint_t (*)()) 0; 1371 break; 1372 case DDI_INTROP_ENABLE: 1373 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1374 return (DDI_FAILURE); 1375 1376 /* Call psmi to translate irq with the dip */ 1377 if (psm_intr_ops == NULL) 1378 return (DDI_FAILURE); 1379 1380 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1381 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1382 (int *)&hdlp->ih_vector); 1383 1384 /* Add the interrupt handler */ 1385 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1386 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1387 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1388 return (DDI_FAILURE); 1389 break; 1390 case DDI_INTROP_DISABLE: 1391 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1392 return (DDI_FAILURE); 1393 1394 /* Call psm_ops() to translate irq with the dip */ 1395 if (psm_intr_ops == NULL) 1396 return (DDI_FAILURE); 1397 1398 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1399 (void) (*psm_intr_ops)(rdip, hdlp, 1400 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1401 1402 /* Remove the interrupt handler */ 1403 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1404 hdlp->ih_cb_func, hdlp->ih_vector); 1405 break; 1406 case DDI_INTROP_SETMASK: 1407 if (psm_intr_ops == NULL) 1408 return (DDI_FAILURE); 1409 1410 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1411 return (DDI_FAILURE); 1412 break; 1413 case DDI_INTROP_CLRMASK: 1414 if (psm_intr_ops == NULL) 1415 return (DDI_FAILURE); 1416 1417 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1418 return (DDI_FAILURE); 1419 break; 1420 case DDI_INTROP_GETPENDING: 1421 if (psm_intr_ops == NULL) 1422 return (DDI_FAILURE); 1423 1424 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1425 result)) { 1426 *(int *)result = 0; 1427 return (DDI_FAILURE); 1428 } 1429 break; 1430 case DDI_INTROP_NAVAIL: 1431 case DDI_INTROP_NINTRS: 1432 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1433 if (*(int *)result == 0) { 1434 /* 1435 * Special case for 'pcic' driver' only. This driver 1436 * driver is a child of 'isa' and 'rootnex' drivers. 1437 * 1438 * See detailed comments on this in the function 1439 * rootnex_get_ispec(). 1440 * 1441 * Children of 'pcic' send 'NINITR' request all the 1442 * way to rootnex driver. But, the 'pdp->par_nintr' 1443 * field may not initialized. So, we fake it here 1444 * to return 1 (a la what PCMCIA nexus does). 1445 */ 1446 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1447 *(int *)result = 1; 1448 else 1449 return (DDI_FAILURE); 1450 } 1451 break; 1452 case DDI_INTROP_SUPPORTED_TYPES: 1453 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1454 break; 1455 default: 1456 return (DDI_FAILURE); 1457 } 1458 1459 return (DDI_SUCCESS); 1460 } 1461 1462 1463 /* 1464 * rootnex_get_ispec() 1465 * convert an interrupt number to an interrupt specification. 1466 * The interrupt number determines which interrupt spec will be 1467 * returned if more than one exists. 1468 * 1469 * Look into the parent private data area of the 'rdip' to find out 1470 * the interrupt specification. First check to make sure there is 1471 * one that matchs "inumber" and then return a pointer to it. 1472 * 1473 * Return NULL if one could not be found. 1474 * 1475 * NOTE: This is needed for rootnex_intr_ops() 1476 */ 1477 static struct intrspec * 1478 rootnex_get_ispec(dev_info_t *rdip, int inum) 1479 { 1480 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1481 1482 /* 1483 * Special case handling for drivers that provide their own 1484 * intrspec structures instead of relying on the DDI framework. 1485 * 1486 * A broken hardware driver in ON could potentially provide its 1487 * own intrspec structure, instead of relying on the hardware. 1488 * If these drivers are children of 'rootnex' then we need to 1489 * continue to provide backward compatibility to them here. 1490 * 1491 * Following check is a special case for 'pcic' driver which 1492 * was found to have broken hardwre andby provides its own intrspec. 1493 * 1494 * Verbatim comments from this driver are shown here: 1495 * "Don't use the ddi_add_intr since we don't have a 1496 * default intrspec in all cases." 1497 * 1498 * Since an 'ispec' may not be always created for it, 1499 * check for that and create one if so. 1500 * 1501 * NOTE: Currently 'pcic' is the only driver found to do this. 1502 */ 1503 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1504 pdp->par_nintr = 1; 1505 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1506 pdp->par_nintr, KM_SLEEP); 1507 } 1508 1509 /* Validate the interrupt number */ 1510 if (inum >= pdp->par_nintr) 1511 return (NULL); 1512 1513 /* Get the interrupt structure pointer and return that */ 1514 return ((struct intrspec *)&pdp->par_intr[inum]); 1515 } 1516 1517 1518 /* 1519 * ****************** 1520 * dma related code 1521 * ****************** 1522 */ 1523 1524 /* 1525 * rootnex_dma_allochdl() 1526 * called from ddi_dma_alloc_handle(). 1527 */ 1528 /*ARGSUSED*/ 1529 static int 1530 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1531 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1532 { 1533 uint64_t maxsegmentsize_ll; 1534 uint_t maxsegmentsize; 1535 ddi_dma_impl_t *hp; 1536 rootnex_dma_t *dma; 1537 uint64_t count_max; 1538 uint64_t seg; 1539 int kmflag; 1540 int e; 1541 1542 1543 /* convert our sleep flags */ 1544 if (waitfp == DDI_DMA_SLEEP) { 1545 kmflag = KM_SLEEP; 1546 } else { 1547 kmflag = KM_NOSLEEP; 1548 } 1549 1550 /* 1551 * We try to do only one memory allocation here. We'll do a little 1552 * pointer manipulation later. If the bind ends up taking more than 1553 * our prealloc's space, we'll have to allocate more memory in the 1554 * bind operation. Not great, but much better than before and the 1555 * best we can do with the current bind interfaces. 1556 */ 1557 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1558 if (hp == NULL) { 1559 if (waitfp != DDI_DMA_DONTWAIT) { 1560 ddi_set_callback(waitfp, arg, 1561 &rootnex_state->r_dvma_call_list_id); 1562 } 1563 return (DDI_DMA_NORESOURCES); 1564 } 1565 1566 /* Do our pointer manipulation now, align the structures */ 1567 hp->dmai_private = (void *)(((uintptr_t)hp + 1568 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1569 dma = (rootnex_dma_t *)hp->dmai_private; 1570 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1571 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1572 1573 /* setup the handle */ 1574 rootnex_clean_dmahdl(hp); 1575 dma->dp_dip = rdip; 1576 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1577 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1578 hp->dmai_minxfer = attr->dma_attr_minxfer; 1579 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1580 hp->dmai_rdip = rdip; 1581 hp->dmai_attr = *attr; 1582 1583 /* we don't need to worry about the SPL since we do a tryenter */ 1584 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1585 1586 /* 1587 * Figure out our maximum segment size. If the segment size is greater 1588 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1589 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1590 * dma_attr_count_max are size-1 type values. 1591 * 1592 * Maximum segment size is the largest physically contiguous chunk of 1593 * memory that we can return from a bind (i.e. the maximum size of a 1594 * single cookie). 1595 */ 1596 1597 /* handle the rollover cases */ 1598 seg = attr->dma_attr_seg + 1; 1599 if (seg < attr->dma_attr_seg) { 1600 seg = attr->dma_attr_seg; 1601 } 1602 count_max = attr->dma_attr_count_max + 1; 1603 if (count_max < attr->dma_attr_count_max) { 1604 count_max = attr->dma_attr_count_max; 1605 } 1606 1607 /* 1608 * granularity may or may not be a power of two. If it isn't, we can't 1609 * use a simple mask. 1610 */ 1611 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1612 dma->dp_granularity_power_2 = B_FALSE; 1613 } else { 1614 dma->dp_granularity_power_2 = B_TRUE; 1615 } 1616 1617 /* 1618 * maxxfer should be a whole multiple of granularity. If we're going to 1619 * break up a window because we're greater than maxxfer, we might as 1620 * well make sure it's maxxfer is a whole multiple so we don't have to 1621 * worry about triming the window later on for this case. 1622 */ 1623 if (attr->dma_attr_granular > 1) { 1624 if (dma->dp_granularity_power_2) { 1625 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1626 (attr->dma_attr_maxxfer & 1627 (attr->dma_attr_granular - 1)); 1628 } else { 1629 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1630 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1631 } 1632 } else { 1633 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1634 } 1635 1636 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1637 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1638 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1639 maxsegmentsize = 0xFFFFFFFF; 1640 } else { 1641 maxsegmentsize = maxsegmentsize_ll; 1642 } 1643 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1644 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1645 1646 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1647 if (rootnex_alloc_check_parms) { 1648 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1649 if (e != DDI_SUCCESS) { 1650 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1651 (void) rootnex_dma_freehdl(dip, rdip, 1652 (ddi_dma_handle_t)hp); 1653 return (e); 1654 } 1655 } 1656 1657 *handlep = (ddi_dma_handle_t)hp; 1658 1659 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1660 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1661 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1662 1663 return (DDI_SUCCESS); 1664 } 1665 1666 1667 /* 1668 * rootnex_dma_freehdl() 1669 * called from ddi_dma_free_handle(). 1670 */ 1671 /*ARGSUSED*/ 1672 static int 1673 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1674 { 1675 ddi_dma_impl_t *hp; 1676 rootnex_dma_t *dma; 1677 1678 1679 hp = (ddi_dma_impl_t *)handle; 1680 dma = (rootnex_dma_t *)hp->dmai_private; 1681 1682 /* unbind should have been called first */ 1683 ASSERT(!dma->dp_inuse); 1684 1685 mutex_destroy(&dma->dp_mutex); 1686 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1687 1688 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1689 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1690 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1691 1692 if (rootnex_state->r_dvma_call_list_id) 1693 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1694 1695 return (DDI_SUCCESS); 1696 } 1697 1698 1699 /* 1700 * rootnex_dma_bindhdl() 1701 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 1702 */ 1703 /*ARGSUSED*/ 1704 static int 1705 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 1706 struct ddi_dma_req *dmareq, ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1707 { 1708 rootnex_sglinfo_t *sinfo; 1709 ddi_dma_attr_t *attr; 1710 ddi_dma_impl_t *hp; 1711 rootnex_dma_t *dma; 1712 int kmflag; 1713 int e; 1714 1715 1716 hp = (ddi_dma_impl_t *)handle; 1717 dma = (rootnex_dma_t *)hp->dmai_private; 1718 sinfo = &dma->dp_sglinfo; 1719 attr = &hp->dmai_attr; 1720 1721 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1722 1723 /* 1724 * This is useful for debugging a driver. Not as useful in a production 1725 * system. The only time this will fail is if you have a driver bug. 1726 */ 1727 if (rootnex_bind_check_inuse) { 1728 /* 1729 * No one else should ever have this lock unless someone else 1730 * is trying to use this handle. So contention on the lock 1731 * is the same as inuse being set. 1732 */ 1733 e = mutex_tryenter(&dma->dp_mutex); 1734 if (e == 0) { 1735 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1736 return (DDI_DMA_INUSE); 1737 } 1738 if (dma->dp_inuse) { 1739 mutex_exit(&dma->dp_mutex); 1740 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1741 return (DDI_DMA_INUSE); 1742 } 1743 dma->dp_inuse = B_TRUE; 1744 mutex_exit(&dma->dp_mutex); 1745 } 1746 1747 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1748 if (rootnex_bind_check_parms) { 1749 e = rootnex_valid_bind_parms(dmareq, attr); 1750 if (e != DDI_SUCCESS) { 1751 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1752 rootnex_clean_dmahdl(hp); 1753 return (e); 1754 } 1755 } 1756 1757 /* save away the original bind info */ 1758 dma->dp_dma = dmareq->dmar_object; 1759 1760 /* 1761 * Figure out a rough estimate of what maximum number of pages this 1762 * buffer could use (a high estimate of course). 1763 */ 1764 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1765 1766 /* 1767 * We'll use the pre-allocated cookies for any bind that will *always* 1768 * fit (more important to be consistent, we don't want to create 1769 * additional degenerate cases). 1770 */ 1771 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1772 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1773 dma->dp_need_to_free_cookie = B_FALSE; 1774 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1775 uint_t, sinfo->si_max_pages); 1776 1777 /* 1778 * For anything larger than that, we'll go ahead and allocate the 1779 * maximum number of pages we expect to see. Hopefuly, we won't be 1780 * seeing this path in the fast path for high performance devices very 1781 * frequently. 1782 * 1783 * a ddi bind interface that allowed the driver to provide storage to 1784 * the bind interface would speed this case up. 1785 */ 1786 } else { 1787 /* convert the sleep flags */ 1788 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1789 kmflag = KM_SLEEP; 1790 } else { 1791 kmflag = KM_NOSLEEP; 1792 } 1793 1794 /* 1795 * Save away how much memory we allocated. If we're doing a 1796 * nosleep, the alloc could fail... 1797 */ 1798 dma->dp_cookie_size = sinfo->si_max_pages * 1799 sizeof (ddi_dma_cookie_t); 1800 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1801 if (dma->dp_cookies == NULL) { 1802 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1803 rootnex_clean_dmahdl(hp); 1804 return (DDI_DMA_NORESOURCES); 1805 } 1806 dma->dp_need_to_free_cookie = B_TRUE; 1807 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1808 sinfo->si_max_pages); 1809 } 1810 hp->dmai_cookie = dma->dp_cookies; 1811 1812 /* 1813 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1814 * looking at the contraints in the dma structure. It will then put some 1815 * additional state about the sgl in the dma struct (i.e. is the sgl 1816 * clean, or do we need to do some munging; how many pages need to be 1817 * copied, etc.) 1818 */ 1819 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1820 &dma->dp_sglinfo); 1821 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1822 1823 /* if we don't need a copy buffer, we don't need to sync */ 1824 if (sinfo->si_copybuf_req == 0) { 1825 hp->dmai_rflags |= DMP_NOSYNC; 1826 } 1827 1828 /* 1829 * if we don't need the copybuf and we don't need to do a partial, we 1830 * hit the fast path. All the high performance devices should be trying 1831 * to hit this path. To hit this path, a device should be able to reach 1832 * all of memory, shouldn't try to bind more than it can transfer, and 1833 * the buffer shouldn't require more cookies than the driver/device can 1834 * handle [sgllen]). 1835 */ 1836 if ((sinfo->si_copybuf_req == 0) && 1837 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1838 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1839 /* 1840 * If the driver supports FMA, insert the handle in the FMA DMA 1841 * handle cache. 1842 */ 1843 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1844 hp->dmai_error.err_cf = rootnex_dma_check; 1845 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1846 } 1847 1848 /* 1849 * copy out the first cookie and ccountp, set the cookie 1850 * pointer to the second cookie. The first cookie is passed 1851 * back on the stack. Additional cookies are accessed via 1852 * ddi_dma_nextcookie() 1853 */ 1854 *cookiep = dma->dp_cookies[0]; 1855 *ccountp = sinfo->si_sgl_size; 1856 hp->dmai_cookie++; 1857 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1858 hp->dmai_nwin = 1; 1859 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1860 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1861 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1862 dma->dp_dma.dmao_size); 1863 return (DDI_DMA_MAPPED); 1864 } 1865 1866 /* 1867 * go to the slow path, we may need to alloc more memory, create 1868 * multiple windows, and munge up a sgl to make the device happy. 1869 */ 1870 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 1871 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 1872 if (dma->dp_need_to_free_cookie) { 1873 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1874 } 1875 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1876 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 1877 return (e); 1878 } 1879 1880 /* 1881 * If the driver supports FMA, insert the handle in the FMA DMA handle 1882 * cache. 1883 */ 1884 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1885 hp->dmai_error.err_cf = rootnex_dma_check; 1886 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1887 } 1888 1889 /* if the first window uses the copy buffer, sync it for the device */ 1890 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 1891 (hp->dmai_rflags & DDI_DMA_WRITE)) { 1892 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1893 DDI_DMA_SYNC_FORDEV); 1894 } 1895 1896 /* 1897 * copy out the first cookie and ccountp, set the cookie pointer to the 1898 * second cookie. Make sure the partial flag is set/cleared correctly. 1899 * If we have a partial map (i.e. multiple windows), the number of 1900 * cookies we return is the number of cookies in the first window. 1901 */ 1902 if (e == DDI_DMA_MAPPED) { 1903 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1904 *ccountp = sinfo->si_sgl_size; 1905 } else { 1906 hp->dmai_rflags |= DDI_DMA_PARTIAL; 1907 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 1908 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 1909 } 1910 *cookiep = dma->dp_cookies[0]; 1911 hp->dmai_cookie++; 1912 1913 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1914 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 1915 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1916 dma->dp_dma.dmao_size); 1917 return (e); 1918 } 1919 1920 1921 /* 1922 * rootnex_dma_unbindhdl() 1923 * called from ddi_dma_unbind_handle() 1924 */ 1925 /*ARGSUSED*/ 1926 static int 1927 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 1928 ddi_dma_handle_t handle) 1929 { 1930 ddi_dma_impl_t *hp; 1931 rootnex_dma_t *dma; 1932 int e; 1933 1934 1935 hp = (ddi_dma_impl_t *)handle; 1936 dma = (rootnex_dma_t *)hp->dmai_private; 1937 1938 /* make sure the buffer wasn't free'd before calling unbind */ 1939 if (rootnex_unbind_verify_buffer) { 1940 e = rootnex_verify_buffer(dma); 1941 if (e != DDI_SUCCESS) { 1942 ASSERT(0); 1943 return (DDI_FAILURE); 1944 } 1945 } 1946 1947 /* sync the current window before unbinding the buffer */ 1948 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 1949 (hp->dmai_rflags & DDI_DMA_READ)) { 1950 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 1951 DDI_DMA_SYNC_FORCPU); 1952 } 1953 1954 /* 1955 * If the driver supports FMA, remove the handle in the FMA DMA handle 1956 * cache. 1957 */ 1958 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 1959 if ((DEVI(rdip)->devi_fmhdl != NULL) && 1960 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 1961 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 1962 } 1963 } 1964 1965 /* 1966 * cleanup and copy buffer or window state. if we didn't use the copy 1967 * buffer or windows, there won't be much to do :-) 1968 */ 1969 rootnex_teardown_copybuf(dma); 1970 rootnex_teardown_windows(dma); 1971 1972 /* 1973 * If we had to allocate space to for the worse case sgl (it didn't 1974 * fit into our pre-allocate buffer), free that up now 1975 */ 1976 if (dma->dp_need_to_free_cookie) { 1977 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 1978 } 1979 1980 /* 1981 * clean up the handle so it's ready for the next bind (i.e. if the 1982 * handle is reused). 1983 */ 1984 rootnex_clean_dmahdl(hp); 1985 1986 if (rootnex_state->r_dvma_call_list_id) 1987 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1988 1989 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1990 DTRACE_PROBE1(rootnex__unbind, uint64_t, 1991 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1992 1993 return (DDI_SUCCESS); 1994 } 1995 1996 1997 /* 1998 * rootnex_verify_buffer() 1999 * verify buffer wasn't free'd 2000 */ 2001 static int 2002 rootnex_verify_buffer(rootnex_dma_t *dma) 2003 { 2004 page_t **pplist; 2005 caddr_t vaddr; 2006 uint_t pcnt; 2007 uint_t poff; 2008 page_t *pp; 2009 char b; 2010 int i; 2011 2012 /* Figure out how many pages this buffer occupies */ 2013 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2014 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2015 } else { 2016 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2017 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2018 } 2019 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2020 2021 switch (dma->dp_dma.dmao_type) { 2022 case DMA_OTYP_PAGES: 2023 /* 2024 * for a linked list of pp's walk through them to make sure 2025 * they're locked and not free. 2026 */ 2027 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2028 for (i = 0; i < pcnt; i++) { 2029 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2030 return (DDI_FAILURE); 2031 } 2032 pp = pp->p_next; 2033 } 2034 break; 2035 2036 case DMA_OTYP_VADDR: 2037 case DMA_OTYP_BUFVADDR: 2038 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2039 /* 2040 * for an array of pp's walk through them to make sure they're 2041 * not free. It's possible that they may not be locked. 2042 */ 2043 if (pplist) { 2044 for (i = 0; i < pcnt; i++) { 2045 if (PP_ISFREE(pplist[i])) { 2046 return (DDI_FAILURE); 2047 } 2048 } 2049 2050 /* For a virtual address, try to peek at each page */ 2051 } else { 2052 if (dma->dp_sglinfo.si_asp == &kas) { 2053 for (i = 0; i < pcnt; i++) { 2054 if (ddi_peek8(NULL, vaddr, &b) == 2055 DDI_FAILURE) 2056 return (DDI_FAILURE); 2057 vaddr += MMU_PAGESIZE; 2058 } 2059 } 2060 } 2061 break; 2062 2063 default: 2064 ASSERT(0); 2065 break; 2066 } 2067 2068 return (DDI_SUCCESS); 2069 } 2070 2071 2072 /* 2073 * rootnex_clean_dmahdl() 2074 * Clean the dma handle. This should be called on a handle alloc and an 2075 * unbind handle. Set the handle state to the default settings. 2076 */ 2077 static void 2078 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2079 { 2080 rootnex_dma_t *dma; 2081 2082 2083 dma = (rootnex_dma_t *)hp->dmai_private; 2084 2085 hp->dmai_nwin = 0; 2086 dma->dp_current_cookie = 0; 2087 dma->dp_copybuf_size = 0; 2088 dma->dp_window = NULL; 2089 dma->dp_cbaddr = NULL; 2090 dma->dp_inuse = B_FALSE; 2091 dma->dp_need_to_free_cookie = B_FALSE; 2092 dma->dp_need_to_free_window = B_FALSE; 2093 dma->dp_partial_required = B_FALSE; 2094 dma->dp_trim_required = B_FALSE; 2095 dma->dp_sglinfo.si_copybuf_req = 0; 2096 #if !defined(__amd64) 2097 dma->dp_cb_remaping = B_FALSE; 2098 dma->dp_kva = NULL; 2099 #endif 2100 2101 /* FMA related initialization */ 2102 hp->dmai_fault = 0; 2103 hp->dmai_fault_check = NULL; 2104 hp->dmai_fault_notify = NULL; 2105 hp->dmai_error.err_ena = 0; 2106 hp->dmai_error.err_status = DDI_FM_OK; 2107 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2108 hp->dmai_error.err_ontrap = NULL; 2109 hp->dmai_error.err_fep = NULL; 2110 hp->dmai_error.err_cf = NULL; 2111 } 2112 2113 2114 /* 2115 * rootnex_valid_alloc_parms() 2116 * Called in ddi_dma_alloc_handle path to validate its parameters. 2117 */ 2118 static int 2119 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2120 { 2121 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2122 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2123 (attr->dma_attr_granular > MMU_PAGESIZE) || 2124 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2125 return (DDI_DMA_BADATTR); 2126 } 2127 2128 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2129 return (DDI_DMA_BADATTR); 2130 } 2131 2132 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2133 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2134 attr->dma_attr_sgllen <= 0) { 2135 return (DDI_DMA_BADATTR); 2136 } 2137 2138 /* We should be able to DMA into every byte offset in a page */ 2139 if (maxsegmentsize < MMU_PAGESIZE) { 2140 return (DDI_DMA_BADATTR); 2141 } 2142 2143 return (DDI_SUCCESS); 2144 } 2145 2146 2147 /* 2148 * rootnex_valid_bind_parms() 2149 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2150 */ 2151 /* ARGSUSED */ 2152 static int 2153 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2154 { 2155 #if !defined(__amd64) 2156 /* 2157 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2158 * we can track the offset for the obsoleted interfaces. 2159 */ 2160 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2161 return (DDI_DMA_TOOBIG); 2162 } 2163 #endif 2164 2165 return (DDI_SUCCESS); 2166 } 2167 2168 2169 /* 2170 * rootnex_get_sgl() 2171 * Called in bind fastpath to get the sgl. Most of this will be replaced 2172 * with a call to the vm layer when vm2.0 comes around... 2173 */ 2174 static void 2175 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2176 rootnex_sglinfo_t *sglinfo) 2177 { 2178 ddi_dma_atyp_t buftype; 2179 rootnex_addr_t raddr; 2180 uint64_t last_page; 2181 uint64_t offset; 2182 uint64_t addrhi; 2183 uint64_t addrlo; 2184 uint64_t maxseg; 2185 page_t **pplist; 2186 uint64_t paddr; 2187 uint32_t psize; 2188 uint32_t size; 2189 caddr_t vaddr; 2190 uint_t pcnt; 2191 page_t *pp; 2192 uint_t cnt; 2193 2194 2195 /* shortcuts */ 2196 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2197 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2198 maxseg = sglinfo->si_max_cookie_size; 2199 buftype = dmar_object->dmao_type; 2200 addrhi = sglinfo->si_max_addr; 2201 addrlo = sglinfo->si_min_addr; 2202 size = dmar_object->dmao_size; 2203 2204 pcnt = 0; 2205 cnt = 0; 2206 2207 /* 2208 * if we were passed down a linked list of pages, i.e. pointer to 2209 * page_t, use this to get our physical address and buf offset. 2210 */ 2211 if (buftype == DMA_OTYP_PAGES) { 2212 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2213 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2214 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2215 MMU_PAGEOFFSET; 2216 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2217 psize = MIN(size, (MMU_PAGESIZE - offset)); 2218 pp = pp->p_next; 2219 sglinfo->si_asp = NULL; 2220 2221 /* 2222 * We weren't passed down a linked list of pages, but if we were passed 2223 * down an array of pages, use this to get our physical address and buf 2224 * offset. 2225 */ 2226 } else if (pplist != NULL) { 2227 ASSERT((buftype == DMA_OTYP_VADDR) || 2228 (buftype == DMA_OTYP_BUFVADDR)); 2229 2230 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2231 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2232 if (sglinfo->si_asp == NULL) { 2233 sglinfo->si_asp = &kas; 2234 } 2235 2236 ASSERT(!PP_ISFREE(pplist[pcnt])); 2237 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2238 paddr += offset; 2239 psize = MIN(size, (MMU_PAGESIZE - offset)); 2240 pcnt++; 2241 2242 /* 2243 * All we have is a virtual address, we'll need to call into the VM 2244 * to get the physical address. 2245 */ 2246 } else { 2247 ASSERT((buftype == DMA_OTYP_VADDR) || 2248 (buftype == DMA_OTYP_BUFVADDR)); 2249 2250 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2251 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2252 if (sglinfo->si_asp == NULL) { 2253 sglinfo->si_asp = &kas; 2254 } 2255 2256 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2257 paddr += offset; 2258 psize = MIN(size, (MMU_PAGESIZE - offset)); 2259 vaddr += psize; 2260 } 2261 2262 #ifdef __xpv 2263 /* 2264 * If we're dom0, we're using a real device so we need to load 2265 * the cookies with MFNs instead of PFNs. 2266 */ 2267 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2268 #else 2269 raddr = paddr; 2270 #endif 2271 2272 /* 2273 * Setup the first cookie with the physical address of the page and the 2274 * size of the page (which takes into account the initial offset into 2275 * the page. 2276 */ 2277 sgl[cnt].dmac_laddress = raddr; 2278 sgl[cnt].dmac_size = psize; 2279 sgl[cnt].dmac_type = 0; 2280 2281 /* 2282 * Save away the buffer offset into the page. We'll need this later in 2283 * the copy buffer code to help figure out the page index within the 2284 * buffer and the offset into the current page. 2285 */ 2286 sglinfo->si_buf_offset = offset; 2287 2288 /* 2289 * If the DMA engine can't reach the physical address, increase how 2290 * much copy buffer we need. We always increase by pagesize so we don't 2291 * have to worry about converting offsets. Set a flag in the cookies 2292 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2293 * last cookie, go to the next cookie (since we separate each page which 2294 * uses the copy buffer in case the copy buffer is not physically 2295 * contiguous. 2296 */ 2297 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2298 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2299 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2300 if ((cnt + 1) < sglinfo->si_max_pages) { 2301 cnt++; 2302 sgl[cnt].dmac_laddress = 0; 2303 sgl[cnt].dmac_size = 0; 2304 sgl[cnt].dmac_type = 0; 2305 } 2306 } 2307 2308 /* 2309 * save this page's physical address so we can figure out if the next 2310 * page is physically contiguous. Keep decrementing size until we are 2311 * done with the buffer. 2312 */ 2313 last_page = raddr & MMU_PAGEMASK; 2314 size -= psize; 2315 2316 while (size > 0) { 2317 /* Get the size for this page (i.e. partial or full page) */ 2318 psize = MIN(size, MMU_PAGESIZE); 2319 2320 if (buftype == DMA_OTYP_PAGES) { 2321 /* get the paddr from the page_t */ 2322 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2323 paddr = pfn_to_pa(pp->p_pagenum); 2324 pp = pp->p_next; 2325 } else if (pplist != NULL) { 2326 /* index into the array of page_t's to get the paddr */ 2327 ASSERT(!PP_ISFREE(pplist[pcnt])); 2328 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2329 pcnt++; 2330 } else { 2331 /* call into the VM to get the paddr */ 2332 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2333 vaddr)); 2334 vaddr += psize; 2335 } 2336 2337 #ifdef __xpv 2338 /* 2339 * If we're dom0, we're using a real device so we need to load 2340 * the cookies with MFNs instead of PFNs. 2341 */ 2342 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2343 #else 2344 raddr = paddr; 2345 #endif 2346 2347 /* check to see if this page needs the copy buffer */ 2348 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2349 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2350 2351 /* 2352 * if there is something in the current cookie, go to 2353 * the next one. We only want one page in a cookie which 2354 * uses the copybuf since the copybuf doesn't have to 2355 * be physically contiguous. 2356 */ 2357 if (sgl[cnt].dmac_size != 0) { 2358 cnt++; 2359 } 2360 sgl[cnt].dmac_laddress = raddr; 2361 sgl[cnt].dmac_size = psize; 2362 #if defined(__amd64) 2363 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2364 #else 2365 /* 2366 * save the buf offset for 32-bit kernel. used in the 2367 * obsoleted interfaces. 2368 */ 2369 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2370 (dmar_object->dmao_size - size); 2371 #endif 2372 /* if this isn't the last cookie, go to the next one */ 2373 if ((cnt + 1) < sglinfo->si_max_pages) { 2374 cnt++; 2375 sgl[cnt].dmac_laddress = 0; 2376 sgl[cnt].dmac_size = 0; 2377 sgl[cnt].dmac_type = 0; 2378 } 2379 2380 /* 2381 * this page didn't need the copy buffer, if it's not physically 2382 * contiguous, or it would put us over a segment boundary, or it 2383 * puts us over the max cookie size, or the current sgl doesn't 2384 * have anything in it. 2385 */ 2386 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2387 !(raddr & sglinfo->si_segmask) || 2388 ((sgl[cnt].dmac_size + psize) > maxseg) || 2389 (sgl[cnt].dmac_size == 0)) { 2390 /* 2391 * if we're not already in a new cookie, go to the next 2392 * cookie. 2393 */ 2394 if (sgl[cnt].dmac_size != 0) { 2395 cnt++; 2396 } 2397 2398 /* save the cookie information */ 2399 sgl[cnt].dmac_laddress = raddr; 2400 sgl[cnt].dmac_size = psize; 2401 #if defined(__amd64) 2402 sgl[cnt].dmac_type = 0; 2403 #else 2404 /* 2405 * save the buf offset for 32-bit kernel. used in the 2406 * obsoleted interfaces. 2407 */ 2408 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2409 #endif 2410 2411 /* 2412 * this page didn't need the copy buffer, it is physically 2413 * contiguous with the last page, and it's <= the max cookie 2414 * size. 2415 */ 2416 } else { 2417 sgl[cnt].dmac_size += psize; 2418 2419 /* 2420 * if this exactly == the maximum cookie size, and 2421 * it isn't the last cookie, go to the next cookie. 2422 */ 2423 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2424 ((cnt + 1) < sglinfo->si_max_pages)) { 2425 cnt++; 2426 sgl[cnt].dmac_laddress = 0; 2427 sgl[cnt].dmac_size = 0; 2428 sgl[cnt].dmac_type = 0; 2429 } 2430 } 2431 2432 /* 2433 * save this page's physical address so we can figure out if the 2434 * next page is physically contiguous. Keep decrementing size 2435 * until we are done with the buffer. 2436 */ 2437 last_page = raddr; 2438 size -= psize; 2439 } 2440 2441 /* we're done, save away how many cookies the sgl has */ 2442 if (sgl[cnt].dmac_size == 0) { 2443 ASSERT(cnt < sglinfo->si_max_pages); 2444 sglinfo->si_sgl_size = cnt; 2445 } else { 2446 sglinfo->si_sgl_size = cnt + 1; 2447 } 2448 } 2449 2450 2451 /* 2452 * rootnex_bind_slowpath() 2453 * Call in the bind path if the calling driver can't use the sgl without 2454 * modifying it. We either need to use the copy buffer and/or we will end up 2455 * with a partial bind. 2456 */ 2457 static int 2458 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2459 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2460 { 2461 rootnex_sglinfo_t *sinfo; 2462 rootnex_window_t *window; 2463 ddi_dma_cookie_t *cookie; 2464 size_t copybuf_used; 2465 size_t dmac_size; 2466 boolean_t partial; 2467 off_t cur_offset; 2468 page_t *cur_pp; 2469 major_t mnum; 2470 int e; 2471 int i; 2472 2473 2474 sinfo = &dma->dp_sglinfo; 2475 copybuf_used = 0; 2476 partial = B_FALSE; 2477 2478 /* 2479 * If we're using the copybuf, set the copybuf state in dma struct. 2480 * Needs to be first since it sets the copy buffer size. 2481 */ 2482 if (sinfo->si_copybuf_req != 0) { 2483 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2484 if (e != DDI_SUCCESS) { 2485 return (e); 2486 } 2487 } else { 2488 dma->dp_copybuf_size = 0; 2489 } 2490 2491 /* 2492 * Figure out if we need to do a partial mapping. If so, figure out 2493 * if we need to trim the buffers when we munge the sgl. 2494 */ 2495 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2496 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2497 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2498 dma->dp_partial_required = B_TRUE; 2499 if (attr->dma_attr_granular != 1) { 2500 dma->dp_trim_required = B_TRUE; 2501 } 2502 } else { 2503 dma->dp_partial_required = B_FALSE; 2504 dma->dp_trim_required = B_FALSE; 2505 } 2506 2507 /* If we need to do a partial bind, make sure the driver supports it */ 2508 if (dma->dp_partial_required && 2509 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2510 2511 mnum = ddi_driver_major(dma->dp_dip); 2512 /* 2513 * patchable which allows us to print one warning per major 2514 * number. 2515 */ 2516 if ((rootnex_bind_warn) && 2517 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2518 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2519 cmn_err(CE_WARN, "!%s: coding error detected, the " 2520 "driver is using ddi_dma_attr(9S) incorrectly. " 2521 "There is a small risk of data corruption in " 2522 "particular with large I/Os. The driver should be " 2523 "replaced with a corrected version for proper " 2524 "system operation. To disable this warning, add " 2525 "'set rootnex:rootnex_bind_warn=0' to " 2526 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2527 } 2528 return (DDI_DMA_TOOBIG); 2529 } 2530 2531 /* 2532 * we might need multiple windows, setup state to handle them. In this 2533 * code path, we will have at least one window. 2534 */ 2535 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2536 if (e != DDI_SUCCESS) { 2537 rootnex_teardown_copybuf(dma); 2538 return (e); 2539 } 2540 2541 window = &dma->dp_window[0]; 2542 cookie = &dma->dp_cookies[0]; 2543 cur_offset = 0; 2544 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2545 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2546 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2547 } 2548 2549 /* loop though all the cookies we got back from get_sgl() */ 2550 for (i = 0; i < sinfo->si_sgl_size; i++) { 2551 /* 2552 * If we're using the copy buffer, check this cookie and setup 2553 * its associated copy buffer state. If this cookie uses the 2554 * copy buffer, make sure we sync this window during dma_sync. 2555 */ 2556 if (dma->dp_copybuf_size > 0) { 2557 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2558 cur_offset, ©buf_used, &cur_pp); 2559 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2560 window->wd_dosync = B_TRUE; 2561 } 2562 } 2563 2564 /* 2565 * save away the cookie size, since it could be modified in 2566 * the windowing code. 2567 */ 2568 dmac_size = cookie->dmac_size; 2569 2570 /* if we went over max copybuf size */ 2571 if (dma->dp_copybuf_size && 2572 (copybuf_used > dma->dp_copybuf_size)) { 2573 partial = B_TRUE; 2574 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2575 cookie, cur_offset, ©buf_used); 2576 if (e != DDI_SUCCESS) { 2577 rootnex_teardown_copybuf(dma); 2578 rootnex_teardown_windows(dma); 2579 return (e); 2580 } 2581 2582 /* 2583 * if the coookie uses the copy buffer, make sure the 2584 * new window we just moved to is set to sync. 2585 */ 2586 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2587 window->wd_dosync = B_TRUE; 2588 } 2589 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2590 dma->dp_dip); 2591 2592 /* if the cookie cnt == max sgllen, move to the next window */ 2593 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2594 partial = B_TRUE; 2595 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2596 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2597 cookie, attr, cur_offset); 2598 if (e != DDI_SUCCESS) { 2599 rootnex_teardown_copybuf(dma); 2600 rootnex_teardown_windows(dma); 2601 return (e); 2602 } 2603 2604 /* 2605 * if the coookie uses the copy buffer, make sure the 2606 * new window we just moved to is set to sync. 2607 */ 2608 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2609 window->wd_dosync = B_TRUE; 2610 } 2611 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2612 dma->dp_dip); 2613 2614 /* else if we will be over maxxfer */ 2615 } else if ((window->wd_size + dmac_size) > 2616 dma->dp_maxxfer) { 2617 partial = B_TRUE; 2618 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2619 cookie); 2620 if (e != DDI_SUCCESS) { 2621 rootnex_teardown_copybuf(dma); 2622 rootnex_teardown_windows(dma); 2623 return (e); 2624 } 2625 2626 /* 2627 * if the coookie uses the copy buffer, make sure the 2628 * new window we just moved to is set to sync. 2629 */ 2630 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2631 window->wd_dosync = B_TRUE; 2632 } 2633 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2634 dma->dp_dip); 2635 2636 /* else this cookie fits in the current window */ 2637 } else { 2638 window->wd_cookie_cnt++; 2639 window->wd_size += dmac_size; 2640 } 2641 2642 /* track our offset into the buffer, go to the next cookie */ 2643 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2644 ASSERT(cookie->dmac_size <= dmac_size); 2645 cur_offset += dmac_size; 2646 cookie++; 2647 } 2648 2649 /* if we ended up with a zero sized window in the end, clean it up */ 2650 if (window->wd_size == 0) { 2651 hp->dmai_nwin--; 2652 window--; 2653 } 2654 2655 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2656 2657 if (!partial) { 2658 return (DDI_DMA_MAPPED); 2659 } 2660 2661 ASSERT(dma->dp_partial_required); 2662 return (DDI_DMA_PARTIAL_MAP); 2663 } 2664 2665 2666 /* 2667 * rootnex_setup_copybuf() 2668 * Called in bind slowpath. Figures out if we're going to use the copy 2669 * buffer, and if we do, sets up the basic state to handle it. 2670 */ 2671 static int 2672 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2673 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2674 { 2675 rootnex_sglinfo_t *sinfo; 2676 ddi_dma_attr_t lattr; 2677 size_t max_copybuf; 2678 int cansleep; 2679 int e; 2680 #if !defined(__amd64) 2681 int vmflag; 2682 #endif 2683 2684 2685 sinfo = &dma->dp_sglinfo; 2686 2687 /* read this first so it's consistent through the routine */ 2688 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 2689 2690 /* We need to call into the rootnex on ddi_dma_sync() */ 2691 hp->dmai_rflags &= ~DMP_NOSYNC; 2692 2693 /* make sure the copybuf size <= the max size */ 2694 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2695 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2696 2697 #if !defined(__amd64) 2698 /* 2699 * if we don't have kva space to copy to/from, allocate the KVA space 2700 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2701 * the 64-bit kernel. 2702 */ 2703 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2704 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2705 2706 /* convert the sleep flags */ 2707 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2708 vmflag = VM_SLEEP; 2709 } else { 2710 vmflag = VM_NOSLEEP; 2711 } 2712 2713 /* allocate Kernel VA space that we can bcopy to/from */ 2714 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2715 vmflag); 2716 if (dma->dp_kva == NULL) { 2717 return (DDI_DMA_NORESOURCES); 2718 } 2719 } 2720 #endif 2721 2722 /* convert the sleep flags */ 2723 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2724 cansleep = 1; 2725 } else { 2726 cansleep = 0; 2727 } 2728 2729 /* 2730 * Allocate the actual copy buffer. This needs to fit within the DMA 2731 * engine limits, so we can't use kmem_alloc... We don't need 2732 * contiguous memory (sgllen) since we will be forcing windows on 2733 * sgllen anyway. 2734 */ 2735 lattr = *attr; 2736 lattr.dma_attr_align = MMU_PAGESIZE; 2737 /* 2738 * this should be < 0 to indicate no limit, but due to a bug in 2739 * the rootnex, we'll set it to the maximum positive int. 2740 */ 2741 lattr.dma_attr_sgllen = 0x7fffffff; 2742 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2743 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2744 if (e != DDI_SUCCESS) { 2745 #if !defined(__amd64) 2746 if (dma->dp_kva != NULL) { 2747 vmem_free(heap_arena, dma->dp_kva, 2748 dma->dp_copybuf_size); 2749 } 2750 #endif 2751 return (DDI_DMA_NORESOURCES); 2752 } 2753 2754 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2755 size_t, dma->dp_copybuf_size); 2756 2757 return (DDI_SUCCESS); 2758 } 2759 2760 2761 /* 2762 * rootnex_setup_windows() 2763 * Called in bind slowpath to setup the window state. We always have windows 2764 * in the slowpath. Even if the window count = 1. 2765 */ 2766 static int 2767 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2768 ddi_dma_attr_t *attr, int kmflag) 2769 { 2770 rootnex_window_t *windowp; 2771 rootnex_sglinfo_t *sinfo; 2772 size_t copy_state_size; 2773 size_t win_state_size; 2774 size_t state_available; 2775 size_t space_needed; 2776 uint_t copybuf_win; 2777 uint_t maxxfer_win; 2778 size_t space_used; 2779 uint_t sglwin; 2780 2781 2782 sinfo = &dma->dp_sglinfo; 2783 2784 dma->dp_current_win = 0; 2785 hp->dmai_nwin = 0; 2786 2787 /* If we don't need to do a partial, we only have one window */ 2788 if (!dma->dp_partial_required) { 2789 dma->dp_max_win = 1; 2790 2791 /* 2792 * we need multiple windows, need to figure out the worse case number 2793 * of windows. 2794 */ 2795 } else { 2796 /* 2797 * if we need windows because we need more copy buffer that 2798 * we allow, the worse case number of windows we could need 2799 * here would be (copybuf space required / copybuf space that 2800 * we have) plus one for remainder, and plus 2 to handle the 2801 * extra pages on the trim for the first and last pages of the 2802 * buffer (a page is the minimum window size so under the right 2803 * attr settings, you could have a window for each page). 2804 * The last page will only be hit here if the size is not a 2805 * multiple of the granularity (which theoretically shouldn't 2806 * be the case but never has been enforced, so we could have 2807 * broken things without it). 2808 */ 2809 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 2810 ASSERT(dma->dp_copybuf_size > 0); 2811 copybuf_win = (sinfo->si_copybuf_req / 2812 dma->dp_copybuf_size) + 1 + 2; 2813 } else { 2814 copybuf_win = 0; 2815 } 2816 2817 /* 2818 * if we need windows because we have more cookies than the H/W 2819 * can handle, the number of windows we would need here would 2820 * be (cookie count / cookies count H/W supports) plus one for 2821 * remainder, and plus 2 to handle the extra pages on the trim 2822 * (see above comment about trim) 2823 */ 2824 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 2825 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 2826 + 1) + 2; 2827 } else { 2828 sglwin = 0; 2829 } 2830 2831 /* 2832 * if we need windows because we're binding more memory than the 2833 * H/W can transfer at once, the number of windows we would need 2834 * here would be (xfer count / max xfer H/W supports) plus one 2835 * for remainder, and plus 2 to handle the extra pages on the 2836 * trim (see above comment about trim) 2837 */ 2838 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 2839 maxxfer_win = (dma->dp_dma.dmao_size / 2840 dma->dp_maxxfer) + 1 + 2; 2841 } else { 2842 maxxfer_win = 0; 2843 } 2844 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 2845 ASSERT(dma->dp_max_win > 0); 2846 } 2847 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 2848 2849 /* 2850 * Get space for window and potential copy buffer state. Before we 2851 * go and allocate memory, see if we can get away with using what's 2852 * left in the pre-allocted state or the dynamically allocated sgl. 2853 */ 2854 space_used = (uintptr_t)(sinfo->si_sgl_size * 2855 sizeof (ddi_dma_cookie_t)); 2856 2857 /* if we dynamically allocated space for the cookies */ 2858 if (dma->dp_need_to_free_cookie) { 2859 /* if we have more space in the pre-allocted buffer, use it */ 2860 ASSERT(space_used <= dma->dp_cookie_size); 2861 if ((dma->dp_cookie_size - space_used) <= 2862 rootnex_state->r_prealloc_size) { 2863 state_available = rootnex_state->r_prealloc_size; 2864 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 2865 2866 /* 2867 * else, we have more free space in the dynamically allocated 2868 * buffer, i.e. the buffer wasn't worse case fragmented so we 2869 * didn't need a lot of cookies. 2870 */ 2871 } else { 2872 state_available = dma->dp_cookie_size - space_used; 2873 windowp = (rootnex_window_t *) 2874 &dma->dp_cookies[sinfo->si_sgl_size]; 2875 } 2876 2877 /* we used the pre-alloced buffer */ 2878 } else { 2879 ASSERT(space_used <= rootnex_state->r_prealloc_size); 2880 state_available = rootnex_state->r_prealloc_size - space_used; 2881 windowp = (rootnex_window_t *) 2882 &dma->dp_cookies[sinfo->si_sgl_size]; 2883 } 2884 2885 /* 2886 * figure out how much state we need to track the copy buffer. Add an 2887 * addition 8 bytes for pointer alignemnt later. 2888 */ 2889 if (dma->dp_copybuf_size > 0) { 2890 copy_state_size = sinfo->si_max_pages * 2891 sizeof (rootnex_pgmap_t); 2892 } else { 2893 copy_state_size = 0; 2894 } 2895 /* add an additional 8 bytes for pointer alignment */ 2896 space_needed = win_state_size + copy_state_size + 0x8; 2897 2898 /* if we have enough space already, use it */ 2899 if (state_available >= space_needed) { 2900 dma->dp_window = windowp; 2901 dma->dp_need_to_free_window = B_FALSE; 2902 2903 /* not enough space, need to allocate more. */ 2904 } else { 2905 dma->dp_window = kmem_alloc(space_needed, kmflag); 2906 if (dma->dp_window == NULL) { 2907 return (DDI_DMA_NORESOURCES); 2908 } 2909 dma->dp_need_to_free_window = B_TRUE; 2910 dma->dp_window_size = space_needed; 2911 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 2912 dma->dp_dip, size_t, space_needed); 2913 } 2914 2915 /* 2916 * we allocate copy buffer state and window state at the same time. 2917 * setup our copy buffer state pointers. Make sure it's aligned. 2918 */ 2919 if (dma->dp_copybuf_size > 0) { 2920 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 2921 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 2922 2923 #if !defined(__amd64) 2924 /* 2925 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 2926 * false/NULL. Should be quicker to bzero vs loop and set. 2927 */ 2928 bzero(dma->dp_pgmap, copy_state_size); 2929 #endif 2930 } else { 2931 dma->dp_pgmap = NULL; 2932 } 2933 2934 return (DDI_SUCCESS); 2935 } 2936 2937 2938 /* 2939 * rootnex_teardown_copybuf() 2940 * cleans up after rootnex_setup_copybuf() 2941 */ 2942 static void 2943 rootnex_teardown_copybuf(rootnex_dma_t *dma) 2944 { 2945 #if !defined(__amd64) 2946 int i; 2947 2948 /* 2949 * if we allocated kernel heap VMEM space, go through all the pages and 2950 * map out any of the ones that we're mapped into the kernel heap VMEM 2951 * arena. Then free the VMEM space. 2952 */ 2953 if (dma->dp_kva != NULL) { 2954 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 2955 if (dma->dp_pgmap[i].pm_mapped) { 2956 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 2957 MMU_PAGESIZE, HAT_UNLOAD); 2958 dma->dp_pgmap[i].pm_mapped = B_FALSE; 2959 } 2960 } 2961 2962 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 2963 } 2964 2965 #endif 2966 2967 /* if we allocated a copy buffer, free it */ 2968 if (dma->dp_cbaddr != NULL) { 2969 i_ddi_mem_free(dma->dp_cbaddr, NULL); 2970 } 2971 } 2972 2973 2974 /* 2975 * rootnex_teardown_windows() 2976 * cleans up after rootnex_setup_windows() 2977 */ 2978 static void 2979 rootnex_teardown_windows(rootnex_dma_t *dma) 2980 { 2981 /* 2982 * if we had to allocate window state on the last bind (because we 2983 * didn't have enough pre-allocated space in the handle), free it. 2984 */ 2985 if (dma->dp_need_to_free_window) { 2986 kmem_free(dma->dp_window, dma->dp_window_size); 2987 } 2988 } 2989 2990 2991 /* 2992 * rootnex_init_win() 2993 * Called in bind slow path during creation of a new window. Initializes 2994 * window state to default values. 2995 */ 2996 /*ARGSUSED*/ 2997 static void 2998 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2999 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3000 { 3001 hp->dmai_nwin++; 3002 window->wd_dosync = B_FALSE; 3003 window->wd_offset = cur_offset; 3004 window->wd_size = 0; 3005 window->wd_first_cookie = cookie; 3006 window->wd_cookie_cnt = 0; 3007 window->wd_trim.tr_trim_first = B_FALSE; 3008 window->wd_trim.tr_trim_last = B_FALSE; 3009 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3010 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3011 #if !defined(__amd64) 3012 window->wd_remap_copybuf = dma->dp_cb_remaping; 3013 #endif 3014 } 3015 3016 3017 /* 3018 * rootnex_setup_cookie() 3019 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3020 * the sgl uses the copy buffer, we need to go through each cookie, figure 3021 * out if it uses the copy buffer, and if it does, save away everything we'll 3022 * need during sync. 3023 */ 3024 static void 3025 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3026 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3027 page_t **cur_pp) 3028 { 3029 boolean_t copybuf_sz_power_2; 3030 rootnex_sglinfo_t *sinfo; 3031 paddr_t paddr; 3032 uint_t pidx; 3033 uint_t pcnt; 3034 off_t poff; 3035 #if defined(__amd64) 3036 pfn_t pfn; 3037 #else 3038 page_t **pplist; 3039 #endif 3040 3041 sinfo = &dma->dp_sglinfo; 3042 3043 /* 3044 * Calculate the page index relative to the start of the buffer. The 3045 * index to the current page for our buffer is the offset into the 3046 * first page of the buffer plus our current offset into the buffer 3047 * itself, shifted of course... 3048 */ 3049 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3050 ASSERT(pidx < sinfo->si_max_pages); 3051 3052 /* if this cookie uses the copy buffer */ 3053 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3054 /* 3055 * NOTE: we know that since this cookie uses the copy buffer, it 3056 * is <= MMU_PAGESIZE. 3057 */ 3058 3059 /* 3060 * get the offset into the page. For the 64-bit kernel, get the 3061 * pfn which we'll use with seg kpm. 3062 */ 3063 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3064 #if defined(__amd64) 3065 /* mfn_to_pfn() is a NOP on i86pc */ 3066 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3067 #endif /* __amd64 */ 3068 3069 /* figure out if the copybuf size is a power of 2 */ 3070 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3071 copybuf_sz_power_2 = B_FALSE; 3072 } else { 3073 copybuf_sz_power_2 = B_TRUE; 3074 } 3075 3076 /* This page uses the copy buffer */ 3077 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3078 3079 /* 3080 * save the copy buffer KVA that we'll use with this page. 3081 * if we still fit within the copybuf, it's a simple add. 3082 * otherwise, we need to wrap over using & or % accordingly. 3083 */ 3084 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3085 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3086 *copybuf_used; 3087 } else { 3088 if (copybuf_sz_power_2) { 3089 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3090 (uintptr_t)dma->dp_cbaddr + 3091 (*copybuf_used & 3092 (dma->dp_copybuf_size - 1))); 3093 } else { 3094 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3095 (uintptr_t)dma->dp_cbaddr + 3096 (*copybuf_used % dma->dp_copybuf_size)); 3097 } 3098 } 3099 3100 /* 3101 * over write the cookie physical address with the address of 3102 * the physical address of the copy buffer page that we will 3103 * use. 3104 */ 3105 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3106 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3107 3108 #ifdef __xpv 3109 /* 3110 * If we're dom0, we're using a real device so we need to load 3111 * the cookies with MAs instead of PAs. 3112 */ 3113 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3114 #else 3115 cookie->dmac_laddress = paddr; 3116 #endif 3117 3118 /* if we have a kernel VA, it's easy, just save that address */ 3119 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3120 (sinfo->si_asp == &kas)) { 3121 /* 3122 * save away the page aligned virtual address of the 3123 * driver buffer. Offsets are handled in the sync code. 3124 */ 3125 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3126 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3127 & MMU_PAGEMASK); 3128 #if !defined(__amd64) 3129 /* 3130 * we didn't need to, and will never need to map this 3131 * page. 3132 */ 3133 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3134 #endif 3135 3136 /* we don't have a kernel VA. We need one for the bcopy. */ 3137 } else { 3138 #if defined(__amd64) 3139 /* 3140 * for the 64-bit kernel, it's easy. We use seg kpm to 3141 * get a Kernel VA for the corresponding pfn. 3142 */ 3143 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3144 #else 3145 /* 3146 * for the 32-bit kernel, this is a pain. First we'll 3147 * save away the page_t or user VA for this page. This 3148 * is needed in rootnex_dma_win() when we switch to a 3149 * new window which requires us to re-map the copy 3150 * buffer. 3151 */ 3152 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3153 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3154 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3155 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3156 } else if (pplist != NULL) { 3157 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3158 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3159 } else { 3160 dma->dp_pgmap[pidx].pm_pp = NULL; 3161 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3162 (((uintptr_t) 3163 dmar_object->dmao_obj.virt_obj.v_addr + 3164 cur_offset) & MMU_PAGEMASK); 3165 } 3166 3167 /* 3168 * save away the page aligned virtual address which was 3169 * allocated from the kernel heap arena (taking into 3170 * account if we need more copy buffer than we alloced 3171 * and use multiple windows to handle this, i.e. &,%). 3172 * NOTE: there isn't and physical memory backing up this 3173 * virtual address space currently. 3174 */ 3175 if ((*copybuf_used + MMU_PAGESIZE) <= 3176 dma->dp_copybuf_size) { 3177 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3178 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3179 MMU_PAGEMASK); 3180 } else { 3181 if (copybuf_sz_power_2) { 3182 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3183 (((uintptr_t)dma->dp_kva + 3184 (*copybuf_used & 3185 (dma->dp_copybuf_size - 1))) & 3186 MMU_PAGEMASK); 3187 } else { 3188 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3189 (((uintptr_t)dma->dp_kva + 3190 (*copybuf_used % 3191 dma->dp_copybuf_size)) & 3192 MMU_PAGEMASK); 3193 } 3194 } 3195 3196 /* 3197 * if we haven't used up the available copy buffer yet, 3198 * map the kva to the physical page. 3199 */ 3200 if (!dma->dp_cb_remaping && ((*copybuf_used + 3201 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3202 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3203 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3204 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3205 dma->dp_pgmap[pidx].pm_kaddr); 3206 } else { 3207 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3208 sinfo->si_asp, 3209 dma->dp_pgmap[pidx].pm_kaddr); 3210 } 3211 3212 /* 3213 * we've used up the available copy buffer, this page 3214 * will have to be mapped during rootnex_dma_win() when 3215 * we switch to a new window which requires a re-map 3216 * the copy buffer. (32-bit kernel only) 3217 */ 3218 } else { 3219 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3220 } 3221 #endif 3222 /* go to the next page_t */ 3223 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3224 *cur_pp = (*cur_pp)->p_next; 3225 } 3226 } 3227 3228 /* add to the copy buffer count */ 3229 *copybuf_used += MMU_PAGESIZE; 3230 3231 /* 3232 * This cookie doesn't use the copy buffer. Walk through the pages this 3233 * cookie occupies to reflect this. 3234 */ 3235 } else { 3236 /* 3237 * figure out how many pages the cookie occupies. We need to 3238 * use the original page offset of the buffer and the cookies 3239 * offset in the buffer to do this. 3240 */ 3241 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3242 pcnt = mmu_btopr(cookie->dmac_size + poff); 3243 3244 while (pcnt > 0) { 3245 #if !defined(__amd64) 3246 /* 3247 * the 32-bit kernel doesn't have seg kpm, so we need 3248 * to map in the driver buffer (if it didn't come down 3249 * with a kernel VA) on the fly. Since this page doesn't 3250 * use the copy buffer, it's not, or will it ever, have 3251 * to be mapped in. 3252 */ 3253 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3254 #endif 3255 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3256 3257 /* 3258 * we need to update pidx and cur_pp or we'll loose 3259 * track of where we are. 3260 */ 3261 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3262 *cur_pp = (*cur_pp)->p_next; 3263 } 3264 pidx++; 3265 pcnt--; 3266 } 3267 } 3268 } 3269 3270 3271 /* 3272 * rootnex_sgllen_window_boundary() 3273 * Called in the bind slow path when the next cookie causes us to exceed (in 3274 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3275 * length supported by the DMA H/W. 3276 */ 3277 static int 3278 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3279 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3280 off_t cur_offset) 3281 { 3282 off_t new_offset; 3283 size_t trim_sz; 3284 off_t coffset; 3285 3286 3287 /* 3288 * if we know we'll never have to trim, it's pretty easy. Just move to 3289 * the next window and init it. We're done. 3290 */ 3291 if (!dma->dp_trim_required) { 3292 (*windowp)++; 3293 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3294 (*windowp)->wd_cookie_cnt++; 3295 (*windowp)->wd_size = cookie->dmac_size; 3296 return (DDI_SUCCESS); 3297 } 3298 3299 /* figure out how much we need to trim from the window */ 3300 ASSERT(attr->dma_attr_granular != 0); 3301 if (dma->dp_granularity_power_2) { 3302 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3303 } else { 3304 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3305 } 3306 3307 /* The window's a whole multiple of granularity. We're done */ 3308 if (trim_sz == 0) { 3309 (*windowp)++; 3310 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3311 (*windowp)->wd_cookie_cnt++; 3312 (*windowp)->wd_size = cookie->dmac_size; 3313 return (DDI_SUCCESS); 3314 } 3315 3316 /* 3317 * The window's not a whole multiple of granularity, since we know this 3318 * is due to the sgllen, we need to go back to the last cookie and trim 3319 * that one, add the left over part of the old cookie into the new 3320 * window, and then add in the new cookie into the new window. 3321 */ 3322 3323 /* 3324 * make sure the driver isn't making us do something bad... Trimming and 3325 * sgllen == 1 don't go together. 3326 */ 3327 if (attr->dma_attr_sgllen == 1) { 3328 return (DDI_DMA_NOMAPPING); 3329 } 3330 3331 /* 3332 * first, setup the current window to account for the trim. Need to go 3333 * back to the last cookie for this. 3334 */ 3335 cookie--; 3336 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3337 (*windowp)->wd_trim.tr_last_cookie = cookie; 3338 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3339 ASSERT(cookie->dmac_size > trim_sz); 3340 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3341 (*windowp)->wd_size -= trim_sz; 3342 3343 /* save the buffer offsets for the next window */ 3344 coffset = cookie->dmac_size - trim_sz; 3345 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3346 3347 /* 3348 * set this now in case this is the first window. all other cases are 3349 * set in dma_win() 3350 */ 3351 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3352 3353 /* 3354 * initialize the next window using what's left over in the previous 3355 * cookie. 3356 */ 3357 (*windowp)++; 3358 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3359 (*windowp)->wd_cookie_cnt++; 3360 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3361 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3362 (*windowp)->wd_trim.tr_first_size = trim_sz; 3363 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3364 (*windowp)->wd_dosync = B_TRUE; 3365 } 3366 3367 /* 3368 * now go back to the current cookie and add it to the new window. set 3369 * the new window size to the what was left over from the previous 3370 * cookie and what's in the current cookie. 3371 */ 3372 cookie++; 3373 (*windowp)->wd_cookie_cnt++; 3374 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3375 3376 /* 3377 * trim plus the next cookie could put us over maxxfer (a cookie can be 3378 * a max size of maxxfer). Handle that case. 3379 */ 3380 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3381 /* 3382 * maxxfer is already a whole multiple of granularity, and this 3383 * trim will be <= the previous trim (since a cookie can't be 3384 * larger than maxxfer). Make things simple here. 3385 */ 3386 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3387 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3388 (*windowp)->wd_trim.tr_last_cookie = cookie; 3389 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3390 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3391 (*windowp)->wd_size -= trim_sz; 3392 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3393 3394 /* save the buffer offsets for the next window */ 3395 coffset = cookie->dmac_size - trim_sz; 3396 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3397 3398 /* setup the next window */ 3399 (*windowp)++; 3400 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3401 (*windowp)->wd_cookie_cnt++; 3402 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3403 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3404 coffset; 3405 (*windowp)->wd_trim.tr_first_size = trim_sz; 3406 } 3407 3408 return (DDI_SUCCESS); 3409 } 3410 3411 3412 /* 3413 * rootnex_copybuf_window_boundary() 3414 * Called in bind slowpath when we get to a window boundary because we used 3415 * up all the copy buffer that we have. 3416 */ 3417 static int 3418 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3419 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3420 size_t *copybuf_used) 3421 { 3422 rootnex_sglinfo_t *sinfo; 3423 off_t new_offset; 3424 size_t trim_sz; 3425 paddr_t paddr; 3426 off_t coffset; 3427 uint_t pidx; 3428 off_t poff; 3429 3430 3431 sinfo = &dma->dp_sglinfo; 3432 3433 /* 3434 * the copy buffer should be a whole multiple of page size. We know that 3435 * this cookie is <= MMU_PAGESIZE. 3436 */ 3437 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3438 3439 /* 3440 * from now on, all new windows in this bind need to be re-mapped during 3441 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3442 * space... 3443 */ 3444 #if !defined(__amd64) 3445 dma->dp_cb_remaping = B_TRUE; 3446 #endif 3447 3448 /* reset copybuf used */ 3449 *copybuf_used = 0; 3450 3451 /* 3452 * if we don't have to trim (since granularity is set to 1), go to the 3453 * next window and add the current cookie to it. We know the current 3454 * cookie uses the copy buffer since we're in this code path. 3455 */ 3456 if (!dma->dp_trim_required) { 3457 (*windowp)++; 3458 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3459 3460 /* Add this cookie to the new window */ 3461 (*windowp)->wd_cookie_cnt++; 3462 (*windowp)->wd_size += cookie->dmac_size; 3463 *copybuf_used += MMU_PAGESIZE; 3464 return (DDI_SUCCESS); 3465 } 3466 3467 /* 3468 * *** may need to trim, figure it out. 3469 */ 3470 3471 /* figure out how much we need to trim from the window */ 3472 if (dma->dp_granularity_power_2) { 3473 trim_sz = (*windowp)->wd_size & 3474 (hp->dmai_attr.dma_attr_granular - 1); 3475 } else { 3476 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3477 } 3478 3479 /* 3480 * if the window's a whole multiple of granularity, go to the next 3481 * window, init it, then add in the current cookie. We know the current 3482 * cookie uses the copy buffer since we're in this code path. 3483 */ 3484 if (trim_sz == 0) { 3485 (*windowp)++; 3486 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3487 3488 /* Add this cookie to the new window */ 3489 (*windowp)->wd_cookie_cnt++; 3490 (*windowp)->wd_size += cookie->dmac_size; 3491 *copybuf_used += MMU_PAGESIZE; 3492 return (DDI_SUCCESS); 3493 } 3494 3495 /* 3496 * *** We figured it out, we definitly need to trim 3497 */ 3498 3499 /* 3500 * make sure the driver isn't making us do something bad... 3501 * Trimming and sgllen == 1 don't go together. 3502 */ 3503 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3504 return (DDI_DMA_NOMAPPING); 3505 } 3506 3507 /* 3508 * first, setup the current window to account for the trim. Need to go 3509 * back to the last cookie for this. Some of the last cookie will be in 3510 * the current window, and some of the last cookie will be in the new 3511 * window. All of the current cookie will be in the new window. 3512 */ 3513 cookie--; 3514 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3515 (*windowp)->wd_trim.tr_last_cookie = cookie; 3516 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3517 ASSERT(cookie->dmac_size > trim_sz); 3518 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3519 (*windowp)->wd_size -= trim_sz; 3520 3521 /* 3522 * we're trimming the last cookie (not the current cookie). So that 3523 * last cookie may have or may not have been using the copy buffer ( 3524 * we know the cookie passed in uses the copy buffer since we're in 3525 * this code path). 3526 * 3527 * If the last cookie doesn't use the copy buffer, nothing special to 3528 * do. However, if it does uses the copy buffer, it will be both the 3529 * last page in the current window and the first page in the next 3530 * window. Since we are reusing the copy buffer (and KVA space on the 3531 * 32-bit kernel), this page will use the end of the copy buffer in the 3532 * current window, and the start of the copy buffer in the next window. 3533 * Track that info... The cookie physical address was already set to 3534 * the copy buffer physical address in setup_cookie.. 3535 */ 3536 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3537 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3538 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3539 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3540 (*windowp)->wd_trim.tr_last_pidx = pidx; 3541 (*windowp)->wd_trim.tr_last_cbaddr = 3542 dma->dp_pgmap[pidx].pm_cbaddr; 3543 #if !defined(__amd64) 3544 (*windowp)->wd_trim.tr_last_kaddr = 3545 dma->dp_pgmap[pidx].pm_kaddr; 3546 #endif 3547 } 3548 3549 /* save the buffer offsets for the next window */ 3550 coffset = cookie->dmac_size - trim_sz; 3551 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3552 3553 /* 3554 * set this now in case this is the first window. all other cases are 3555 * set in dma_win() 3556 */ 3557 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3558 3559 /* 3560 * initialize the next window using what's left over in the previous 3561 * cookie. 3562 */ 3563 (*windowp)++; 3564 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3565 (*windowp)->wd_cookie_cnt++; 3566 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3567 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3568 (*windowp)->wd_trim.tr_first_size = trim_sz; 3569 3570 /* 3571 * again, we're tracking if the last cookie uses the copy buffer. 3572 * read the comment above for more info on why we need to track 3573 * additional state. 3574 * 3575 * For the first cookie in the new window, we need reset the physical 3576 * address to DMA into to the start of the copy buffer plus any 3577 * initial page offset which may be present. 3578 */ 3579 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3580 (*windowp)->wd_dosync = B_TRUE; 3581 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3582 (*windowp)->wd_trim.tr_first_pidx = pidx; 3583 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3584 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3585 3586 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3587 poff; 3588 #ifdef __xpv 3589 /* 3590 * If we're dom0, we're using a real device so we need to load 3591 * the cookies with MAs instead of PAs. 3592 */ 3593 (*windowp)->wd_trim.tr_first_paddr = 3594 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3595 #else 3596 (*windowp)->wd_trim.tr_first_paddr = paddr; 3597 #endif 3598 3599 #if !defined(__amd64) 3600 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3601 #endif 3602 /* account for the cookie copybuf usage in the new window */ 3603 *copybuf_used += MMU_PAGESIZE; 3604 3605 /* 3606 * every piece of code has to have a hack, and here is this 3607 * ones :-) 3608 * 3609 * There is a complex interaction between setup_cookie and the 3610 * copybuf window boundary. The complexity had to be in either 3611 * the maxxfer window, or the copybuf window, and I chose the 3612 * copybuf code. 3613 * 3614 * So in this code path, we have taken the last cookie, 3615 * virtually broken it in half due to the trim, and it happens 3616 * to use the copybuf which further complicates life. At the 3617 * same time, we have already setup the current cookie, which 3618 * is now wrong. More background info: the current cookie uses 3619 * the copybuf, so it is only a page long max. So we need to 3620 * fix the current cookies copy buffer address, physical 3621 * address, and kva for the 32-bit kernel. We due this by 3622 * bumping them by page size (of course, we can't due this on 3623 * the physical address since the copy buffer may not be 3624 * physically contiguous). 3625 */ 3626 cookie++; 3627 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3628 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3629 3630 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3631 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3632 #ifdef __xpv 3633 /* 3634 * If we're dom0, we're using a real device so we need to load 3635 * the cookies with MAs instead of PAs. 3636 */ 3637 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3638 #else 3639 cookie->dmac_laddress = paddr; 3640 #endif 3641 3642 #if !defined(__amd64) 3643 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3644 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3645 #endif 3646 } else { 3647 /* go back to the current cookie */ 3648 cookie++; 3649 } 3650 3651 /* 3652 * add the current cookie to the new window. set the new window size to 3653 * the what was left over from the previous cookie and what's in the 3654 * current cookie. 3655 */ 3656 (*windowp)->wd_cookie_cnt++; 3657 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3658 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3659 3660 /* 3661 * we know that the cookie passed in always uses the copy buffer. We 3662 * wouldn't be here if it didn't. 3663 */ 3664 *copybuf_used += MMU_PAGESIZE; 3665 3666 return (DDI_SUCCESS); 3667 } 3668 3669 3670 /* 3671 * rootnex_maxxfer_window_boundary() 3672 * Called in bind slowpath when we get to a window boundary because we will 3673 * go over maxxfer. 3674 */ 3675 static int 3676 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3677 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3678 { 3679 size_t dmac_size; 3680 off_t new_offset; 3681 size_t trim_sz; 3682 off_t coffset; 3683 3684 3685 /* 3686 * calculate how much we have to trim off of the current cookie to equal 3687 * maxxfer. We don't have to account for granularity here since our 3688 * maxxfer already takes that into account. 3689 */ 3690 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3691 ASSERT(trim_sz <= cookie->dmac_size); 3692 ASSERT(trim_sz <= dma->dp_maxxfer); 3693 3694 /* save cookie size since we need it later and we might change it */ 3695 dmac_size = cookie->dmac_size; 3696 3697 /* 3698 * if we're not trimming the entire cookie, setup the current window to 3699 * account for the trim. 3700 */ 3701 if (trim_sz < cookie->dmac_size) { 3702 (*windowp)->wd_cookie_cnt++; 3703 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3704 (*windowp)->wd_trim.tr_last_cookie = cookie; 3705 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3706 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3707 (*windowp)->wd_size = dma->dp_maxxfer; 3708 3709 /* 3710 * set the adjusted cookie size now in case this is the first 3711 * window. All other windows are taken care of in get win 3712 */ 3713 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3714 } 3715 3716 /* 3717 * coffset is the current offset within the cookie, new_offset is the 3718 * current offset with the entire buffer. 3719 */ 3720 coffset = dmac_size - trim_sz; 3721 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3722 3723 /* initialize the next window */ 3724 (*windowp)++; 3725 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3726 (*windowp)->wd_cookie_cnt++; 3727 (*windowp)->wd_size = trim_sz; 3728 if (trim_sz < dmac_size) { 3729 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3730 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3731 coffset; 3732 (*windowp)->wd_trim.tr_first_size = trim_sz; 3733 } 3734 3735 return (DDI_SUCCESS); 3736 } 3737 3738 3739 /* 3740 * rootnex_dma_sync() 3741 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 3742 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 3743 * is set, ddi_dma_sync() returns immediately passing back success. 3744 */ 3745 /*ARGSUSED*/ 3746 static int 3747 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3748 off_t off, size_t len, uint_t cache_flags) 3749 { 3750 rootnex_sglinfo_t *sinfo; 3751 rootnex_pgmap_t *cbpage; 3752 rootnex_window_t *win; 3753 ddi_dma_impl_t *hp; 3754 rootnex_dma_t *dma; 3755 caddr_t fromaddr; 3756 caddr_t toaddr; 3757 uint_t psize; 3758 off_t offset; 3759 uint_t pidx; 3760 size_t size; 3761 off_t poff; 3762 int e; 3763 3764 3765 hp = (ddi_dma_impl_t *)handle; 3766 dma = (rootnex_dma_t *)hp->dmai_private; 3767 sinfo = &dma->dp_sglinfo; 3768 3769 /* 3770 * if we don't have any windows, we don't need to sync. A copybuf 3771 * will cause us to have at least one window. 3772 */ 3773 if (dma->dp_window == NULL) { 3774 return (DDI_SUCCESS); 3775 } 3776 3777 /* This window may not need to be sync'd */ 3778 win = &dma->dp_window[dma->dp_current_win]; 3779 if (!win->wd_dosync) { 3780 return (DDI_SUCCESS); 3781 } 3782 3783 /* handle off and len special cases */ 3784 if ((off == 0) || (rootnex_sync_ignore_params)) { 3785 offset = win->wd_offset; 3786 } else { 3787 offset = off; 3788 } 3789 if ((len == 0) || (rootnex_sync_ignore_params)) { 3790 size = win->wd_size; 3791 } else { 3792 size = len; 3793 } 3794 3795 /* check the sync args to make sure they make a little sense */ 3796 if (rootnex_sync_check_parms) { 3797 e = rootnex_valid_sync_parms(hp, win, offset, size, 3798 cache_flags); 3799 if (e != DDI_SUCCESS) { 3800 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 3801 return (DDI_FAILURE); 3802 } 3803 } 3804 3805 /* 3806 * special case the first page to handle the offset into the page. The 3807 * offset to the current page for our buffer is the offset into the 3808 * first page of the buffer plus our current offset into the buffer 3809 * itself, masked of course. 3810 */ 3811 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 3812 psize = MIN((MMU_PAGESIZE - poff), size); 3813 3814 /* go through all the pages that we want to sync */ 3815 while (size > 0) { 3816 /* 3817 * Calculate the page index relative to the start of the buffer. 3818 * The index to the current page for our buffer is the offset 3819 * into the first page of the buffer plus our current offset 3820 * into the buffer itself, shifted of course... 3821 */ 3822 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 3823 ASSERT(pidx < sinfo->si_max_pages); 3824 3825 /* 3826 * if this page uses the copy buffer, we need to sync it, 3827 * otherwise, go on to the next page. 3828 */ 3829 cbpage = &dma->dp_pgmap[pidx]; 3830 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 3831 (cbpage->pm_uses_copybuf == B_FALSE)); 3832 if (cbpage->pm_uses_copybuf) { 3833 /* cbaddr and kaddr should be page aligned */ 3834 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 3835 MMU_PAGEOFFSET) == 0); 3836 ASSERT(((uintptr_t)cbpage->pm_kaddr & 3837 MMU_PAGEOFFSET) == 0); 3838 3839 /* 3840 * if we're copying for the device, we are going to 3841 * copy from the drivers buffer and to the rootnex 3842 * allocated copy buffer. 3843 */ 3844 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 3845 fromaddr = cbpage->pm_kaddr + poff; 3846 toaddr = cbpage->pm_cbaddr + poff; 3847 DTRACE_PROBE2(rootnex__sync__dev, 3848 dev_info_t *, dma->dp_dip, size_t, psize); 3849 3850 /* 3851 * if we're copying for the cpu/kernel, we are going to 3852 * copy from the rootnex allocated copy buffer to the 3853 * drivers buffer. 3854 */ 3855 } else { 3856 fromaddr = cbpage->pm_cbaddr + poff; 3857 toaddr = cbpage->pm_kaddr + poff; 3858 DTRACE_PROBE2(rootnex__sync__cpu, 3859 dev_info_t *, dma->dp_dip, size_t, psize); 3860 } 3861 3862 bcopy(fromaddr, toaddr, psize); 3863 } 3864 3865 /* 3866 * decrement size until we're done, update our offset into the 3867 * buffer, and get the next page size. 3868 */ 3869 size -= psize; 3870 offset += psize; 3871 psize = MIN(MMU_PAGESIZE, size); 3872 3873 /* page offset is zero for the rest of this loop */ 3874 poff = 0; 3875 } 3876 3877 return (DDI_SUCCESS); 3878 } 3879 3880 3881 /* 3882 * rootnex_valid_sync_parms() 3883 * checks the parameters passed to sync to verify they are correct. 3884 */ 3885 static int 3886 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 3887 off_t offset, size_t size, uint_t cache_flags) 3888 { 3889 off_t woffset; 3890 3891 3892 /* 3893 * the first part of the test to make sure the offset passed in is 3894 * within the window. 3895 */ 3896 if (offset < win->wd_offset) { 3897 return (DDI_FAILURE); 3898 } 3899 3900 /* 3901 * second and last part of the test to make sure the offset and length 3902 * passed in is within the window. 3903 */ 3904 woffset = offset - win->wd_offset; 3905 if ((woffset + size) > win->wd_size) { 3906 return (DDI_FAILURE); 3907 } 3908 3909 /* 3910 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 3911 * be set too. 3912 */ 3913 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 3914 (hp->dmai_rflags & DDI_DMA_WRITE)) { 3915 return (DDI_SUCCESS); 3916 } 3917 3918 /* 3919 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 3920 * should be set. Also DDI_DMA_READ should be set in the flags. 3921 */ 3922 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 3923 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 3924 (hp->dmai_rflags & DDI_DMA_READ)) { 3925 return (DDI_SUCCESS); 3926 } 3927 3928 return (DDI_FAILURE); 3929 } 3930 3931 3932 /* 3933 * rootnex_dma_win() 3934 * called from ddi_dma_getwin() 3935 */ 3936 /*ARGSUSED*/ 3937 static int 3938 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3939 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 3940 uint_t *ccountp) 3941 { 3942 rootnex_window_t *window; 3943 rootnex_trim_t *trim; 3944 ddi_dma_impl_t *hp; 3945 rootnex_dma_t *dma; 3946 #if !defined(__amd64) 3947 rootnex_sglinfo_t *sinfo; 3948 rootnex_pgmap_t *pmap; 3949 uint_t pidx; 3950 uint_t pcnt; 3951 off_t poff; 3952 int i; 3953 #endif 3954 3955 3956 hp = (ddi_dma_impl_t *)handle; 3957 dma = (rootnex_dma_t *)hp->dmai_private; 3958 #if !defined(__amd64) 3959 sinfo = &dma->dp_sglinfo; 3960 #endif 3961 3962 /* If we try and get a window which doesn't exist, return failure */ 3963 if (win >= hp->dmai_nwin) { 3964 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3965 return (DDI_FAILURE); 3966 } 3967 3968 /* 3969 * if we don't have any windows, and they're asking for the first 3970 * window, setup the cookie pointer to the first cookie in the bind. 3971 * setup our return values, then increment the cookie since we return 3972 * the first cookie on the stack. 3973 */ 3974 if (dma->dp_window == NULL) { 3975 if (win != 0) { 3976 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 3977 return (DDI_FAILURE); 3978 } 3979 hp->dmai_cookie = dma->dp_cookies; 3980 *offp = 0; 3981 *lenp = dma->dp_dma.dmao_size; 3982 *ccountp = dma->dp_sglinfo.si_sgl_size; 3983 *cookiep = hp->dmai_cookie[0]; 3984 hp->dmai_cookie++; 3985 return (DDI_SUCCESS); 3986 } 3987 3988 /* sync the old window before moving on to the new one */ 3989 window = &dma->dp_window[dma->dp_current_win]; 3990 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 3991 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 3992 DDI_DMA_SYNC_FORCPU); 3993 } 3994 3995 #if !defined(__amd64) 3996 /* 3997 * before we move to the next window, if we need to re-map, unmap all 3998 * the pages in this window. 3999 */ 4000 if (dma->dp_cb_remaping) { 4001 /* 4002 * If we switch to this window again, we'll need to map in 4003 * on the fly next time. 4004 */ 4005 window->wd_remap_copybuf = B_TRUE; 4006 4007 /* 4008 * calculate the page index into the buffer where this window 4009 * starts, and the number of pages this window takes up. 4010 */ 4011 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4012 MMU_PAGESHIFT; 4013 poff = (sinfo->si_buf_offset + window->wd_offset) & 4014 MMU_PAGEOFFSET; 4015 pcnt = mmu_btopr(window->wd_size + poff); 4016 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4017 4018 /* unmap pages which are currently mapped in this window */ 4019 for (i = 0; i < pcnt; i++) { 4020 if (dma->dp_pgmap[pidx].pm_mapped) { 4021 hat_unload(kas.a_hat, 4022 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4023 HAT_UNLOAD); 4024 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4025 } 4026 pidx++; 4027 } 4028 } 4029 #endif 4030 4031 /* 4032 * Move to the new window. 4033 * NOTE: current_win must be set for sync to work right 4034 */ 4035 dma->dp_current_win = win; 4036 window = &dma->dp_window[win]; 4037 4038 /* if needed, adjust the first and/or last cookies for trim */ 4039 trim = &window->wd_trim; 4040 if (trim->tr_trim_first) { 4041 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4042 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4043 #if !defined(__amd64) 4044 window->wd_first_cookie->dmac_type = 4045 (window->wd_first_cookie->dmac_type & 4046 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4047 #endif 4048 if (trim->tr_first_copybuf_win) { 4049 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4050 trim->tr_first_cbaddr; 4051 #if !defined(__amd64) 4052 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4053 trim->tr_first_kaddr; 4054 #endif 4055 } 4056 } 4057 if (trim->tr_trim_last) { 4058 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4059 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4060 if (trim->tr_last_copybuf_win) { 4061 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4062 trim->tr_last_cbaddr; 4063 #if !defined(__amd64) 4064 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4065 trim->tr_last_kaddr; 4066 #endif 4067 } 4068 } 4069 4070 /* 4071 * setup the cookie pointer to the first cookie in the window. setup 4072 * our return values, then increment the cookie since we return the 4073 * first cookie on the stack. 4074 */ 4075 hp->dmai_cookie = window->wd_first_cookie; 4076 *offp = window->wd_offset; 4077 *lenp = window->wd_size; 4078 *ccountp = window->wd_cookie_cnt; 4079 *cookiep = hp->dmai_cookie[0]; 4080 hp->dmai_cookie++; 4081 4082 #if !defined(__amd64) 4083 /* re-map copybuf if required for this window */ 4084 if (dma->dp_cb_remaping) { 4085 /* 4086 * calculate the page index into the buffer where this 4087 * window starts. 4088 */ 4089 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4090 MMU_PAGESHIFT; 4091 ASSERT(pidx < sinfo->si_max_pages); 4092 4093 /* 4094 * the first page can get unmapped if it's shared with the 4095 * previous window. Even if the rest of this window is already 4096 * mapped in, we need to still check this one. 4097 */ 4098 pmap = &dma->dp_pgmap[pidx]; 4099 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4100 if (pmap->pm_pp != NULL) { 4101 pmap->pm_mapped = B_TRUE; 4102 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4103 } else if (pmap->pm_vaddr != NULL) { 4104 pmap->pm_mapped = B_TRUE; 4105 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4106 pmap->pm_kaddr); 4107 } 4108 } 4109 pidx++; 4110 4111 /* map in the rest of the pages if required */ 4112 if (window->wd_remap_copybuf) { 4113 window->wd_remap_copybuf = B_FALSE; 4114 4115 /* figure out many pages this window takes up */ 4116 poff = (sinfo->si_buf_offset + window->wd_offset) & 4117 MMU_PAGEOFFSET; 4118 pcnt = mmu_btopr(window->wd_size + poff); 4119 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4120 4121 /* map pages which require it */ 4122 for (i = 1; i < pcnt; i++) { 4123 pmap = &dma->dp_pgmap[pidx]; 4124 if (pmap->pm_uses_copybuf) { 4125 ASSERT(pmap->pm_mapped == B_FALSE); 4126 if (pmap->pm_pp != NULL) { 4127 pmap->pm_mapped = B_TRUE; 4128 i86_pp_map(pmap->pm_pp, 4129 pmap->pm_kaddr); 4130 } else if (pmap->pm_vaddr != NULL) { 4131 pmap->pm_mapped = B_TRUE; 4132 i86_va_map(pmap->pm_vaddr, 4133 sinfo->si_asp, 4134 pmap->pm_kaddr); 4135 } 4136 } 4137 pidx++; 4138 } 4139 } 4140 } 4141 #endif 4142 4143 /* if the new window uses the copy buffer, sync it for the device */ 4144 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4145 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4146 DDI_DMA_SYNC_FORDEV); 4147 } 4148 4149 return (DDI_SUCCESS); 4150 } 4151 4152 4153 4154 /* 4155 * ************************ 4156 * obsoleted dma routines 4157 * ************************ 4158 */ 4159 4160 /* 4161 * rootnex_dma_map() 4162 * called from ddi_dma_setup() 4163 */ 4164 /* ARGSUSED */ 4165 static int 4166 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, struct ddi_dma_req *dmareq, 4167 ddi_dma_handle_t *handlep) 4168 { 4169 #if defined(__amd64) 4170 /* 4171 * this interface is not supported in 64-bit x86 kernel. See comment in 4172 * rootnex_dma_mctl() 4173 */ 4174 return (DDI_DMA_NORESOURCES); 4175 4176 #else /* 32-bit x86 kernel */ 4177 ddi_dma_handle_t *lhandlep; 4178 ddi_dma_handle_t lhandle; 4179 ddi_dma_cookie_t cookie; 4180 ddi_dma_attr_t dma_attr; 4181 ddi_dma_lim_t *dma_lim; 4182 uint_t ccnt; 4183 int e; 4184 4185 4186 /* 4187 * if the driver is just testing to see if it's possible to do the bind, 4188 * we'll use local state. Otherwise, use the handle pointer passed in. 4189 */ 4190 if (handlep == NULL) { 4191 lhandlep = &lhandle; 4192 } else { 4193 lhandlep = handlep; 4194 } 4195 4196 /* convert the limit structure to a dma_attr one */ 4197 dma_lim = dmareq->dmar_limits; 4198 dma_attr.dma_attr_version = DMA_ATTR_V0; 4199 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4200 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4201 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4202 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4203 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4204 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4205 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4206 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4207 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4208 dma_attr.dma_attr_align = MMU_PAGESIZE; 4209 dma_attr.dma_attr_flags = 0; 4210 4211 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4212 dmareq->dmar_arg, lhandlep); 4213 if (e != DDI_SUCCESS) { 4214 return (e); 4215 } 4216 4217 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4218 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4219 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4220 return (e); 4221 } 4222 4223 /* 4224 * if the driver is just testing to see if it's possible to do the bind, 4225 * free up the local state and return the result. 4226 */ 4227 if (handlep == NULL) { 4228 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4229 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4230 if (e == DDI_DMA_MAPPED) { 4231 return (DDI_DMA_MAPOK); 4232 } else { 4233 return (DDI_DMA_NOMAPPING); 4234 } 4235 } 4236 4237 return (e); 4238 #endif /* defined(__amd64) */ 4239 } 4240 4241 4242 /* 4243 * rootnex_dma_mctl() 4244 * 4245 */ 4246 /* ARGSUSED */ 4247 static int 4248 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4249 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4250 uint_t cache_flags) 4251 { 4252 #if defined(__amd64) 4253 /* 4254 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4255 * common implementation in genunix, so they no longer have x86 4256 * specific functionality which called into dma_ctl. 4257 * 4258 * The rest of the obsoleted interfaces were never supported in the 4259 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4260 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4261 * implementation issues. 4262 * 4263 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4264 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4265 * reflect that now too... 4266 * 4267 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4268 * not going to put this functionality into the 64-bit x86 kernel now. 4269 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4270 * that in a future release. 4271 */ 4272 return (DDI_FAILURE); 4273 4274 #else /* 32-bit x86 kernel */ 4275 ddi_dma_cookie_t lcookie; 4276 ddi_dma_cookie_t *cookie; 4277 rootnex_window_t *window; 4278 ddi_dma_impl_t *hp; 4279 rootnex_dma_t *dma; 4280 uint_t nwin; 4281 uint_t ccnt; 4282 size_t len; 4283 off_t off; 4284 int e; 4285 4286 4287 /* 4288 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4289 * hacky since were optimizing for the current interfaces and so we can 4290 * cleanup the mess in genunix. Hopefully we will remove the this 4291 * obsoleted routines someday soon. 4292 */ 4293 4294 switch (request) { 4295 4296 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4297 hp = (ddi_dma_impl_t *)handle; 4298 cookie = (ddi_dma_cookie_t *)objpp; 4299 4300 /* 4301 * convert segment to cookie. We don't distinguish between the 4302 * two :-) 4303 */ 4304 *cookie = *hp->dmai_cookie; 4305 *lenp = cookie->dmac_size; 4306 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4307 return (DDI_SUCCESS); 4308 4309 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4310 hp = (ddi_dma_impl_t *)handle; 4311 dma = (rootnex_dma_t *)hp->dmai_private; 4312 4313 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4314 return (DDI_DMA_STALE); 4315 } 4316 4317 /* handle the case where we don't have any windows */ 4318 if (dma->dp_window == NULL) { 4319 /* 4320 * if seg == NULL, and we don't have any windows, 4321 * return the first cookie in the sgl. 4322 */ 4323 if (*lenp == NULL) { 4324 dma->dp_current_cookie = 0; 4325 hp->dmai_cookie = dma->dp_cookies; 4326 *objpp = (caddr_t)handle; 4327 return (DDI_SUCCESS); 4328 4329 /* if we have more cookies, go to the next cookie */ 4330 } else { 4331 if ((dma->dp_current_cookie + 1) >= 4332 dma->dp_sglinfo.si_sgl_size) { 4333 return (DDI_DMA_DONE); 4334 } 4335 dma->dp_current_cookie++; 4336 hp->dmai_cookie++; 4337 return (DDI_SUCCESS); 4338 } 4339 } 4340 4341 /* We have one or more windows */ 4342 window = &dma->dp_window[dma->dp_current_win]; 4343 4344 /* 4345 * if seg == NULL, return the first cookie in the current 4346 * window 4347 */ 4348 if (*lenp == NULL) { 4349 dma->dp_current_cookie = 0; 4350 hp->dmai_cookie = window->wd_first_cookie; 4351 4352 /* 4353 * go to the next cookie in the window then see if we done with 4354 * this window. 4355 */ 4356 } else { 4357 if ((dma->dp_current_cookie + 1) >= 4358 window->wd_cookie_cnt) { 4359 return (DDI_DMA_DONE); 4360 } 4361 dma->dp_current_cookie++; 4362 hp->dmai_cookie++; 4363 } 4364 *objpp = (caddr_t)handle; 4365 return (DDI_SUCCESS); 4366 4367 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4368 hp = (ddi_dma_impl_t *)handle; 4369 dma = (rootnex_dma_t *)hp->dmai_private; 4370 4371 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4372 return (DDI_DMA_STALE); 4373 } 4374 4375 /* if win == NULL, return the first window in the bind */ 4376 if (*offp == NULL) { 4377 nwin = 0; 4378 4379 /* 4380 * else, go to the next window then see if we're done with all 4381 * the windows. 4382 */ 4383 } else { 4384 nwin = dma->dp_current_win + 1; 4385 if (nwin >= hp->dmai_nwin) { 4386 return (DDI_DMA_DONE); 4387 } 4388 } 4389 4390 /* switch to the next window */ 4391 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4392 &lcookie, &ccnt); 4393 ASSERT(e == DDI_SUCCESS); 4394 if (e != DDI_SUCCESS) { 4395 return (DDI_DMA_STALE); 4396 } 4397 4398 /* reset the cookie back to the first cookie in the window */ 4399 if (dma->dp_window != NULL) { 4400 window = &dma->dp_window[dma->dp_current_win]; 4401 hp->dmai_cookie = window->wd_first_cookie; 4402 } else { 4403 hp->dmai_cookie = dma->dp_cookies; 4404 } 4405 4406 *objpp = (caddr_t)handle; 4407 return (DDI_SUCCESS); 4408 4409 case DDI_DMA_FREE: /* ddi_dma_free() */ 4410 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4411 (void) rootnex_dma_freehdl(dip, rdip, handle); 4412 if (rootnex_state->r_dvma_call_list_id) { 4413 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4414 } 4415 return (DDI_SUCCESS); 4416 4417 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4418 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4419 /* should never get here, handled in genunix */ 4420 ASSERT(0); 4421 return (DDI_FAILURE); 4422 4423 case DDI_DMA_KVADDR: 4424 case DDI_DMA_GETERR: 4425 case DDI_DMA_COFF: 4426 return (DDI_FAILURE); 4427 } 4428 4429 return (DDI_FAILURE); 4430 #endif /* defined(__amd64) */ 4431 } 4432 4433 4434 /* 4435 * ********* 4436 * FMA Code 4437 * ********* 4438 */ 4439 4440 /* 4441 * rootnex_fm_init() 4442 * FMA init busop 4443 */ 4444 /* ARGSUSED */ 4445 static int 4446 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4447 ddi_iblock_cookie_t *ibc) 4448 { 4449 *ibc = rootnex_state->r_err_ibc; 4450 4451 return (ddi_system_fmcap); 4452 } 4453 4454 /* 4455 * rootnex_dma_check() 4456 * Function called after a dma fault occurred to find out whether the 4457 * fault address is associated with a driver that is able to handle faults 4458 * and recover from faults. 4459 */ 4460 /* ARGSUSED */ 4461 static int 4462 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4463 const void *not_used) 4464 { 4465 rootnex_window_t *window; 4466 uint64_t start_addr; 4467 uint64_t fault_addr; 4468 ddi_dma_impl_t *hp; 4469 rootnex_dma_t *dma; 4470 uint64_t end_addr; 4471 size_t csize; 4472 int i; 4473 int j; 4474 4475 4476 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4477 hp = (ddi_dma_impl_t *)handle; 4478 ASSERT(hp); 4479 4480 dma = (rootnex_dma_t *)hp->dmai_private; 4481 4482 /* Get the address that we need to search for */ 4483 fault_addr = *(uint64_t *)addr; 4484 4485 /* 4486 * if we don't have any windows, we can just walk through all the 4487 * cookies. 4488 */ 4489 if (dma->dp_window == NULL) { 4490 /* for each cookie */ 4491 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4492 /* 4493 * if the faulted address is within the physical address 4494 * range of the cookie, return DDI_FM_NONFATAL. 4495 */ 4496 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4497 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4498 dma->dp_cookies[i].dmac_size))) { 4499 return (DDI_FM_NONFATAL); 4500 } 4501 } 4502 4503 /* fault_addr not within this DMA handle */ 4504 return (DDI_FM_UNKNOWN); 4505 } 4506 4507 /* we have mutiple windows, walk through each window */ 4508 for (i = 0; i < hp->dmai_nwin; i++) { 4509 window = &dma->dp_window[i]; 4510 4511 /* Go through all the cookies in the window */ 4512 for (j = 0; j < window->wd_cookie_cnt; j++) { 4513 4514 start_addr = window->wd_first_cookie[j].dmac_laddress; 4515 csize = window->wd_first_cookie[j].dmac_size; 4516 4517 /* 4518 * if we are trimming the first cookie in the window, 4519 * and this is the first cookie, adjust the start 4520 * address and size of the cookie to account for the 4521 * trim. 4522 */ 4523 if (window->wd_trim.tr_trim_first && (j == 0)) { 4524 start_addr = window->wd_trim.tr_first_paddr; 4525 csize = window->wd_trim.tr_first_size; 4526 } 4527 4528 /* 4529 * if we are trimming the last cookie in the window, 4530 * and this is the last cookie, adjust the start 4531 * address and size of the cookie to account for the 4532 * trim. 4533 */ 4534 if (window->wd_trim.tr_trim_last && 4535 (j == (window->wd_cookie_cnt - 1))) { 4536 start_addr = window->wd_trim.tr_last_paddr; 4537 csize = window->wd_trim.tr_last_size; 4538 } 4539 4540 end_addr = start_addr + csize; 4541 4542 /* 4543 * if the faulted address is within the physical address 4544 * range of the cookie, return DDI_FM_NONFATAL. 4545 */ 4546 if ((fault_addr >= start_addr) && 4547 (fault_addr <= end_addr)) { 4548 return (DDI_FM_NONFATAL); 4549 } 4550 } 4551 } 4552 4553 /* fault_addr not within this DMA handle */ 4554 return (DDI_FM_UNKNOWN); 4555 } 4556