1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #else 71 #include <sys/intel_iommu.h> 72 #endif 73 74 75 /* 76 * enable/disable extra checking of function parameters. Useful for debugging 77 * drivers. 78 */ 79 #ifdef DEBUG 80 int rootnex_alloc_check_parms = 1; 81 int rootnex_bind_check_parms = 1; 82 int rootnex_bind_check_inuse = 1; 83 int rootnex_unbind_verify_buffer = 0; 84 int rootnex_sync_check_parms = 1; 85 #else 86 int rootnex_alloc_check_parms = 0; 87 int rootnex_bind_check_parms = 0; 88 int rootnex_bind_check_inuse = 0; 89 int rootnex_unbind_verify_buffer = 0; 90 int rootnex_sync_check_parms = 0; 91 #endif 92 93 /* Master Abort and Target Abort panic flag */ 94 int rootnex_fm_ma_ta_panic_flag = 0; 95 96 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 97 int rootnex_bind_fail = 1; 98 int rootnex_bind_warn = 1; 99 uint8_t *rootnex_warn_list; 100 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 101 #define ROOTNEX_BIND_WARNING (0x1 << 0) 102 103 /* 104 * revert back to old broken behavior of always sync'ing entire copy buffer. 105 * This is useful if be have a buggy driver which doesn't correctly pass in 106 * the offset and size into ddi_dma_sync(). 107 */ 108 int rootnex_sync_ignore_params = 0; 109 110 /* 111 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 112 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 113 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 114 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 115 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 116 * (< 8K). We will still need to allocate the copy buffer during bind though 117 * (if we need one). These can only be modified in /etc/system before rootnex 118 * attach. 119 */ 120 #if defined(__amd64) 121 int rootnex_prealloc_cookies = 65; 122 int rootnex_prealloc_windows = 4; 123 int rootnex_prealloc_copybuf = 2; 124 #else 125 int rootnex_prealloc_cookies = 33; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #endif 129 130 /* driver global state */ 131 static rootnex_state_t *rootnex_state; 132 133 /* shortcut to rootnex counters */ 134 static uint64_t *rootnex_cnt; 135 136 /* 137 * XXX - does x86 even need these or are they left over from the SPARC days? 138 */ 139 /* statically defined integer/boolean properties for the root node */ 140 static rootnex_intprop_t rootnex_intprp[] = { 141 { "PAGESIZE", PAGESIZE }, 142 { "MMU_PAGESIZE", MMU_PAGESIZE }, 143 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 144 { DDI_RELATIVE_ADDRESSING, 1 }, 145 }; 146 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 147 148 #ifdef __xpv 149 typedef maddr_t rootnex_addr_t; 150 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 151 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 152 #else 153 typedef paddr_t rootnex_addr_t; 154 #endif 155 156 #if !defined(__xpv) 157 char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica"; 158 #endif 159 160 static struct cb_ops rootnex_cb_ops = { 161 nodev, /* open */ 162 nodev, /* close */ 163 nodev, /* strategy */ 164 nodev, /* print */ 165 nodev, /* dump */ 166 nodev, /* read */ 167 nodev, /* write */ 168 nodev, /* ioctl */ 169 nodev, /* devmap */ 170 nodev, /* mmap */ 171 nodev, /* segmap */ 172 nochpoll, /* chpoll */ 173 ddi_prop_op, /* cb_prop_op */ 174 NULL, /* struct streamtab */ 175 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 176 CB_REV, /* Rev */ 177 nodev, /* cb_aread */ 178 nodev /* cb_awrite */ 179 }; 180 181 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 182 off_t offset, off_t len, caddr_t *vaddrp); 183 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 184 struct hat *hat, struct seg *seg, caddr_t addr, 185 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 186 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 187 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 188 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 189 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 190 ddi_dma_handle_t *handlep); 191 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 192 ddi_dma_handle_t handle); 193 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 194 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 195 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 196 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 197 ddi_dma_handle_t handle); 198 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 200 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 202 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 203 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 205 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 206 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 207 ddi_ctl_enum_t ctlop, void *arg, void *result); 208 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 209 ddi_iblock_cookie_t *ibc); 210 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 211 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 212 213 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 214 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 215 ddi_dma_handle_t *handlep); 216 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 217 ddi_dma_handle_t handle); 218 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 219 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 220 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 221 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle); 223 #if !defined(__xpv) 224 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 225 ddi_dma_handle_t handle); 226 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 227 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 228 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 229 ddi_dma_cookie_t *cookiep, uint_t ccount); 230 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 231 ddi_dma_handle_t handle); 232 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 233 #endif 234 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 235 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 236 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 237 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 238 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 239 240 static struct bus_ops rootnex_bus_ops = { 241 BUSO_REV, 242 rootnex_map, 243 NULL, 244 NULL, 245 NULL, 246 rootnex_map_fault, 247 rootnex_dma_map, 248 rootnex_dma_allochdl, 249 rootnex_dma_freehdl, 250 rootnex_dma_bindhdl, 251 rootnex_dma_unbindhdl, 252 rootnex_dma_sync, 253 rootnex_dma_win, 254 rootnex_dma_mctl, 255 rootnex_ctlops, 256 ddi_bus_prop_op, 257 i_ddi_rootnex_get_eventcookie, 258 i_ddi_rootnex_add_eventcall, 259 i_ddi_rootnex_remove_eventcall, 260 i_ddi_rootnex_post_event, 261 0, /* bus_intr_ctl */ 262 0, /* bus_config */ 263 0, /* bus_unconfig */ 264 rootnex_fm_init, /* bus_fm_init */ 265 NULL, /* bus_fm_fini */ 266 NULL, /* bus_fm_access_enter */ 267 NULL, /* bus_fm_access_exit */ 268 NULL, /* bus_powr */ 269 rootnex_intr_ops /* bus_intr_op */ 270 }; 271 272 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 273 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 274 275 static struct dev_ops rootnex_ops = { 276 DEVO_REV, 277 0, 278 ddi_no_info, 279 nulldev, 280 nulldev, 281 rootnex_attach, 282 rootnex_detach, 283 nulldev, 284 &rootnex_cb_ops, 285 &rootnex_bus_ops, 286 NULL, 287 ddi_quiesce_not_needed, /* quiesce */ 288 }; 289 290 static struct modldrv rootnex_modldrv = { 291 &mod_driverops, 292 "i86pc root nexus", 293 &rootnex_ops 294 }; 295 296 static struct modlinkage rootnex_modlinkage = { 297 MODREV_1, 298 (void *)&rootnex_modldrv, 299 NULL 300 }; 301 302 #if !defined(__xpv) 303 static iommulib_nexops_t iommulib_nexops = { 304 IOMMU_NEXOPS_VERSION, 305 "Rootnex IOMMU ops Vers 1.1", 306 NULL, 307 rootnex_coredma_allochdl, 308 rootnex_coredma_freehdl, 309 rootnex_coredma_bindhdl, 310 rootnex_coredma_unbindhdl, 311 rootnex_coredma_reset_cookies, 312 rootnex_coredma_get_cookies, 313 rootnex_coredma_set_cookies, 314 rootnex_coredma_clear_cookies, 315 rootnex_coredma_get_sleep_flags, 316 rootnex_coredma_sync, 317 rootnex_coredma_win, 318 rootnex_dma_map, 319 rootnex_dma_mctl 320 }; 321 #endif 322 323 /* 324 * extern hacks 325 */ 326 extern struct seg_ops segdev_ops; 327 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 328 #ifdef DDI_MAP_DEBUG 329 extern int ddi_map_debug_flag; 330 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 331 #endif 332 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 333 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 334 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 335 psm_intr_op_t, int *); 336 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 337 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 338 339 /* 340 * Use device arena to use for device control register mappings. 341 * Various kernel memory walkers (debugger, dtrace) need to know 342 * to avoid this address range to prevent undesired device activity. 343 */ 344 extern void *device_arena_alloc(size_t size, int vm_flag); 345 extern void device_arena_free(void * vaddr, size_t size); 346 347 348 /* 349 * Internal functions 350 */ 351 static int rootnex_dma_init(); 352 static void rootnex_add_props(dev_info_t *); 353 static int rootnex_ctl_reportdev(dev_info_t *dip); 354 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 355 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 356 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 357 static int rootnex_map_handle(ddi_map_req_t *mp); 358 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 359 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 360 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 361 ddi_dma_attr_t *attr); 362 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 363 rootnex_sglinfo_t *sglinfo); 364 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 365 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 366 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 367 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 368 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 369 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 370 ddi_dma_attr_t *attr, int kmflag); 371 static void rootnex_teardown_windows(rootnex_dma_t *dma); 372 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 373 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 374 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 375 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 376 size_t *copybuf_used, page_t **cur_pp); 377 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 378 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 379 ddi_dma_attr_t *attr, off_t cur_offset); 380 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 381 rootnex_dma_t *dma, rootnex_window_t **windowp, 382 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 383 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 384 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 385 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 386 off_t offset, size_t size, uint_t cache_flags); 387 static int rootnex_verify_buffer(rootnex_dma_t *dma); 388 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 389 const void *comp_addr, const void *not_used); 390 391 /* 392 * _init() 393 * 394 */ 395 int 396 _init(void) 397 { 398 399 rootnex_state = NULL; 400 return (mod_install(&rootnex_modlinkage)); 401 } 402 403 404 /* 405 * _info() 406 * 407 */ 408 int 409 _info(struct modinfo *modinfop) 410 { 411 return (mod_info(&rootnex_modlinkage, modinfop)); 412 } 413 414 415 /* 416 * _fini() 417 * 418 */ 419 int 420 _fini(void) 421 { 422 return (EBUSY); 423 } 424 425 426 /* 427 * rootnex_attach() 428 * 429 */ 430 static int 431 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 432 { 433 int fmcap; 434 int e; 435 436 switch (cmd) { 437 case DDI_ATTACH: 438 break; 439 case DDI_RESUME: 440 return (DDI_SUCCESS); 441 default: 442 return (DDI_FAILURE); 443 } 444 445 /* 446 * We should only have one instance of rootnex. Save it away since we 447 * don't have an easy way to get it back later. 448 */ 449 ASSERT(rootnex_state == NULL); 450 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 451 452 rootnex_state->r_dip = dip; 453 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 454 rootnex_state->r_reserved_msg_printed = B_FALSE; 455 rootnex_cnt = &rootnex_state->r_counters[0]; 456 rootnex_state->r_intel_iommu_enabled = B_FALSE; 457 458 /* 459 * Set minimum fm capability level for i86pc platforms and then 460 * initialize error handling. Since we're the rootnex, we don't 461 * care what's returned in the fmcap field. 462 */ 463 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 464 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 465 fmcap = ddi_system_fmcap; 466 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 467 468 /* initialize DMA related state */ 469 e = rootnex_dma_init(); 470 if (e != DDI_SUCCESS) { 471 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 472 return (DDI_FAILURE); 473 } 474 475 /* Add static root node properties */ 476 rootnex_add_props(dip); 477 478 /* since we can't call ddi_report_dev() */ 479 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 480 481 /* Initialize rootnex event handle */ 482 i_ddi_rootnex_init_events(dip); 483 484 #if !defined(__xpv) 485 #if defined(__amd64) 486 /* probe intel iommu */ 487 intel_iommu_probe_and_parse(); 488 489 /* attach the iommu nodes */ 490 if (intel_iommu_support) { 491 if (intel_iommu_attach_dmar_nodes() == DDI_SUCCESS) { 492 rootnex_state->r_intel_iommu_enabled = B_TRUE; 493 } else { 494 intel_iommu_release_dmar_info(); 495 } 496 } 497 #endif 498 499 e = iommulib_nexus_register(dip, &iommulib_nexops, 500 &rootnex_state->r_iommulib_handle); 501 502 ASSERT(e == DDI_SUCCESS); 503 #endif 504 505 return (DDI_SUCCESS); 506 } 507 508 509 /* 510 * rootnex_detach() 511 * 512 */ 513 /*ARGSUSED*/ 514 static int 515 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 516 { 517 switch (cmd) { 518 case DDI_SUSPEND: 519 break; 520 default: 521 return (DDI_FAILURE); 522 } 523 524 return (DDI_SUCCESS); 525 } 526 527 528 /* 529 * rootnex_dma_init() 530 * 531 */ 532 /*ARGSUSED*/ 533 static int 534 rootnex_dma_init() 535 { 536 size_t bufsize; 537 538 539 /* 540 * size of our cookie/window/copybuf state needed in dma bind that we 541 * pre-alloc in dma_alloc_handle 542 */ 543 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 544 rootnex_state->r_prealloc_size = 545 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 546 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 547 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 548 549 /* 550 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 551 * allocate 16 extra bytes for struct pointer alignment 552 * (p->dmai_private & dma->dp_prealloc_buffer) 553 */ 554 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 555 rootnex_state->r_prealloc_size + 0x10; 556 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 557 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 558 if (rootnex_state->r_dmahdl_cache == NULL) { 559 return (DDI_FAILURE); 560 } 561 562 /* 563 * allocate array to track which major numbers we have printed warnings 564 * for. 565 */ 566 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 567 KM_SLEEP); 568 569 return (DDI_SUCCESS); 570 } 571 572 573 /* 574 * rootnex_add_props() 575 * 576 */ 577 static void 578 rootnex_add_props(dev_info_t *dip) 579 { 580 rootnex_intprop_t *rpp; 581 int i; 582 583 /* Add static integer/boolean properties to the root node */ 584 rpp = rootnex_intprp; 585 for (i = 0; i < NROOT_INTPROPS; i++) { 586 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 587 rpp[i].prop_name, rpp[i].prop_value); 588 } 589 } 590 591 592 593 /* 594 * ************************* 595 * ctlops related routines 596 * ************************* 597 */ 598 599 /* 600 * rootnex_ctlops() 601 * 602 */ 603 /*ARGSUSED*/ 604 static int 605 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 606 void *arg, void *result) 607 { 608 int n, *ptr; 609 struct ddi_parent_private_data *pdp; 610 611 switch (ctlop) { 612 case DDI_CTLOPS_DMAPMAPC: 613 /* 614 * Return 'partial' to indicate that dma mapping 615 * has to be done in the main MMU. 616 */ 617 return (DDI_DMA_PARTIAL); 618 619 case DDI_CTLOPS_BTOP: 620 /* 621 * Convert byte count input to physical page units. 622 * (byte counts that are not a page-size multiple 623 * are rounded down) 624 */ 625 *(ulong_t *)result = btop(*(ulong_t *)arg); 626 return (DDI_SUCCESS); 627 628 case DDI_CTLOPS_PTOB: 629 /* 630 * Convert size in physical pages to bytes 631 */ 632 *(ulong_t *)result = ptob(*(ulong_t *)arg); 633 return (DDI_SUCCESS); 634 635 case DDI_CTLOPS_BTOPR: 636 /* 637 * Convert byte count input to physical page units 638 * (byte counts that are not a page-size multiple 639 * are rounded up) 640 */ 641 *(ulong_t *)result = btopr(*(ulong_t *)arg); 642 return (DDI_SUCCESS); 643 644 case DDI_CTLOPS_INITCHILD: 645 return (impl_ddi_sunbus_initchild(arg)); 646 647 case DDI_CTLOPS_UNINITCHILD: 648 impl_ddi_sunbus_removechild(arg); 649 return (DDI_SUCCESS); 650 651 case DDI_CTLOPS_REPORTDEV: 652 return (rootnex_ctl_reportdev(rdip)); 653 654 case DDI_CTLOPS_IOMIN: 655 /* 656 * Nothing to do here but reflect back.. 657 */ 658 return (DDI_SUCCESS); 659 660 case DDI_CTLOPS_REGSIZE: 661 case DDI_CTLOPS_NREGS: 662 break; 663 664 case DDI_CTLOPS_SIDDEV: 665 if (ndi_dev_is_prom_node(rdip)) 666 return (DDI_SUCCESS); 667 if (ndi_dev_is_persistent_node(rdip)) 668 return (DDI_SUCCESS); 669 return (DDI_FAILURE); 670 671 case DDI_CTLOPS_POWER: 672 return ((*pm_platform_power)((power_req_t *)arg)); 673 674 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 675 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 676 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 677 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 678 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 679 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 680 if (!rootnex_state->r_reserved_msg_printed) { 681 rootnex_state->r_reserved_msg_printed = B_TRUE; 682 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 683 "1 or more reserved/obsolete operations."); 684 } 685 return (DDI_FAILURE); 686 687 default: 688 return (DDI_FAILURE); 689 } 690 /* 691 * The rest are for "hardware" properties 692 */ 693 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 694 return (DDI_FAILURE); 695 696 if (ctlop == DDI_CTLOPS_NREGS) { 697 ptr = (int *)result; 698 *ptr = pdp->par_nreg; 699 } else { 700 off_t *size = (off_t *)result; 701 702 ptr = (int *)arg; 703 n = *ptr; 704 if (n >= pdp->par_nreg) { 705 return (DDI_FAILURE); 706 } 707 *size = (off_t)pdp->par_reg[n].regspec_size; 708 } 709 return (DDI_SUCCESS); 710 } 711 712 713 /* 714 * rootnex_ctl_reportdev() 715 * 716 */ 717 static int 718 rootnex_ctl_reportdev(dev_info_t *dev) 719 { 720 int i, n, len, f_len = 0; 721 char *buf; 722 723 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 724 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 725 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 726 len = strlen(buf); 727 728 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 729 730 struct regspec *rp = sparc_pd_getreg(dev, i); 731 732 if (i == 0) 733 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 734 ": "); 735 else 736 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 737 " and "); 738 len = strlen(buf); 739 740 switch (rp->regspec_bustype) { 741 742 case BTEISA: 743 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 744 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 745 break; 746 747 case BTISA: 748 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 749 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 750 break; 751 752 default: 753 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 754 "space %x offset %x", 755 rp->regspec_bustype, rp->regspec_addr); 756 break; 757 } 758 len = strlen(buf); 759 } 760 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 761 int pri; 762 763 if (i != 0) { 764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 765 ","); 766 len = strlen(buf); 767 } 768 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 769 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 770 " sparc ipl %d", pri); 771 len = strlen(buf); 772 } 773 #ifdef DEBUG 774 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 775 cmn_err(CE_NOTE, "next message is truncated: " 776 "printed length 1024, real length %d", f_len); 777 } 778 #endif /* DEBUG */ 779 cmn_err(CE_CONT, "?%s\n", buf); 780 kmem_free(buf, REPORTDEV_BUFSIZE); 781 return (DDI_SUCCESS); 782 } 783 784 785 /* 786 * ****************** 787 * map related code 788 * ****************** 789 */ 790 791 /* 792 * rootnex_map() 793 * 794 */ 795 static int 796 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 797 off_t len, caddr_t *vaddrp) 798 { 799 struct regspec *rp, tmp_reg; 800 ddi_map_req_t mr = *mp; /* Get private copy of request */ 801 int error; 802 803 mp = &mr; 804 805 switch (mp->map_op) { 806 case DDI_MO_MAP_LOCKED: 807 case DDI_MO_UNMAP: 808 case DDI_MO_MAP_HANDLE: 809 break; 810 default: 811 #ifdef DDI_MAP_DEBUG 812 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 813 mp->map_op); 814 #endif /* DDI_MAP_DEBUG */ 815 return (DDI_ME_UNIMPLEMENTED); 816 } 817 818 if (mp->map_flags & DDI_MF_USER_MAPPING) { 819 #ifdef DDI_MAP_DEBUG 820 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 821 #endif /* DDI_MAP_DEBUG */ 822 return (DDI_ME_UNIMPLEMENTED); 823 } 824 825 /* 826 * First, if given an rnumber, convert it to a regspec... 827 * (Presumably, this is on behalf of a child of the root node?) 828 */ 829 830 if (mp->map_type == DDI_MT_RNUMBER) { 831 832 int rnumber = mp->map_obj.rnumber; 833 #ifdef DDI_MAP_DEBUG 834 static char *out_of_range = 835 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 836 #endif /* DDI_MAP_DEBUG */ 837 838 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 839 if (rp == NULL) { 840 #ifdef DDI_MAP_DEBUG 841 cmn_err(CE_WARN, out_of_range, rnumber, 842 ddi_get_name(rdip)); 843 #endif /* DDI_MAP_DEBUG */ 844 return (DDI_ME_RNUMBER_RANGE); 845 } 846 847 /* 848 * Convert the given ddi_map_req_t from rnumber to regspec... 849 */ 850 851 mp->map_type = DDI_MT_REGSPEC; 852 mp->map_obj.rp = rp; 853 } 854 855 /* 856 * Adjust offset and length correspnding to called values... 857 * XXX: A non-zero length means override the one in the regspec 858 * XXX: (regardless of what's in the parent's range?) 859 */ 860 861 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 862 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 863 864 #ifdef DDI_MAP_DEBUG 865 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 866 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 867 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 868 len, mp->map_handlep); 869 #endif /* DDI_MAP_DEBUG */ 870 871 /* 872 * I/O or memory mapping: 873 * 874 * <bustype=0, addr=x, len=x>: memory 875 * <bustype=1, addr=x, len=x>: i/o 876 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 877 */ 878 879 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 880 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 881 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 882 ddi_get_name(rdip), rp->regspec_bustype, 883 rp->regspec_addr, rp->regspec_size); 884 return (DDI_ME_INVAL); 885 } 886 887 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 888 /* 889 * compatibility i/o mapping 890 */ 891 rp->regspec_bustype += (uint_t)offset; 892 } else { 893 /* 894 * Normal memory or i/o mapping 895 */ 896 rp->regspec_addr += (uint_t)offset; 897 } 898 899 if (len != 0) 900 rp->regspec_size = (uint_t)len; 901 902 #ifdef DDI_MAP_DEBUG 903 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 904 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 905 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 906 offset, len, mp->map_handlep); 907 #endif /* DDI_MAP_DEBUG */ 908 909 /* 910 * Apply any parent ranges at this level, if applicable. 911 * (This is where nexus specific regspec translation takes place. 912 * Use of this function is implicit agreement that translation is 913 * provided via ddi_apply_range.) 914 */ 915 916 #ifdef DDI_MAP_DEBUG 917 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 918 ddi_get_name(dip), ddi_get_name(rdip)); 919 #endif /* DDI_MAP_DEBUG */ 920 921 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 922 return (error); 923 924 switch (mp->map_op) { 925 case DDI_MO_MAP_LOCKED: 926 927 /* 928 * Set up the locked down kernel mapping to the regspec... 929 */ 930 931 return (rootnex_map_regspec(mp, vaddrp)); 932 933 case DDI_MO_UNMAP: 934 935 /* 936 * Release mapping... 937 */ 938 939 return (rootnex_unmap_regspec(mp, vaddrp)); 940 941 case DDI_MO_MAP_HANDLE: 942 943 return (rootnex_map_handle(mp)); 944 945 default: 946 return (DDI_ME_UNIMPLEMENTED); 947 } 948 } 949 950 951 /* 952 * rootnex_map_fault() 953 * 954 * fault in mappings for requestors 955 */ 956 /*ARGSUSED*/ 957 static int 958 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 959 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 960 uint_t lock) 961 { 962 963 #ifdef DDI_MAP_DEBUG 964 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 965 ddi_map_debug(" Seg <%s>\n", 966 seg->s_ops == &segdev_ops ? "segdev" : 967 seg == &kvseg ? "segkmem" : "NONE!"); 968 #endif /* DDI_MAP_DEBUG */ 969 970 /* 971 * This is all terribly broken, but it is a start 972 * 973 * XXX Note that this test means that segdev_ops 974 * must be exported from seg_dev.c. 975 * XXX What about devices with their own segment drivers? 976 */ 977 if (seg->s_ops == &segdev_ops) { 978 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 979 980 if (hat == NULL) { 981 /* 982 * This is one plausible interpretation of 983 * a null hat i.e. use the first hat on the 984 * address space hat list which by convention is 985 * the hat of the system MMU. At alternative 986 * would be to panic .. this might well be better .. 987 */ 988 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 989 hat = seg->s_as->a_hat; 990 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 991 } 992 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 993 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 994 } else if (seg == &kvseg && dp == NULL) { 995 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 996 HAT_LOAD_LOCK); 997 } else 998 return (DDI_FAILURE); 999 return (DDI_SUCCESS); 1000 } 1001 1002 1003 /* 1004 * rootnex_map_regspec() 1005 * we don't support mapping of I/O cards above 4Gb 1006 */ 1007 static int 1008 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1009 { 1010 rootnex_addr_t rbase; 1011 void *cvaddr; 1012 uint_t npages, pgoffset; 1013 struct regspec *rp; 1014 ddi_acc_hdl_t *hp; 1015 ddi_acc_impl_t *ap; 1016 uint_t hat_acc_flags; 1017 paddr_t pbase; 1018 1019 rp = mp->map_obj.rp; 1020 hp = mp->map_handlep; 1021 1022 #ifdef DDI_MAP_DEBUG 1023 ddi_map_debug( 1024 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1025 rp->regspec_bustype, rp->regspec_addr, 1026 rp->regspec_size, mp->map_handlep); 1027 #endif /* DDI_MAP_DEBUG */ 1028 1029 /* 1030 * I/O or memory mapping 1031 * 1032 * <bustype=0, addr=x, len=x>: memory 1033 * <bustype=1, addr=x, len=x>: i/o 1034 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1035 */ 1036 1037 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1038 cmn_err(CE_WARN, "rootnex: invalid register spec" 1039 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1040 rp->regspec_addr, rp->regspec_size); 1041 return (DDI_FAILURE); 1042 } 1043 1044 if (rp->regspec_bustype != 0) { 1045 /* 1046 * I/O space - needs a handle. 1047 */ 1048 if (hp == NULL) { 1049 return (DDI_FAILURE); 1050 } 1051 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1052 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1053 impl_acc_hdl_init(hp); 1054 1055 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1056 #ifdef DDI_MAP_DEBUG 1057 ddi_map_debug("rootnex_map_regspec: mmap() " 1058 "to I/O space is not supported.\n"); 1059 #endif /* DDI_MAP_DEBUG */ 1060 return (DDI_ME_INVAL); 1061 } else { 1062 /* 1063 * 1275-compliant vs. compatibility i/o mapping 1064 */ 1065 *vaddrp = 1066 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1067 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1068 ((caddr_t)(uintptr_t)rp->regspec_addr); 1069 #ifdef __xpv 1070 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1071 hp->ah_pfn = xen_assign_pfn( 1072 mmu_btop((ulong_t)rp->regspec_addr & 1073 MMU_PAGEMASK)); 1074 } else { 1075 hp->ah_pfn = mmu_btop( 1076 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1077 } 1078 #else 1079 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1080 MMU_PAGEMASK); 1081 #endif 1082 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1083 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1084 } 1085 1086 #ifdef DDI_MAP_DEBUG 1087 ddi_map_debug( 1088 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1089 rp->regspec_size, *vaddrp); 1090 #endif /* DDI_MAP_DEBUG */ 1091 return (DDI_SUCCESS); 1092 } 1093 1094 /* 1095 * Memory space 1096 */ 1097 1098 if (hp != NULL) { 1099 /* 1100 * hat layer ignores 1101 * hp->ah_acc.devacc_attr_endian_flags. 1102 */ 1103 switch (hp->ah_acc.devacc_attr_dataorder) { 1104 case DDI_STRICTORDER_ACC: 1105 hat_acc_flags = HAT_STRICTORDER; 1106 break; 1107 case DDI_UNORDERED_OK_ACC: 1108 hat_acc_flags = HAT_UNORDERED_OK; 1109 break; 1110 case DDI_MERGING_OK_ACC: 1111 hat_acc_flags = HAT_MERGING_OK; 1112 break; 1113 case DDI_LOADCACHING_OK_ACC: 1114 hat_acc_flags = HAT_LOADCACHING_OK; 1115 break; 1116 case DDI_STORECACHING_OK_ACC: 1117 hat_acc_flags = HAT_STORECACHING_OK; 1118 break; 1119 } 1120 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1121 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1122 impl_acc_hdl_init(hp); 1123 hp->ah_hat_flags = hat_acc_flags; 1124 } else { 1125 hat_acc_flags = HAT_STRICTORDER; 1126 } 1127 1128 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1129 #ifdef __xpv 1130 /* 1131 * If we're dom0, we're using a real device so we need to translate 1132 * the MA to a PA. 1133 */ 1134 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1135 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1136 } else { 1137 pbase = rbase; 1138 } 1139 #else 1140 pbase = rbase; 1141 #endif 1142 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1143 1144 if (rp->regspec_size == 0) { 1145 #ifdef DDI_MAP_DEBUG 1146 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1147 #endif /* DDI_MAP_DEBUG */ 1148 return (DDI_ME_INVAL); 1149 } 1150 1151 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1152 /* extra cast to make gcc happy */ 1153 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1154 } else { 1155 npages = mmu_btopr(rp->regspec_size + pgoffset); 1156 1157 #ifdef DDI_MAP_DEBUG 1158 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1159 "physical %llx", npages, pbase); 1160 #endif /* DDI_MAP_DEBUG */ 1161 1162 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1163 if (cvaddr == NULL) 1164 return (DDI_ME_NORESOURCES); 1165 1166 /* 1167 * Now map in the pages we've allocated... 1168 */ 1169 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1170 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1171 HAT_LOAD_LOCK); 1172 *vaddrp = (caddr_t)cvaddr + pgoffset; 1173 1174 /* save away pfn and npages for FMA */ 1175 hp = mp->map_handlep; 1176 if (hp) { 1177 hp->ah_pfn = mmu_btop(pbase); 1178 hp->ah_pnum = npages; 1179 } 1180 } 1181 1182 #ifdef DDI_MAP_DEBUG 1183 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1184 #endif /* DDI_MAP_DEBUG */ 1185 return (DDI_SUCCESS); 1186 } 1187 1188 1189 /* 1190 * rootnex_unmap_regspec() 1191 * 1192 */ 1193 static int 1194 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1195 { 1196 caddr_t addr = (caddr_t)*vaddrp; 1197 uint_t npages, pgoffset; 1198 struct regspec *rp; 1199 1200 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1201 return (0); 1202 1203 rp = mp->map_obj.rp; 1204 1205 if (rp->regspec_size == 0) { 1206 #ifdef DDI_MAP_DEBUG 1207 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1208 #endif /* DDI_MAP_DEBUG */ 1209 return (DDI_ME_INVAL); 1210 } 1211 1212 /* 1213 * I/O or memory mapping: 1214 * 1215 * <bustype=0, addr=x, len=x>: memory 1216 * <bustype=1, addr=x, len=x>: i/o 1217 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1218 */ 1219 if (rp->regspec_bustype != 0) { 1220 /* 1221 * This is I/O space, which requires no particular 1222 * processing on unmap since it isn't mapped in the 1223 * first place. 1224 */ 1225 return (DDI_SUCCESS); 1226 } 1227 1228 /* 1229 * Memory space 1230 */ 1231 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1232 npages = mmu_btopr(rp->regspec_size + pgoffset); 1233 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1234 device_arena_free(addr - pgoffset, ptob(npages)); 1235 1236 /* 1237 * Destroy the pointer - the mapping has logically gone 1238 */ 1239 *vaddrp = NULL; 1240 1241 return (DDI_SUCCESS); 1242 } 1243 1244 1245 /* 1246 * rootnex_map_handle() 1247 * 1248 */ 1249 static int 1250 rootnex_map_handle(ddi_map_req_t *mp) 1251 { 1252 rootnex_addr_t rbase; 1253 ddi_acc_hdl_t *hp; 1254 uint_t pgoffset; 1255 struct regspec *rp; 1256 paddr_t pbase; 1257 1258 rp = mp->map_obj.rp; 1259 1260 #ifdef DDI_MAP_DEBUG 1261 ddi_map_debug( 1262 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1263 rp->regspec_bustype, rp->regspec_addr, 1264 rp->regspec_size, mp->map_handlep); 1265 #endif /* DDI_MAP_DEBUG */ 1266 1267 /* 1268 * I/O or memory mapping: 1269 * 1270 * <bustype=0, addr=x, len=x>: memory 1271 * <bustype=1, addr=x, len=x>: i/o 1272 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1273 */ 1274 if (rp->regspec_bustype != 0) { 1275 /* 1276 * This refers to I/O space, and we don't support "mapping" 1277 * I/O space to a user. 1278 */ 1279 return (DDI_FAILURE); 1280 } 1281 1282 /* 1283 * Set up the hat_flags for the mapping. 1284 */ 1285 hp = mp->map_handlep; 1286 1287 switch (hp->ah_acc.devacc_attr_endian_flags) { 1288 case DDI_NEVERSWAP_ACC: 1289 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1290 break; 1291 case DDI_STRUCTURE_LE_ACC: 1292 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1293 break; 1294 case DDI_STRUCTURE_BE_ACC: 1295 return (DDI_FAILURE); 1296 default: 1297 return (DDI_REGS_ACC_CONFLICT); 1298 } 1299 1300 switch (hp->ah_acc.devacc_attr_dataorder) { 1301 case DDI_STRICTORDER_ACC: 1302 break; 1303 case DDI_UNORDERED_OK_ACC: 1304 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1305 break; 1306 case DDI_MERGING_OK_ACC: 1307 hp->ah_hat_flags |= HAT_MERGING_OK; 1308 break; 1309 case DDI_LOADCACHING_OK_ACC: 1310 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1311 break; 1312 case DDI_STORECACHING_OK_ACC: 1313 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1314 break; 1315 default: 1316 return (DDI_FAILURE); 1317 } 1318 1319 rbase = (rootnex_addr_t)rp->regspec_addr & 1320 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1321 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1322 1323 if (rp->regspec_size == 0) 1324 return (DDI_ME_INVAL); 1325 1326 #ifdef __xpv 1327 /* 1328 * If we're dom0, we're using a real device so we need to translate 1329 * the MA to a PA. 1330 */ 1331 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1332 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1333 (rbase & MMU_PAGEOFFSET); 1334 } else { 1335 pbase = rbase; 1336 } 1337 #else 1338 pbase = rbase; 1339 #endif 1340 1341 hp->ah_pfn = mmu_btop(pbase); 1342 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1343 1344 return (DDI_SUCCESS); 1345 } 1346 1347 1348 1349 /* 1350 * ************************ 1351 * interrupt related code 1352 * ************************ 1353 */ 1354 1355 /* 1356 * rootnex_intr_ops() 1357 * bus_intr_op() function for interrupt support 1358 */ 1359 /* ARGSUSED */ 1360 static int 1361 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1362 ddi_intr_handle_impl_t *hdlp, void *result) 1363 { 1364 struct intrspec *ispec; 1365 struct ddi_parent_private_data *pdp; 1366 1367 DDI_INTR_NEXDBG((CE_CONT, 1368 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1369 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1370 1371 /* Process the interrupt operation */ 1372 switch (intr_op) { 1373 case DDI_INTROP_GETCAP: 1374 /* First check with pcplusmp */ 1375 if (psm_intr_ops == NULL) 1376 return (DDI_FAILURE); 1377 1378 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1379 *(int *)result = 0; 1380 return (DDI_FAILURE); 1381 } 1382 break; 1383 case DDI_INTROP_SETCAP: 1384 if (psm_intr_ops == NULL) 1385 return (DDI_FAILURE); 1386 1387 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1388 return (DDI_FAILURE); 1389 break; 1390 case DDI_INTROP_ALLOC: 1391 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1392 return (DDI_FAILURE); 1393 hdlp->ih_pri = ispec->intrspec_pri; 1394 *(int *)result = hdlp->ih_scratch1; 1395 break; 1396 case DDI_INTROP_FREE: 1397 pdp = ddi_get_parent_data(rdip); 1398 /* 1399 * Special case for 'pcic' driver' only. 1400 * If an intrspec was created for it, clean it up here 1401 * See detailed comments on this in the function 1402 * rootnex_get_ispec(). 1403 */ 1404 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1405 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1406 pdp->par_nintr); 1407 /* 1408 * Set it to zero; so that 1409 * DDI framework doesn't free it again 1410 */ 1411 pdp->par_intr = NULL; 1412 pdp->par_nintr = 0; 1413 } 1414 break; 1415 case DDI_INTROP_GETPRI: 1416 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1417 return (DDI_FAILURE); 1418 *(int *)result = ispec->intrspec_pri; 1419 break; 1420 case DDI_INTROP_SETPRI: 1421 /* Validate the interrupt priority passed to us */ 1422 if (*(int *)result > LOCK_LEVEL) 1423 return (DDI_FAILURE); 1424 1425 /* Ensure that PSM is all initialized and ispec is ok */ 1426 if ((psm_intr_ops == NULL) || 1427 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1428 return (DDI_FAILURE); 1429 1430 /* Change the priority */ 1431 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1432 PSM_FAILURE) 1433 return (DDI_FAILURE); 1434 1435 /* update the ispec with the new priority */ 1436 ispec->intrspec_pri = *(int *)result; 1437 break; 1438 case DDI_INTROP_ADDISR: 1439 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1440 return (DDI_FAILURE); 1441 ispec->intrspec_func = hdlp->ih_cb_func; 1442 break; 1443 case DDI_INTROP_REMISR: 1444 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 return (DDI_FAILURE); 1446 ispec->intrspec_func = (uint_t (*)()) 0; 1447 break; 1448 case DDI_INTROP_ENABLE: 1449 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1450 return (DDI_FAILURE); 1451 1452 /* Call psmi to translate irq with the dip */ 1453 if (psm_intr_ops == NULL) 1454 return (DDI_FAILURE); 1455 1456 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1457 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1458 (int *)&hdlp->ih_vector); 1459 1460 /* Add the interrupt handler */ 1461 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1462 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1463 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1464 return (DDI_FAILURE); 1465 break; 1466 case DDI_INTROP_DISABLE: 1467 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1468 return (DDI_FAILURE); 1469 1470 /* Call psm_ops() to translate irq with the dip */ 1471 if (psm_intr_ops == NULL) 1472 return (DDI_FAILURE); 1473 1474 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1475 (void) (*psm_intr_ops)(rdip, hdlp, 1476 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1477 1478 /* Remove the interrupt handler */ 1479 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1480 hdlp->ih_cb_func, hdlp->ih_vector); 1481 break; 1482 case DDI_INTROP_SETMASK: 1483 if (psm_intr_ops == NULL) 1484 return (DDI_FAILURE); 1485 1486 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1487 return (DDI_FAILURE); 1488 break; 1489 case DDI_INTROP_CLRMASK: 1490 if (psm_intr_ops == NULL) 1491 return (DDI_FAILURE); 1492 1493 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1494 return (DDI_FAILURE); 1495 break; 1496 case DDI_INTROP_GETPENDING: 1497 if (psm_intr_ops == NULL) 1498 return (DDI_FAILURE); 1499 1500 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1501 result)) { 1502 *(int *)result = 0; 1503 return (DDI_FAILURE); 1504 } 1505 break; 1506 case DDI_INTROP_NAVAIL: 1507 case DDI_INTROP_NINTRS: 1508 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1509 if (*(int *)result == 0) { 1510 /* 1511 * Special case for 'pcic' driver' only. This driver 1512 * driver is a child of 'isa' and 'rootnex' drivers. 1513 * 1514 * See detailed comments on this in the function 1515 * rootnex_get_ispec(). 1516 * 1517 * Children of 'pcic' send 'NINITR' request all the 1518 * way to rootnex driver. But, the 'pdp->par_nintr' 1519 * field may not initialized. So, we fake it here 1520 * to return 1 (a la what PCMCIA nexus does). 1521 */ 1522 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1523 *(int *)result = 1; 1524 else 1525 return (DDI_FAILURE); 1526 } 1527 break; 1528 case DDI_INTROP_SUPPORTED_TYPES: 1529 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1530 break; 1531 default: 1532 return (DDI_FAILURE); 1533 } 1534 1535 return (DDI_SUCCESS); 1536 } 1537 1538 1539 /* 1540 * rootnex_get_ispec() 1541 * convert an interrupt number to an interrupt specification. 1542 * The interrupt number determines which interrupt spec will be 1543 * returned if more than one exists. 1544 * 1545 * Look into the parent private data area of the 'rdip' to find out 1546 * the interrupt specification. First check to make sure there is 1547 * one that matchs "inumber" and then return a pointer to it. 1548 * 1549 * Return NULL if one could not be found. 1550 * 1551 * NOTE: This is needed for rootnex_intr_ops() 1552 */ 1553 static struct intrspec * 1554 rootnex_get_ispec(dev_info_t *rdip, int inum) 1555 { 1556 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1557 1558 /* 1559 * Special case handling for drivers that provide their own 1560 * intrspec structures instead of relying on the DDI framework. 1561 * 1562 * A broken hardware driver in ON could potentially provide its 1563 * own intrspec structure, instead of relying on the hardware. 1564 * If these drivers are children of 'rootnex' then we need to 1565 * continue to provide backward compatibility to them here. 1566 * 1567 * Following check is a special case for 'pcic' driver which 1568 * was found to have broken hardwre andby provides its own intrspec. 1569 * 1570 * Verbatim comments from this driver are shown here: 1571 * "Don't use the ddi_add_intr since we don't have a 1572 * default intrspec in all cases." 1573 * 1574 * Since an 'ispec' may not be always created for it, 1575 * check for that and create one if so. 1576 * 1577 * NOTE: Currently 'pcic' is the only driver found to do this. 1578 */ 1579 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1580 pdp->par_nintr = 1; 1581 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1582 pdp->par_nintr, KM_SLEEP); 1583 } 1584 1585 /* Validate the interrupt number */ 1586 if (inum >= pdp->par_nintr) 1587 return (NULL); 1588 1589 /* Get the interrupt structure pointer and return that */ 1590 return ((struct intrspec *)&pdp->par_intr[inum]); 1591 } 1592 1593 1594 /* 1595 * ****************** 1596 * dma related code 1597 * ****************** 1598 */ 1599 1600 /*ARGSUSED*/ 1601 static int 1602 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1603 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1604 ddi_dma_handle_t *handlep) 1605 { 1606 uint64_t maxsegmentsize_ll; 1607 uint_t maxsegmentsize; 1608 ddi_dma_impl_t *hp; 1609 rootnex_dma_t *dma; 1610 uint64_t count_max; 1611 uint64_t seg; 1612 int kmflag; 1613 int e; 1614 1615 1616 /* convert our sleep flags */ 1617 if (waitfp == DDI_DMA_SLEEP) { 1618 kmflag = KM_SLEEP; 1619 } else { 1620 kmflag = KM_NOSLEEP; 1621 } 1622 1623 /* 1624 * We try to do only one memory allocation here. We'll do a little 1625 * pointer manipulation later. If the bind ends up taking more than 1626 * our prealloc's space, we'll have to allocate more memory in the 1627 * bind operation. Not great, but much better than before and the 1628 * best we can do with the current bind interfaces. 1629 */ 1630 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1631 if (hp == NULL) { 1632 if (waitfp != DDI_DMA_DONTWAIT) { 1633 ddi_set_callback(waitfp, arg, 1634 &rootnex_state->r_dvma_call_list_id); 1635 } 1636 return (DDI_DMA_NORESOURCES); 1637 } 1638 1639 /* Do our pointer manipulation now, align the structures */ 1640 hp->dmai_private = (void *)(((uintptr_t)hp + 1641 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1642 dma = (rootnex_dma_t *)hp->dmai_private; 1643 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1644 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1645 1646 /* setup the handle */ 1647 rootnex_clean_dmahdl(hp); 1648 dma->dp_dip = rdip; 1649 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1650 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1651 hp->dmai_minxfer = attr->dma_attr_minxfer; 1652 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1653 hp->dmai_rdip = rdip; 1654 hp->dmai_attr = *attr; 1655 1656 /* we don't need to worry about the SPL since we do a tryenter */ 1657 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1658 1659 /* 1660 * Figure out our maximum segment size. If the segment size is greater 1661 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1662 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1663 * dma_attr_count_max are size-1 type values. 1664 * 1665 * Maximum segment size is the largest physically contiguous chunk of 1666 * memory that we can return from a bind (i.e. the maximum size of a 1667 * single cookie). 1668 */ 1669 1670 /* handle the rollover cases */ 1671 seg = attr->dma_attr_seg + 1; 1672 if (seg < attr->dma_attr_seg) { 1673 seg = attr->dma_attr_seg; 1674 } 1675 count_max = attr->dma_attr_count_max + 1; 1676 if (count_max < attr->dma_attr_count_max) { 1677 count_max = attr->dma_attr_count_max; 1678 } 1679 1680 /* 1681 * granularity may or may not be a power of two. If it isn't, we can't 1682 * use a simple mask. 1683 */ 1684 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1685 dma->dp_granularity_power_2 = B_FALSE; 1686 } else { 1687 dma->dp_granularity_power_2 = B_TRUE; 1688 } 1689 1690 /* 1691 * maxxfer should be a whole multiple of granularity. If we're going to 1692 * break up a window because we're greater than maxxfer, we might as 1693 * well make sure it's maxxfer is a whole multiple so we don't have to 1694 * worry about triming the window later on for this case. 1695 */ 1696 if (attr->dma_attr_granular > 1) { 1697 if (dma->dp_granularity_power_2) { 1698 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1699 (attr->dma_attr_maxxfer & 1700 (attr->dma_attr_granular - 1)); 1701 } else { 1702 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1703 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1704 } 1705 } else { 1706 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1707 } 1708 1709 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1710 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1711 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1712 maxsegmentsize = 0xFFFFFFFF; 1713 } else { 1714 maxsegmentsize = maxsegmentsize_ll; 1715 } 1716 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1717 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1718 1719 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1720 if (rootnex_alloc_check_parms) { 1721 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1722 if (e != DDI_SUCCESS) { 1723 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1724 (void) rootnex_dma_freehdl(dip, rdip, 1725 (ddi_dma_handle_t)hp); 1726 return (e); 1727 } 1728 } 1729 1730 *handlep = (ddi_dma_handle_t)hp; 1731 1732 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1733 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1734 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1735 1736 return (DDI_SUCCESS); 1737 } 1738 1739 1740 /* 1741 * rootnex_dma_allochdl() 1742 * called from ddi_dma_alloc_handle(). 1743 */ 1744 static int 1745 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1746 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1747 { 1748 #if !defined(__xpv) 1749 uint_t error = ENOTSUP; 1750 int retval; 1751 1752 retval = iommulib_nex_open(rdip, &error); 1753 1754 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1755 /* No IOMMU */ 1756 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1757 handlep)); 1758 } else if (retval != DDI_SUCCESS) { 1759 return (DDI_FAILURE); 1760 } 1761 1762 ASSERT(IOMMU_USED(rdip)); 1763 1764 /* has an IOMMU */ 1765 return (iommulib_nexdma_allochdl(dip, rdip, attr, 1766 waitfp, arg, handlep)); 1767 #else 1768 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1769 handlep)); 1770 #endif 1771 } 1772 1773 /*ARGSUSED*/ 1774 static int 1775 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1776 ddi_dma_handle_t handle) 1777 { 1778 ddi_dma_impl_t *hp; 1779 rootnex_dma_t *dma; 1780 1781 1782 hp = (ddi_dma_impl_t *)handle; 1783 dma = (rootnex_dma_t *)hp->dmai_private; 1784 1785 /* unbind should have been called first */ 1786 ASSERT(!dma->dp_inuse); 1787 1788 mutex_destroy(&dma->dp_mutex); 1789 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1790 1791 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1792 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1793 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1794 1795 if (rootnex_state->r_dvma_call_list_id) 1796 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1797 1798 return (DDI_SUCCESS); 1799 } 1800 1801 /* 1802 * rootnex_dma_freehdl() 1803 * called from ddi_dma_free_handle(). 1804 */ 1805 static int 1806 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1807 { 1808 #if !defined(__xpv) 1809 if (IOMMU_USED(rdip)) { 1810 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1811 } 1812 #endif 1813 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1814 } 1815 1816 1817 /*ARGSUSED*/ 1818 static int 1819 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1820 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1821 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1822 { 1823 rootnex_sglinfo_t *sinfo; 1824 ddi_dma_attr_t *attr; 1825 ddi_dma_impl_t *hp; 1826 rootnex_dma_t *dma; 1827 int kmflag; 1828 int e; 1829 1830 1831 hp = (ddi_dma_impl_t *)handle; 1832 dma = (rootnex_dma_t *)hp->dmai_private; 1833 sinfo = &dma->dp_sglinfo; 1834 attr = &hp->dmai_attr; 1835 1836 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1837 dma->dp_sleep_flags = KM_SLEEP; 1838 } else { 1839 dma->dp_sleep_flags = KM_NOSLEEP; 1840 } 1841 1842 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1843 1844 /* 1845 * This is useful for debugging a driver. Not as useful in a production 1846 * system. The only time this will fail is if you have a driver bug. 1847 */ 1848 if (rootnex_bind_check_inuse) { 1849 /* 1850 * No one else should ever have this lock unless someone else 1851 * is trying to use this handle. So contention on the lock 1852 * is the same as inuse being set. 1853 */ 1854 e = mutex_tryenter(&dma->dp_mutex); 1855 if (e == 0) { 1856 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1857 return (DDI_DMA_INUSE); 1858 } 1859 if (dma->dp_inuse) { 1860 mutex_exit(&dma->dp_mutex); 1861 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1862 return (DDI_DMA_INUSE); 1863 } 1864 dma->dp_inuse = B_TRUE; 1865 mutex_exit(&dma->dp_mutex); 1866 } 1867 1868 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1869 if (rootnex_bind_check_parms) { 1870 e = rootnex_valid_bind_parms(dmareq, attr); 1871 if (e != DDI_SUCCESS) { 1872 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1873 rootnex_clean_dmahdl(hp); 1874 return (e); 1875 } 1876 } 1877 1878 /* save away the original bind info */ 1879 dma->dp_dma = dmareq->dmar_object; 1880 1881 #if !defined(__xpv) 1882 if (rootnex_state->r_intel_iommu_enabled) { 1883 e = intel_iommu_map_sgl(handle, dmareq, 1884 rootnex_state->r_prealloc_cookies); 1885 1886 switch (e) { 1887 case IOMMU_SGL_SUCCESS: 1888 goto rootnex_sgl_end; 1889 1890 case IOMMU_SGL_DISABLE: 1891 goto rootnex_sgl_start; 1892 1893 case IOMMU_SGL_NORESOURCES: 1894 cmn_err(CE_WARN, "iommu map sgl failed for %s", 1895 ddi_node_name(dma->dp_dip)); 1896 rootnex_clean_dmahdl(hp); 1897 return (DDI_DMA_NORESOURCES); 1898 1899 default: 1900 cmn_err(CE_WARN, 1901 "undefined value returned from" 1902 " intel_iommu_map_sgl: %d", 1903 e); 1904 rootnex_clean_dmahdl(hp); 1905 return (DDI_DMA_NORESOURCES); 1906 } 1907 } 1908 #endif 1909 1910 rootnex_sgl_start: 1911 /* 1912 * Figure out a rough estimate of what maximum number of pages this 1913 * buffer could use (a high estimate of course). 1914 */ 1915 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1916 1917 /* 1918 * We'll use the pre-allocated cookies for any bind that will *always* 1919 * fit (more important to be consistent, we don't want to create 1920 * additional degenerate cases). 1921 */ 1922 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1923 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1924 dma->dp_need_to_free_cookie = B_FALSE; 1925 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1926 uint_t, sinfo->si_max_pages); 1927 1928 /* 1929 * For anything larger than that, we'll go ahead and allocate the 1930 * maximum number of pages we expect to see. Hopefuly, we won't be 1931 * seeing this path in the fast path for high performance devices very 1932 * frequently. 1933 * 1934 * a ddi bind interface that allowed the driver to provide storage to 1935 * the bind interface would speed this case up. 1936 */ 1937 } else { 1938 /* convert the sleep flags */ 1939 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1940 kmflag = KM_SLEEP; 1941 } else { 1942 kmflag = KM_NOSLEEP; 1943 } 1944 1945 /* 1946 * Save away how much memory we allocated. If we're doing a 1947 * nosleep, the alloc could fail... 1948 */ 1949 dma->dp_cookie_size = sinfo->si_max_pages * 1950 sizeof (ddi_dma_cookie_t); 1951 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1952 if (dma->dp_cookies == NULL) { 1953 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1954 rootnex_clean_dmahdl(hp); 1955 return (DDI_DMA_NORESOURCES); 1956 } 1957 dma->dp_need_to_free_cookie = B_TRUE; 1958 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1959 sinfo->si_max_pages); 1960 } 1961 hp->dmai_cookie = dma->dp_cookies; 1962 1963 /* 1964 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1965 * looking at the contraints in the dma structure. It will then put some 1966 * additional state about the sgl in the dma struct (i.e. is the sgl 1967 * clean, or do we need to do some munging; how many pages need to be 1968 * copied, etc.) 1969 */ 1970 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1971 &dma->dp_sglinfo); 1972 1973 rootnex_sgl_end: 1974 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1975 /* if we don't need a copy buffer, we don't need to sync */ 1976 if (sinfo->si_copybuf_req == 0) { 1977 hp->dmai_rflags |= DMP_NOSYNC; 1978 } 1979 1980 /* 1981 * if we don't need the copybuf and we don't need to do a partial, we 1982 * hit the fast path. All the high performance devices should be trying 1983 * to hit this path. To hit this path, a device should be able to reach 1984 * all of memory, shouldn't try to bind more than it can transfer, and 1985 * the buffer shouldn't require more cookies than the driver/device can 1986 * handle [sgllen]). 1987 */ 1988 if ((sinfo->si_copybuf_req == 0) && 1989 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1990 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1991 /* 1992 * If the driver supports FMA, insert the handle in the FMA DMA 1993 * handle cache. 1994 */ 1995 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1996 hp->dmai_error.err_cf = rootnex_dma_check; 1997 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1998 } 1999 2000 /* 2001 * copy out the first cookie and ccountp, set the cookie 2002 * pointer to the second cookie. The first cookie is passed 2003 * back on the stack. Additional cookies are accessed via 2004 * ddi_dma_nextcookie() 2005 */ 2006 *cookiep = dma->dp_cookies[0]; 2007 *ccountp = sinfo->si_sgl_size; 2008 hp->dmai_cookie++; 2009 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2010 hp->dmai_nwin = 1; 2011 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2012 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 2013 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2014 dma->dp_dma.dmao_size); 2015 return (DDI_DMA_MAPPED); 2016 } 2017 2018 /* 2019 * go to the slow path, we may need to alloc more memory, create 2020 * multiple windows, and munge up a sgl to make the device happy. 2021 */ 2022 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2023 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2024 if (dma->dp_need_to_free_cookie) { 2025 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2026 } 2027 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2028 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2029 return (e); 2030 } 2031 2032 /* 2033 * If the driver supports FMA, insert the handle in the FMA DMA handle 2034 * cache. 2035 */ 2036 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 2037 hp->dmai_error.err_cf = rootnex_dma_check; 2038 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 2039 } 2040 2041 /* if the first window uses the copy buffer, sync it for the device */ 2042 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2043 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2044 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2045 DDI_DMA_SYNC_FORDEV); 2046 } 2047 2048 /* 2049 * copy out the first cookie and ccountp, set the cookie pointer to the 2050 * second cookie. Make sure the partial flag is set/cleared correctly. 2051 * If we have a partial map (i.e. multiple windows), the number of 2052 * cookies we return is the number of cookies in the first window. 2053 */ 2054 if (e == DDI_DMA_MAPPED) { 2055 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2056 *ccountp = sinfo->si_sgl_size; 2057 } else { 2058 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2059 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2060 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2061 } 2062 *cookiep = dma->dp_cookies[0]; 2063 hp->dmai_cookie++; 2064 2065 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2066 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2067 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2068 dma->dp_dma.dmao_size); 2069 return (e); 2070 } 2071 2072 2073 /* 2074 * rootnex_dma_bindhdl() 2075 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2076 */ 2077 static int 2078 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2079 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2080 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2081 { 2082 #if !defined(__xpv) 2083 if (IOMMU_USED(rdip)) { 2084 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2085 cookiep, ccountp)); 2086 } 2087 #endif 2088 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2089 cookiep, ccountp)); 2090 } 2091 2092 /*ARGSUSED*/ 2093 static int 2094 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2095 ddi_dma_handle_t handle) 2096 { 2097 ddi_dma_impl_t *hp; 2098 rootnex_dma_t *dma; 2099 int e; 2100 2101 hp = (ddi_dma_impl_t *)handle; 2102 dma = (rootnex_dma_t *)hp->dmai_private; 2103 2104 /* make sure the buffer wasn't free'd before calling unbind */ 2105 if (rootnex_unbind_verify_buffer) { 2106 e = rootnex_verify_buffer(dma); 2107 if (e != DDI_SUCCESS) { 2108 ASSERT(0); 2109 return (DDI_FAILURE); 2110 } 2111 } 2112 2113 /* sync the current window before unbinding the buffer */ 2114 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2115 (hp->dmai_rflags & DDI_DMA_READ)) { 2116 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2117 DDI_DMA_SYNC_FORCPU); 2118 } 2119 2120 /* 2121 * If the driver supports FMA, remove the handle in the FMA DMA handle 2122 * cache. 2123 */ 2124 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 2125 if ((DEVI(rdip)->devi_fmhdl != NULL) && 2126 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 2127 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 2128 } 2129 } 2130 2131 /* 2132 * cleanup and copy buffer or window state. if we didn't use the copy 2133 * buffer or windows, there won't be much to do :-) 2134 */ 2135 rootnex_teardown_copybuf(dma); 2136 rootnex_teardown_windows(dma); 2137 2138 #if !defined(__xpv) 2139 /* 2140 * If intel iommu enabled, clean up the page tables and free the dvma 2141 */ 2142 if (rootnex_state->r_intel_iommu_enabled) { 2143 intel_iommu_unmap_sgl(handle); 2144 } 2145 #endif 2146 2147 /* 2148 * If we had to allocate space to for the worse case sgl (it didn't 2149 * fit into our pre-allocate buffer), free that up now 2150 */ 2151 if (dma->dp_need_to_free_cookie) { 2152 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2153 } 2154 2155 /* 2156 * clean up the handle so it's ready for the next bind (i.e. if the 2157 * handle is reused). 2158 */ 2159 rootnex_clean_dmahdl(hp); 2160 2161 if (rootnex_state->r_dvma_call_list_id) 2162 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2163 2164 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2165 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2166 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2167 2168 return (DDI_SUCCESS); 2169 } 2170 2171 /* 2172 * rootnex_dma_unbindhdl() 2173 * called from ddi_dma_unbind_handle() 2174 */ 2175 /*ARGSUSED*/ 2176 static int 2177 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2178 ddi_dma_handle_t handle) 2179 { 2180 #if !defined(__xpv) 2181 if (IOMMU_USED(rdip)) { 2182 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2183 } 2184 #endif 2185 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2186 } 2187 2188 #if !defined(__xpv) 2189 2190 static int 2191 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2192 { 2193 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2194 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2195 2196 if (dma->dp_sleep_flags != KM_SLEEP && 2197 dma->dp_sleep_flags != KM_NOSLEEP) 2198 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2199 return (dma->dp_sleep_flags); 2200 } 2201 /*ARGSUSED*/ 2202 static void 2203 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2204 { 2205 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2206 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2207 rootnex_window_t *window; 2208 2209 if (dma->dp_window) { 2210 window = &dma->dp_window[dma->dp_current_win]; 2211 hp->dmai_cookie = window->wd_first_cookie; 2212 } else { 2213 hp->dmai_cookie = dma->dp_cookies; 2214 } 2215 hp->dmai_cookie++; 2216 } 2217 2218 /*ARGSUSED*/ 2219 static int 2220 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2221 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2222 { 2223 int i; 2224 int km_flags; 2225 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2226 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2227 rootnex_window_t *window; 2228 ddi_dma_cookie_t *cp; 2229 ddi_dma_cookie_t *cookie; 2230 2231 ASSERT(*cookiepp == NULL); 2232 ASSERT(*ccountp == 0); 2233 2234 if (dma->dp_window) { 2235 window = &dma->dp_window[dma->dp_current_win]; 2236 cp = window->wd_first_cookie; 2237 *ccountp = window->wd_cookie_cnt; 2238 } else { 2239 cp = dma->dp_cookies; 2240 *ccountp = dma->dp_sglinfo.si_sgl_size; 2241 } 2242 2243 km_flags = rootnex_coredma_get_sleep_flags(handle); 2244 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2245 if (cookie == NULL) { 2246 return (DDI_DMA_NORESOURCES); 2247 } 2248 2249 for (i = 0; i < *ccountp; i++) { 2250 cookie[i].dmac_notused = cp[i].dmac_notused; 2251 cookie[i].dmac_type = cp[i].dmac_type; 2252 cookie[i].dmac_address = cp[i].dmac_address; 2253 cookie[i].dmac_size = cp[i].dmac_size; 2254 } 2255 2256 *cookiepp = cookie; 2257 2258 return (DDI_SUCCESS); 2259 } 2260 2261 /*ARGSUSED*/ 2262 static int 2263 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2264 ddi_dma_cookie_t *cookiep, uint_t ccount) 2265 { 2266 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2267 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2268 rootnex_window_t *window; 2269 ddi_dma_cookie_t *cur_cookiep; 2270 2271 ASSERT(cookiep); 2272 ASSERT(ccount != 0); 2273 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2274 2275 if (dma->dp_window) { 2276 window = &dma->dp_window[dma->dp_current_win]; 2277 dma->dp_saved_cookies = window->wd_first_cookie; 2278 window->wd_first_cookie = cookiep; 2279 ASSERT(ccount == window->wd_cookie_cnt); 2280 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2281 + window->wd_first_cookie; 2282 } else { 2283 dma->dp_saved_cookies = dma->dp_cookies; 2284 dma->dp_cookies = cookiep; 2285 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2286 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2287 + dma->dp_cookies; 2288 } 2289 2290 dma->dp_need_to_switch_cookies = B_TRUE; 2291 hp->dmai_cookie = cur_cookiep; 2292 2293 return (DDI_SUCCESS); 2294 } 2295 2296 /*ARGSUSED*/ 2297 static int 2298 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2299 { 2300 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2301 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2302 rootnex_window_t *window; 2303 ddi_dma_cookie_t *cur_cookiep; 2304 ddi_dma_cookie_t *cookie_array; 2305 uint_t ccount; 2306 2307 /* check if cookies have not been switched */ 2308 if (dma->dp_need_to_switch_cookies == B_FALSE) 2309 return (DDI_SUCCESS); 2310 2311 ASSERT(dma->dp_saved_cookies); 2312 2313 if (dma->dp_window) { 2314 window = &dma->dp_window[dma->dp_current_win]; 2315 cookie_array = window->wd_first_cookie; 2316 window->wd_first_cookie = dma->dp_saved_cookies; 2317 dma->dp_saved_cookies = NULL; 2318 ccount = window->wd_cookie_cnt; 2319 cur_cookiep = (hp->dmai_cookie - cookie_array) 2320 + window->wd_first_cookie; 2321 } else { 2322 cookie_array = dma->dp_cookies; 2323 dma->dp_cookies = dma->dp_saved_cookies; 2324 dma->dp_saved_cookies = NULL; 2325 ccount = dma->dp_sglinfo.si_sgl_size; 2326 cur_cookiep = (hp->dmai_cookie - cookie_array) 2327 + dma->dp_cookies; 2328 } 2329 2330 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2331 2332 hp->dmai_cookie = cur_cookiep; 2333 2334 dma->dp_need_to_switch_cookies = B_FALSE; 2335 2336 return (DDI_SUCCESS); 2337 } 2338 2339 #endif 2340 2341 /* 2342 * rootnex_verify_buffer() 2343 * verify buffer wasn't free'd 2344 */ 2345 static int 2346 rootnex_verify_buffer(rootnex_dma_t *dma) 2347 { 2348 page_t **pplist; 2349 caddr_t vaddr; 2350 uint_t pcnt; 2351 uint_t poff; 2352 page_t *pp; 2353 char b; 2354 int i; 2355 2356 /* Figure out how many pages this buffer occupies */ 2357 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2358 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2359 } else { 2360 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2361 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2362 } 2363 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2364 2365 switch (dma->dp_dma.dmao_type) { 2366 case DMA_OTYP_PAGES: 2367 /* 2368 * for a linked list of pp's walk through them to make sure 2369 * they're locked and not free. 2370 */ 2371 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2372 for (i = 0; i < pcnt; i++) { 2373 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2374 return (DDI_FAILURE); 2375 } 2376 pp = pp->p_next; 2377 } 2378 break; 2379 2380 case DMA_OTYP_VADDR: 2381 case DMA_OTYP_BUFVADDR: 2382 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2383 /* 2384 * for an array of pp's walk through them to make sure they're 2385 * not free. It's possible that they may not be locked. 2386 */ 2387 if (pplist) { 2388 for (i = 0; i < pcnt; i++) { 2389 if (PP_ISFREE(pplist[i])) { 2390 return (DDI_FAILURE); 2391 } 2392 } 2393 2394 /* For a virtual address, try to peek at each page */ 2395 } else { 2396 if (dma->dp_sglinfo.si_asp == &kas) { 2397 for (i = 0; i < pcnt; i++) { 2398 if (ddi_peek8(NULL, vaddr, &b) == 2399 DDI_FAILURE) 2400 return (DDI_FAILURE); 2401 vaddr += MMU_PAGESIZE; 2402 } 2403 } 2404 } 2405 break; 2406 2407 default: 2408 ASSERT(0); 2409 break; 2410 } 2411 2412 return (DDI_SUCCESS); 2413 } 2414 2415 2416 /* 2417 * rootnex_clean_dmahdl() 2418 * Clean the dma handle. This should be called on a handle alloc and an 2419 * unbind handle. Set the handle state to the default settings. 2420 */ 2421 static void 2422 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2423 { 2424 rootnex_dma_t *dma; 2425 2426 2427 dma = (rootnex_dma_t *)hp->dmai_private; 2428 2429 hp->dmai_nwin = 0; 2430 dma->dp_current_cookie = 0; 2431 dma->dp_copybuf_size = 0; 2432 dma->dp_window = NULL; 2433 dma->dp_cbaddr = NULL; 2434 dma->dp_inuse = B_FALSE; 2435 dma->dp_need_to_free_cookie = B_FALSE; 2436 dma->dp_need_to_switch_cookies = B_FALSE; 2437 dma->dp_saved_cookies = NULL; 2438 dma->dp_sleep_flags = KM_PANIC; 2439 dma->dp_need_to_free_window = B_FALSE; 2440 dma->dp_partial_required = B_FALSE; 2441 dma->dp_trim_required = B_FALSE; 2442 dma->dp_sglinfo.si_copybuf_req = 0; 2443 #if !defined(__amd64) 2444 dma->dp_cb_remaping = B_FALSE; 2445 dma->dp_kva = NULL; 2446 #endif 2447 2448 /* FMA related initialization */ 2449 hp->dmai_fault = 0; 2450 hp->dmai_fault_check = NULL; 2451 hp->dmai_fault_notify = NULL; 2452 hp->dmai_error.err_ena = 0; 2453 hp->dmai_error.err_status = DDI_FM_OK; 2454 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2455 hp->dmai_error.err_ontrap = NULL; 2456 hp->dmai_error.err_fep = NULL; 2457 hp->dmai_error.err_cf = NULL; 2458 } 2459 2460 2461 /* 2462 * rootnex_valid_alloc_parms() 2463 * Called in ddi_dma_alloc_handle path to validate its parameters. 2464 */ 2465 static int 2466 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2467 { 2468 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2469 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2470 (attr->dma_attr_granular > MMU_PAGESIZE) || 2471 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2472 return (DDI_DMA_BADATTR); 2473 } 2474 2475 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2476 return (DDI_DMA_BADATTR); 2477 } 2478 2479 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2480 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2481 attr->dma_attr_sgllen <= 0) { 2482 return (DDI_DMA_BADATTR); 2483 } 2484 2485 /* We should be able to DMA into every byte offset in a page */ 2486 if (maxsegmentsize < MMU_PAGESIZE) { 2487 return (DDI_DMA_BADATTR); 2488 } 2489 2490 return (DDI_SUCCESS); 2491 } 2492 2493 2494 /* 2495 * rootnex_valid_bind_parms() 2496 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2497 */ 2498 /* ARGSUSED */ 2499 static int 2500 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2501 { 2502 #if !defined(__amd64) 2503 /* 2504 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2505 * we can track the offset for the obsoleted interfaces. 2506 */ 2507 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2508 return (DDI_DMA_TOOBIG); 2509 } 2510 #endif 2511 2512 return (DDI_SUCCESS); 2513 } 2514 2515 2516 /* 2517 * rootnex_get_sgl() 2518 * Called in bind fastpath to get the sgl. Most of this will be replaced 2519 * with a call to the vm layer when vm2.0 comes around... 2520 */ 2521 static void 2522 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2523 rootnex_sglinfo_t *sglinfo) 2524 { 2525 ddi_dma_atyp_t buftype; 2526 rootnex_addr_t raddr; 2527 uint64_t last_page; 2528 uint64_t offset; 2529 uint64_t addrhi; 2530 uint64_t addrlo; 2531 uint64_t maxseg; 2532 page_t **pplist; 2533 uint64_t paddr; 2534 uint32_t psize; 2535 uint32_t size; 2536 caddr_t vaddr; 2537 uint_t pcnt; 2538 page_t *pp; 2539 uint_t cnt; 2540 2541 2542 /* shortcuts */ 2543 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2544 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2545 maxseg = sglinfo->si_max_cookie_size; 2546 buftype = dmar_object->dmao_type; 2547 addrhi = sglinfo->si_max_addr; 2548 addrlo = sglinfo->si_min_addr; 2549 size = dmar_object->dmao_size; 2550 2551 pcnt = 0; 2552 cnt = 0; 2553 2554 /* 2555 * if we were passed down a linked list of pages, i.e. pointer to 2556 * page_t, use this to get our physical address and buf offset. 2557 */ 2558 if (buftype == DMA_OTYP_PAGES) { 2559 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2560 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2561 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2562 MMU_PAGEOFFSET; 2563 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2564 psize = MIN(size, (MMU_PAGESIZE - offset)); 2565 pp = pp->p_next; 2566 sglinfo->si_asp = NULL; 2567 2568 /* 2569 * We weren't passed down a linked list of pages, but if we were passed 2570 * down an array of pages, use this to get our physical address and buf 2571 * offset. 2572 */ 2573 } else if (pplist != NULL) { 2574 ASSERT((buftype == DMA_OTYP_VADDR) || 2575 (buftype == DMA_OTYP_BUFVADDR)); 2576 2577 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2578 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2579 if (sglinfo->si_asp == NULL) { 2580 sglinfo->si_asp = &kas; 2581 } 2582 2583 ASSERT(!PP_ISFREE(pplist[pcnt])); 2584 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2585 paddr += offset; 2586 psize = MIN(size, (MMU_PAGESIZE - offset)); 2587 pcnt++; 2588 2589 /* 2590 * All we have is a virtual address, we'll need to call into the VM 2591 * to get the physical address. 2592 */ 2593 } else { 2594 ASSERT((buftype == DMA_OTYP_VADDR) || 2595 (buftype == DMA_OTYP_BUFVADDR)); 2596 2597 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2598 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2599 if (sglinfo->si_asp == NULL) { 2600 sglinfo->si_asp = &kas; 2601 } 2602 2603 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2604 paddr += offset; 2605 psize = MIN(size, (MMU_PAGESIZE - offset)); 2606 vaddr += psize; 2607 } 2608 2609 #ifdef __xpv 2610 /* 2611 * If we're dom0, we're using a real device so we need to load 2612 * the cookies with MFNs instead of PFNs. 2613 */ 2614 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2615 #else 2616 raddr = paddr; 2617 #endif 2618 2619 /* 2620 * Setup the first cookie with the physical address of the page and the 2621 * size of the page (which takes into account the initial offset into 2622 * the page. 2623 */ 2624 sgl[cnt].dmac_laddress = raddr; 2625 sgl[cnt].dmac_size = psize; 2626 sgl[cnt].dmac_type = 0; 2627 2628 /* 2629 * Save away the buffer offset into the page. We'll need this later in 2630 * the copy buffer code to help figure out the page index within the 2631 * buffer and the offset into the current page. 2632 */ 2633 sglinfo->si_buf_offset = offset; 2634 2635 /* 2636 * If the DMA engine can't reach the physical address, increase how 2637 * much copy buffer we need. We always increase by pagesize so we don't 2638 * have to worry about converting offsets. Set a flag in the cookies 2639 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2640 * last cookie, go to the next cookie (since we separate each page which 2641 * uses the copy buffer in case the copy buffer is not physically 2642 * contiguous. 2643 */ 2644 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2645 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2646 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2647 if ((cnt + 1) < sglinfo->si_max_pages) { 2648 cnt++; 2649 sgl[cnt].dmac_laddress = 0; 2650 sgl[cnt].dmac_size = 0; 2651 sgl[cnt].dmac_type = 0; 2652 } 2653 } 2654 2655 /* 2656 * save this page's physical address so we can figure out if the next 2657 * page is physically contiguous. Keep decrementing size until we are 2658 * done with the buffer. 2659 */ 2660 last_page = raddr & MMU_PAGEMASK; 2661 size -= psize; 2662 2663 while (size > 0) { 2664 /* Get the size for this page (i.e. partial or full page) */ 2665 psize = MIN(size, MMU_PAGESIZE); 2666 2667 if (buftype == DMA_OTYP_PAGES) { 2668 /* get the paddr from the page_t */ 2669 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2670 paddr = pfn_to_pa(pp->p_pagenum); 2671 pp = pp->p_next; 2672 } else if (pplist != NULL) { 2673 /* index into the array of page_t's to get the paddr */ 2674 ASSERT(!PP_ISFREE(pplist[pcnt])); 2675 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2676 pcnt++; 2677 } else { 2678 /* call into the VM to get the paddr */ 2679 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2680 vaddr)); 2681 vaddr += psize; 2682 } 2683 2684 #ifdef __xpv 2685 /* 2686 * If we're dom0, we're using a real device so we need to load 2687 * the cookies with MFNs instead of PFNs. 2688 */ 2689 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2690 #else 2691 raddr = paddr; 2692 #endif 2693 /* check to see if this page needs the copy buffer */ 2694 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2695 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2696 2697 /* 2698 * if there is something in the current cookie, go to 2699 * the next one. We only want one page in a cookie which 2700 * uses the copybuf since the copybuf doesn't have to 2701 * be physically contiguous. 2702 */ 2703 if (sgl[cnt].dmac_size != 0) { 2704 cnt++; 2705 } 2706 sgl[cnt].dmac_laddress = raddr; 2707 sgl[cnt].dmac_size = psize; 2708 #if defined(__amd64) 2709 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2710 #else 2711 /* 2712 * save the buf offset for 32-bit kernel. used in the 2713 * obsoleted interfaces. 2714 */ 2715 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2716 (dmar_object->dmao_size - size); 2717 #endif 2718 /* if this isn't the last cookie, go to the next one */ 2719 if ((cnt + 1) < sglinfo->si_max_pages) { 2720 cnt++; 2721 sgl[cnt].dmac_laddress = 0; 2722 sgl[cnt].dmac_size = 0; 2723 sgl[cnt].dmac_type = 0; 2724 } 2725 2726 /* 2727 * this page didn't need the copy buffer, if it's not physically 2728 * contiguous, or it would put us over a segment boundary, or it 2729 * puts us over the max cookie size, or the current sgl doesn't 2730 * have anything in it. 2731 */ 2732 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2733 !(raddr & sglinfo->si_segmask) || 2734 ((sgl[cnt].dmac_size + psize) > maxseg) || 2735 (sgl[cnt].dmac_size == 0)) { 2736 /* 2737 * if we're not already in a new cookie, go to the next 2738 * cookie. 2739 */ 2740 if (sgl[cnt].dmac_size != 0) { 2741 cnt++; 2742 } 2743 2744 /* save the cookie information */ 2745 sgl[cnt].dmac_laddress = raddr; 2746 sgl[cnt].dmac_size = psize; 2747 #if defined(__amd64) 2748 sgl[cnt].dmac_type = 0; 2749 #else 2750 /* 2751 * save the buf offset for 32-bit kernel. used in the 2752 * obsoleted interfaces. 2753 */ 2754 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2755 #endif 2756 2757 /* 2758 * this page didn't need the copy buffer, it is physically 2759 * contiguous with the last page, and it's <= the max cookie 2760 * size. 2761 */ 2762 } else { 2763 sgl[cnt].dmac_size += psize; 2764 2765 /* 2766 * if this exactly == the maximum cookie size, and 2767 * it isn't the last cookie, go to the next cookie. 2768 */ 2769 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2770 ((cnt + 1) < sglinfo->si_max_pages)) { 2771 cnt++; 2772 sgl[cnt].dmac_laddress = 0; 2773 sgl[cnt].dmac_size = 0; 2774 sgl[cnt].dmac_type = 0; 2775 } 2776 } 2777 2778 /* 2779 * save this page's physical address so we can figure out if the 2780 * next page is physically contiguous. Keep decrementing size 2781 * until we are done with the buffer. 2782 */ 2783 last_page = raddr; 2784 size -= psize; 2785 } 2786 2787 /* we're done, save away how many cookies the sgl has */ 2788 if (sgl[cnt].dmac_size == 0) { 2789 ASSERT(cnt < sglinfo->si_max_pages); 2790 sglinfo->si_sgl_size = cnt; 2791 } else { 2792 sglinfo->si_sgl_size = cnt + 1; 2793 } 2794 } 2795 2796 2797 /* 2798 * rootnex_bind_slowpath() 2799 * Call in the bind path if the calling driver can't use the sgl without 2800 * modifying it. We either need to use the copy buffer and/or we will end up 2801 * with a partial bind. 2802 */ 2803 static int 2804 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2805 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2806 { 2807 rootnex_sglinfo_t *sinfo; 2808 rootnex_window_t *window; 2809 ddi_dma_cookie_t *cookie; 2810 size_t copybuf_used; 2811 size_t dmac_size; 2812 boolean_t partial; 2813 off_t cur_offset; 2814 page_t *cur_pp; 2815 major_t mnum; 2816 int e; 2817 int i; 2818 2819 2820 sinfo = &dma->dp_sglinfo; 2821 copybuf_used = 0; 2822 partial = B_FALSE; 2823 2824 /* 2825 * If we're using the copybuf, set the copybuf state in dma struct. 2826 * Needs to be first since it sets the copy buffer size. 2827 */ 2828 if (sinfo->si_copybuf_req != 0) { 2829 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2830 if (e != DDI_SUCCESS) { 2831 return (e); 2832 } 2833 } else { 2834 dma->dp_copybuf_size = 0; 2835 } 2836 2837 /* 2838 * Figure out if we need to do a partial mapping. If so, figure out 2839 * if we need to trim the buffers when we munge the sgl. 2840 */ 2841 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2842 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2843 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2844 dma->dp_partial_required = B_TRUE; 2845 if (attr->dma_attr_granular != 1) { 2846 dma->dp_trim_required = B_TRUE; 2847 } 2848 } else { 2849 dma->dp_partial_required = B_FALSE; 2850 dma->dp_trim_required = B_FALSE; 2851 } 2852 2853 /* If we need to do a partial bind, make sure the driver supports it */ 2854 if (dma->dp_partial_required && 2855 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2856 2857 mnum = ddi_driver_major(dma->dp_dip); 2858 /* 2859 * patchable which allows us to print one warning per major 2860 * number. 2861 */ 2862 if ((rootnex_bind_warn) && 2863 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2864 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2865 cmn_err(CE_WARN, "!%s: coding error detected, the " 2866 "driver is using ddi_dma_attr(9S) incorrectly. " 2867 "There is a small risk of data corruption in " 2868 "particular with large I/Os. The driver should be " 2869 "replaced with a corrected version for proper " 2870 "system operation. To disable this warning, add " 2871 "'set rootnex:rootnex_bind_warn=0' to " 2872 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2873 } 2874 return (DDI_DMA_TOOBIG); 2875 } 2876 2877 /* 2878 * we might need multiple windows, setup state to handle them. In this 2879 * code path, we will have at least one window. 2880 */ 2881 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2882 if (e != DDI_SUCCESS) { 2883 rootnex_teardown_copybuf(dma); 2884 return (e); 2885 } 2886 2887 window = &dma->dp_window[0]; 2888 cookie = &dma->dp_cookies[0]; 2889 cur_offset = 0; 2890 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2891 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2892 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2893 } 2894 2895 /* loop though all the cookies we got back from get_sgl() */ 2896 for (i = 0; i < sinfo->si_sgl_size; i++) { 2897 /* 2898 * If we're using the copy buffer, check this cookie and setup 2899 * its associated copy buffer state. If this cookie uses the 2900 * copy buffer, make sure we sync this window during dma_sync. 2901 */ 2902 if (dma->dp_copybuf_size > 0) { 2903 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2904 cur_offset, ©buf_used, &cur_pp); 2905 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2906 window->wd_dosync = B_TRUE; 2907 } 2908 } 2909 2910 /* 2911 * save away the cookie size, since it could be modified in 2912 * the windowing code. 2913 */ 2914 dmac_size = cookie->dmac_size; 2915 2916 /* if we went over max copybuf size */ 2917 if (dma->dp_copybuf_size && 2918 (copybuf_used > dma->dp_copybuf_size)) { 2919 partial = B_TRUE; 2920 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2921 cookie, cur_offset, ©buf_used); 2922 if (e != DDI_SUCCESS) { 2923 rootnex_teardown_copybuf(dma); 2924 rootnex_teardown_windows(dma); 2925 return (e); 2926 } 2927 2928 /* 2929 * if the coookie uses the copy buffer, make sure the 2930 * new window we just moved to is set to sync. 2931 */ 2932 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2933 window->wd_dosync = B_TRUE; 2934 } 2935 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2936 dma->dp_dip); 2937 2938 /* if the cookie cnt == max sgllen, move to the next window */ 2939 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2940 partial = B_TRUE; 2941 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2942 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2943 cookie, attr, cur_offset); 2944 if (e != DDI_SUCCESS) { 2945 rootnex_teardown_copybuf(dma); 2946 rootnex_teardown_windows(dma); 2947 return (e); 2948 } 2949 2950 /* 2951 * if the coookie uses the copy buffer, make sure the 2952 * new window we just moved to is set to sync. 2953 */ 2954 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2955 window->wd_dosync = B_TRUE; 2956 } 2957 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2958 dma->dp_dip); 2959 2960 /* else if we will be over maxxfer */ 2961 } else if ((window->wd_size + dmac_size) > 2962 dma->dp_maxxfer) { 2963 partial = B_TRUE; 2964 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2965 cookie); 2966 if (e != DDI_SUCCESS) { 2967 rootnex_teardown_copybuf(dma); 2968 rootnex_teardown_windows(dma); 2969 return (e); 2970 } 2971 2972 /* 2973 * if the coookie uses the copy buffer, make sure the 2974 * new window we just moved to is set to sync. 2975 */ 2976 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2977 window->wd_dosync = B_TRUE; 2978 } 2979 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2980 dma->dp_dip); 2981 2982 /* else this cookie fits in the current window */ 2983 } else { 2984 window->wd_cookie_cnt++; 2985 window->wd_size += dmac_size; 2986 } 2987 2988 /* track our offset into the buffer, go to the next cookie */ 2989 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2990 ASSERT(cookie->dmac_size <= dmac_size); 2991 cur_offset += dmac_size; 2992 cookie++; 2993 } 2994 2995 /* if we ended up with a zero sized window in the end, clean it up */ 2996 if (window->wd_size == 0) { 2997 hp->dmai_nwin--; 2998 window--; 2999 } 3000 3001 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3002 3003 if (!partial) { 3004 return (DDI_DMA_MAPPED); 3005 } 3006 3007 ASSERT(dma->dp_partial_required); 3008 return (DDI_DMA_PARTIAL_MAP); 3009 } 3010 3011 3012 /* 3013 * rootnex_setup_copybuf() 3014 * Called in bind slowpath. Figures out if we're going to use the copy 3015 * buffer, and if we do, sets up the basic state to handle it. 3016 */ 3017 static int 3018 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3019 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3020 { 3021 rootnex_sglinfo_t *sinfo; 3022 ddi_dma_attr_t lattr; 3023 size_t max_copybuf; 3024 int cansleep; 3025 int e; 3026 #if !defined(__amd64) 3027 int vmflag; 3028 #endif 3029 3030 3031 sinfo = &dma->dp_sglinfo; 3032 3033 /* read this first so it's consistent through the routine */ 3034 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3035 3036 /* We need to call into the rootnex on ddi_dma_sync() */ 3037 hp->dmai_rflags &= ~DMP_NOSYNC; 3038 3039 /* make sure the copybuf size <= the max size */ 3040 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3041 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3042 3043 #if !defined(__amd64) 3044 /* 3045 * if we don't have kva space to copy to/from, allocate the KVA space 3046 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3047 * the 64-bit kernel. 3048 */ 3049 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3050 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3051 3052 /* convert the sleep flags */ 3053 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3054 vmflag = VM_SLEEP; 3055 } else { 3056 vmflag = VM_NOSLEEP; 3057 } 3058 3059 /* allocate Kernel VA space that we can bcopy to/from */ 3060 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3061 vmflag); 3062 if (dma->dp_kva == NULL) { 3063 return (DDI_DMA_NORESOURCES); 3064 } 3065 } 3066 #endif 3067 3068 /* convert the sleep flags */ 3069 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3070 cansleep = 1; 3071 } else { 3072 cansleep = 0; 3073 } 3074 3075 /* 3076 * Allocate the actual copy buffer. This needs to fit within the DMA 3077 * engine limits, so we can't use kmem_alloc... We don't need 3078 * contiguous memory (sgllen) since we will be forcing windows on 3079 * sgllen anyway. 3080 */ 3081 lattr = *attr; 3082 lattr.dma_attr_align = MMU_PAGESIZE; 3083 /* 3084 * this should be < 0 to indicate no limit, but due to a bug in 3085 * the rootnex, we'll set it to the maximum positive int. 3086 */ 3087 lattr.dma_attr_sgllen = 0x7fffffff; 3088 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3089 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3090 if (e != DDI_SUCCESS) { 3091 #if !defined(__amd64) 3092 if (dma->dp_kva != NULL) { 3093 vmem_free(heap_arena, dma->dp_kva, 3094 dma->dp_copybuf_size); 3095 } 3096 #endif 3097 return (DDI_DMA_NORESOURCES); 3098 } 3099 3100 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3101 size_t, dma->dp_copybuf_size); 3102 3103 return (DDI_SUCCESS); 3104 } 3105 3106 3107 /* 3108 * rootnex_setup_windows() 3109 * Called in bind slowpath to setup the window state. We always have windows 3110 * in the slowpath. Even if the window count = 1. 3111 */ 3112 static int 3113 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3114 ddi_dma_attr_t *attr, int kmflag) 3115 { 3116 rootnex_window_t *windowp; 3117 rootnex_sglinfo_t *sinfo; 3118 size_t copy_state_size; 3119 size_t win_state_size; 3120 size_t state_available; 3121 size_t space_needed; 3122 uint_t copybuf_win; 3123 uint_t maxxfer_win; 3124 size_t space_used; 3125 uint_t sglwin; 3126 3127 3128 sinfo = &dma->dp_sglinfo; 3129 3130 dma->dp_current_win = 0; 3131 hp->dmai_nwin = 0; 3132 3133 /* If we don't need to do a partial, we only have one window */ 3134 if (!dma->dp_partial_required) { 3135 dma->dp_max_win = 1; 3136 3137 /* 3138 * we need multiple windows, need to figure out the worse case number 3139 * of windows. 3140 */ 3141 } else { 3142 /* 3143 * if we need windows because we need more copy buffer that 3144 * we allow, the worse case number of windows we could need 3145 * here would be (copybuf space required / copybuf space that 3146 * we have) plus one for remainder, and plus 2 to handle the 3147 * extra pages on the trim for the first and last pages of the 3148 * buffer (a page is the minimum window size so under the right 3149 * attr settings, you could have a window for each page). 3150 * The last page will only be hit here if the size is not a 3151 * multiple of the granularity (which theoretically shouldn't 3152 * be the case but never has been enforced, so we could have 3153 * broken things without it). 3154 */ 3155 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3156 ASSERT(dma->dp_copybuf_size > 0); 3157 copybuf_win = (sinfo->si_copybuf_req / 3158 dma->dp_copybuf_size) + 1 + 2; 3159 } else { 3160 copybuf_win = 0; 3161 } 3162 3163 /* 3164 * if we need windows because we have more cookies than the H/W 3165 * can handle, the number of windows we would need here would 3166 * be (cookie count / cookies count H/W supports) plus one for 3167 * remainder, and plus 2 to handle the extra pages on the trim 3168 * (see above comment about trim) 3169 */ 3170 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3171 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 3172 + 1) + 2; 3173 } else { 3174 sglwin = 0; 3175 } 3176 3177 /* 3178 * if we need windows because we're binding more memory than the 3179 * H/W can transfer at once, the number of windows we would need 3180 * here would be (xfer count / max xfer H/W supports) plus one 3181 * for remainder, and plus 2 to handle the extra pages on the 3182 * trim (see above comment about trim) 3183 */ 3184 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3185 maxxfer_win = (dma->dp_dma.dmao_size / 3186 dma->dp_maxxfer) + 1 + 2; 3187 } else { 3188 maxxfer_win = 0; 3189 } 3190 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3191 ASSERT(dma->dp_max_win > 0); 3192 } 3193 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3194 3195 /* 3196 * Get space for window and potential copy buffer state. Before we 3197 * go and allocate memory, see if we can get away with using what's 3198 * left in the pre-allocted state or the dynamically allocated sgl. 3199 */ 3200 space_used = (uintptr_t)(sinfo->si_sgl_size * 3201 sizeof (ddi_dma_cookie_t)); 3202 3203 /* if we dynamically allocated space for the cookies */ 3204 if (dma->dp_need_to_free_cookie) { 3205 /* if we have more space in the pre-allocted buffer, use it */ 3206 ASSERT(space_used <= dma->dp_cookie_size); 3207 if ((dma->dp_cookie_size - space_used) <= 3208 rootnex_state->r_prealloc_size) { 3209 state_available = rootnex_state->r_prealloc_size; 3210 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3211 3212 /* 3213 * else, we have more free space in the dynamically allocated 3214 * buffer, i.e. the buffer wasn't worse case fragmented so we 3215 * didn't need a lot of cookies. 3216 */ 3217 } else { 3218 state_available = dma->dp_cookie_size - space_used; 3219 windowp = (rootnex_window_t *) 3220 &dma->dp_cookies[sinfo->si_sgl_size]; 3221 } 3222 3223 /* we used the pre-alloced buffer */ 3224 } else { 3225 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3226 state_available = rootnex_state->r_prealloc_size - space_used; 3227 windowp = (rootnex_window_t *) 3228 &dma->dp_cookies[sinfo->si_sgl_size]; 3229 } 3230 3231 /* 3232 * figure out how much state we need to track the copy buffer. Add an 3233 * addition 8 bytes for pointer alignemnt later. 3234 */ 3235 if (dma->dp_copybuf_size > 0) { 3236 copy_state_size = sinfo->si_max_pages * 3237 sizeof (rootnex_pgmap_t); 3238 } else { 3239 copy_state_size = 0; 3240 } 3241 /* add an additional 8 bytes for pointer alignment */ 3242 space_needed = win_state_size + copy_state_size + 0x8; 3243 3244 /* if we have enough space already, use it */ 3245 if (state_available >= space_needed) { 3246 dma->dp_window = windowp; 3247 dma->dp_need_to_free_window = B_FALSE; 3248 3249 /* not enough space, need to allocate more. */ 3250 } else { 3251 dma->dp_window = kmem_alloc(space_needed, kmflag); 3252 if (dma->dp_window == NULL) { 3253 return (DDI_DMA_NORESOURCES); 3254 } 3255 dma->dp_need_to_free_window = B_TRUE; 3256 dma->dp_window_size = space_needed; 3257 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3258 dma->dp_dip, size_t, space_needed); 3259 } 3260 3261 /* 3262 * we allocate copy buffer state and window state at the same time. 3263 * setup our copy buffer state pointers. Make sure it's aligned. 3264 */ 3265 if (dma->dp_copybuf_size > 0) { 3266 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3267 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3268 3269 #if !defined(__amd64) 3270 /* 3271 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3272 * false/NULL. Should be quicker to bzero vs loop and set. 3273 */ 3274 bzero(dma->dp_pgmap, copy_state_size); 3275 #endif 3276 } else { 3277 dma->dp_pgmap = NULL; 3278 } 3279 3280 return (DDI_SUCCESS); 3281 } 3282 3283 3284 /* 3285 * rootnex_teardown_copybuf() 3286 * cleans up after rootnex_setup_copybuf() 3287 */ 3288 static void 3289 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3290 { 3291 #if !defined(__amd64) 3292 int i; 3293 3294 /* 3295 * if we allocated kernel heap VMEM space, go through all the pages and 3296 * map out any of the ones that we're mapped into the kernel heap VMEM 3297 * arena. Then free the VMEM space. 3298 */ 3299 if (dma->dp_kva != NULL) { 3300 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3301 if (dma->dp_pgmap[i].pm_mapped) { 3302 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3303 MMU_PAGESIZE, HAT_UNLOAD); 3304 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3305 } 3306 } 3307 3308 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3309 } 3310 3311 #endif 3312 3313 /* if we allocated a copy buffer, free it */ 3314 if (dma->dp_cbaddr != NULL) { 3315 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3316 } 3317 } 3318 3319 3320 /* 3321 * rootnex_teardown_windows() 3322 * cleans up after rootnex_setup_windows() 3323 */ 3324 static void 3325 rootnex_teardown_windows(rootnex_dma_t *dma) 3326 { 3327 /* 3328 * if we had to allocate window state on the last bind (because we 3329 * didn't have enough pre-allocated space in the handle), free it. 3330 */ 3331 if (dma->dp_need_to_free_window) { 3332 kmem_free(dma->dp_window, dma->dp_window_size); 3333 } 3334 } 3335 3336 3337 /* 3338 * rootnex_init_win() 3339 * Called in bind slow path during creation of a new window. Initializes 3340 * window state to default values. 3341 */ 3342 /*ARGSUSED*/ 3343 static void 3344 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3345 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3346 { 3347 hp->dmai_nwin++; 3348 window->wd_dosync = B_FALSE; 3349 window->wd_offset = cur_offset; 3350 window->wd_size = 0; 3351 window->wd_first_cookie = cookie; 3352 window->wd_cookie_cnt = 0; 3353 window->wd_trim.tr_trim_first = B_FALSE; 3354 window->wd_trim.tr_trim_last = B_FALSE; 3355 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3356 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3357 #if !defined(__amd64) 3358 window->wd_remap_copybuf = dma->dp_cb_remaping; 3359 #endif 3360 } 3361 3362 3363 /* 3364 * rootnex_setup_cookie() 3365 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3366 * the sgl uses the copy buffer, we need to go through each cookie, figure 3367 * out if it uses the copy buffer, and if it does, save away everything we'll 3368 * need during sync. 3369 */ 3370 static void 3371 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3372 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3373 page_t **cur_pp) 3374 { 3375 boolean_t copybuf_sz_power_2; 3376 rootnex_sglinfo_t *sinfo; 3377 paddr_t paddr; 3378 uint_t pidx; 3379 uint_t pcnt; 3380 off_t poff; 3381 #if defined(__amd64) 3382 pfn_t pfn; 3383 #else 3384 page_t **pplist; 3385 #endif 3386 3387 sinfo = &dma->dp_sglinfo; 3388 3389 /* 3390 * Calculate the page index relative to the start of the buffer. The 3391 * index to the current page for our buffer is the offset into the 3392 * first page of the buffer plus our current offset into the buffer 3393 * itself, shifted of course... 3394 */ 3395 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3396 ASSERT(pidx < sinfo->si_max_pages); 3397 3398 /* if this cookie uses the copy buffer */ 3399 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3400 /* 3401 * NOTE: we know that since this cookie uses the copy buffer, it 3402 * is <= MMU_PAGESIZE. 3403 */ 3404 3405 /* 3406 * get the offset into the page. For the 64-bit kernel, get the 3407 * pfn which we'll use with seg kpm. 3408 */ 3409 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3410 #if defined(__amd64) 3411 /* mfn_to_pfn() is a NOP on i86pc */ 3412 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3413 #endif /* __amd64 */ 3414 3415 /* figure out if the copybuf size is a power of 2 */ 3416 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3417 copybuf_sz_power_2 = B_FALSE; 3418 } else { 3419 copybuf_sz_power_2 = B_TRUE; 3420 } 3421 3422 /* This page uses the copy buffer */ 3423 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3424 3425 /* 3426 * save the copy buffer KVA that we'll use with this page. 3427 * if we still fit within the copybuf, it's a simple add. 3428 * otherwise, we need to wrap over using & or % accordingly. 3429 */ 3430 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3431 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3432 *copybuf_used; 3433 } else { 3434 if (copybuf_sz_power_2) { 3435 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3436 (uintptr_t)dma->dp_cbaddr + 3437 (*copybuf_used & 3438 (dma->dp_copybuf_size - 1))); 3439 } else { 3440 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3441 (uintptr_t)dma->dp_cbaddr + 3442 (*copybuf_used % dma->dp_copybuf_size)); 3443 } 3444 } 3445 3446 /* 3447 * over write the cookie physical address with the address of 3448 * the physical address of the copy buffer page that we will 3449 * use. 3450 */ 3451 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3452 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3453 3454 #ifdef __xpv 3455 /* 3456 * If we're dom0, we're using a real device so we need to load 3457 * the cookies with MAs instead of PAs. 3458 */ 3459 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3460 #else 3461 cookie->dmac_laddress = paddr; 3462 #endif 3463 3464 /* if we have a kernel VA, it's easy, just save that address */ 3465 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3466 (sinfo->si_asp == &kas)) { 3467 /* 3468 * save away the page aligned virtual address of the 3469 * driver buffer. Offsets are handled in the sync code. 3470 */ 3471 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3472 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3473 & MMU_PAGEMASK); 3474 #if !defined(__amd64) 3475 /* 3476 * we didn't need to, and will never need to map this 3477 * page. 3478 */ 3479 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3480 #endif 3481 3482 /* we don't have a kernel VA. We need one for the bcopy. */ 3483 } else { 3484 #if defined(__amd64) 3485 /* 3486 * for the 64-bit kernel, it's easy. We use seg kpm to 3487 * get a Kernel VA for the corresponding pfn. 3488 */ 3489 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3490 #else 3491 /* 3492 * for the 32-bit kernel, this is a pain. First we'll 3493 * save away the page_t or user VA for this page. This 3494 * is needed in rootnex_dma_win() when we switch to a 3495 * new window which requires us to re-map the copy 3496 * buffer. 3497 */ 3498 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3499 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3500 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3501 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3502 } else if (pplist != NULL) { 3503 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3504 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3505 } else { 3506 dma->dp_pgmap[pidx].pm_pp = NULL; 3507 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3508 (((uintptr_t) 3509 dmar_object->dmao_obj.virt_obj.v_addr + 3510 cur_offset) & MMU_PAGEMASK); 3511 } 3512 3513 /* 3514 * save away the page aligned virtual address which was 3515 * allocated from the kernel heap arena (taking into 3516 * account if we need more copy buffer than we alloced 3517 * and use multiple windows to handle this, i.e. &,%). 3518 * NOTE: there isn't and physical memory backing up this 3519 * virtual address space currently. 3520 */ 3521 if ((*copybuf_used + MMU_PAGESIZE) <= 3522 dma->dp_copybuf_size) { 3523 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3524 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3525 MMU_PAGEMASK); 3526 } else { 3527 if (copybuf_sz_power_2) { 3528 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3529 (((uintptr_t)dma->dp_kva + 3530 (*copybuf_used & 3531 (dma->dp_copybuf_size - 1))) & 3532 MMU_PAGEMASK); 3533 } else { 3534 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3535 (((uintptr_t)dma->dp_kva + 3536 (*copybuf_used % 3537 dma->dp_copybuf_size)) & 3538 MMU_PAGEMASK); 3539 } 3540 } 3541 3542 /* 3543 * if we haven't used up the available copy buffer yet, 3544 * map the kva to the physical page. 3545 */ 3546 if (!dma->dp_cb_remaping && ((*copybuf_used + 3547 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3548 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3549 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3550 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3551 dma->dp_pgmap[pidx].pm_kaddr); 3552 } else { 3553 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3554 sinfo->si_asp, 3555 dma->dp_pgmap[pidx].pm_kaddr); 3556 } 3557 3558 /* 3559 * we've used up the available copy buffer, this page 3560 * will have to be mapped during rootnex_dma_win() when 3561 * we switch to a new window which requires a re-map 3562 * the copy buffer. (32-bit kernel only) 3563 */ 3564 } else { 3565 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3566 } 3567 #endif 3568 /* go to the next page_t */ 3569 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3570 *cur_pp = (*cur_pp)->p_next; 3571 } 3572 } 3573 3574 /* add to the copy buffer count */ 3575 *copybuf_used += MMU_PAGESIZE; 3576 3577 /* 3578 * This cookie doesn't use the copy buffer. Walk through the pages this 3579 * cookie occupies to reflect this. 3580 */ 3581 } else { 3582 /* 3583 * figure out how many pages the cookie occupies. We need to 3584 * use the original page offset of the buffer and the cookies 3585 * offset in the buffer to do this. 3586 */ 3587 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3588 pcnt = mmu_btopr(cookie->dmac_size + poff); 3589 3590 while (pcnt > 0) { 3591 #if !defined(__amd64) 3592 /* 3593 * the 32-bit kernel doesn't have seg kpm, so we need 3594 * to map in the driver buffer (if it didn't come down 3595 * with a kernel VA) on the fly. Since this page doesn't 3596 * use the copy buffer, it's not, or will it ever, have 3597 * to be mapped in. 3598 */ 3599 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3600 #endif 3601 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3602 3603 /* 3604 * we need to update pidx and cur_pp or we'll loose 3605 * track of where we are. 3606 */ 3607 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3608 *cur_pp = (*cur_pp)->p_next; 3609 } 3610 pidx++; 3611 pcnt--; 3612 } 3613 } 3614 } 3615 3616 3617 /* 3618 * rootnex_sgllen_window_boundary() 3619 * Called in the bind slow path when the next cookie causes us to exceed (in 3620 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3621 * length supported by the DMA H/W. 3622 */ 3623 static int 3624 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3625 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3626 off_t cur_offset) 3627 { 3628 off_t new_offset; 3629 size_t trim_sz; 3630 off_t coffset; 3631 3632 3633 /* 3634 * if we know we'll never have to trim, it's pretty easy. Just move to 3635 * the next window and init it. We're done. 3636 */ 3637 if (!dma->dp_trim_required) { 3638 (*windowp)++; 3639 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3640 (*windowp)->wd_cookie_cnt++; 3641 (*windowp)->wd_size = cookie->dmac_size; 3642 return (DDI_SUCCESS); 3643 } 3644 3645 /* figure out how much we need to trim from the window */ 3646 ASSERT(attr->dma_attr_granular != 0); 3647 if (dma->dp_granularity_power_2) { 3648 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3649 } else { 3650 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3651 } 3652 3653 /* The window's a whole multiple of granularity. We're done */ 3654 if (trim_sz == 0) { 3655 (*windowp)++; 3656 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3657 (*windowp)->wd_cookie_cnt++; 3658 (*windowp)->wd_size = cookie->dmac_size; 3659 return (DDI_SUCCESS); 3660 } 3661 3662 /* 3663 * The window's not a whole multiple of granularity, since we know this 3664 * is due to the sgllen, we need to go back to the last cookie and trim 3665 * that one, add the left over part of the old cookie into the new 3666 * window, and then add in the new cookie into the new window. 3667 */ 3668 3669 /* 3670 * make sure the driver isn't making us do something bad... Trimming and 3671 * sgllen == 1 don't go together. 3672 */ 3673 if (attr->dma_attr_sgllen == 1) { 3674 return (DDI_DMA_NOMAPPING); 3675 } 3676 3677 /* 3678 * first, setup the current window to account for the trim. Need to go 3679 * back to the last cookie for this. 3680 */ 3681 cookie--; 3682 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3683 (*windowp)->wd_trim.tr_last_cookie = cookie; 3684 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3685 ASSERT(cookie->dmac_size > trim_sz); 3686 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3687 (*windowp)->wd_size -= trim_sz; 3688 3689 /* save the buffer offsets for the next window */ 3690 coffset = cookie->dmac_size - trim_sz; 3691 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3692 3693 /* 3694 * set this now in case this is the first window. all other cases are 3695 * set in dma_win() 3696 */ 3697 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3698 3699 /* 3700 * initialize the next window using what's left over in the previous 3701 * cookie. 3702 */ 3703 (*windowp)++; 3704 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3705 (*windowp)->wd_cookie_cnt++; 3706 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3707 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3708 (*windowp)->wd_trim.tr_first_size = trim_sz; 3709 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3710 (*windowp)->wd_dosync = B_TRUE; 3711 } 3712 3713 /* 3714 * now go back to the current cookie and add it to the new window. set 3715 * the new window size to the what was left over from the previous 3716 * cookie and what's in the current cookie. 3717 */ 3718 cookie++; 3719 (*windowp)->wd_cookie_cnt++; 3720 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3721 3722 /* 3723 * trim plus the next cookie could put us over maxxfer (a cookie can be 3724 * a max size of maxxfer). Handle that case. 3725 */ 3726 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3727 /* 3728 * maxxfer is already a whole multiple of granularity, and this 3729 * trim will be <= the previous trim (since a cookie can't be 3730 * larger than maxxfer). Make things simple here. 3731 */ 3732 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3733 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3734 (*windowp)->wd_trim.tr_last_cookie = cookie; 3735 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3736 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3737 (*windowp)->wd_size -= trim_sz; 3738 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3739 3740 /* save the buffer offsets for the next window */ 3741 coffset = cookie->dmac_size - trim_sz; 3742 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3743 3744 /* setup the next window */ 3745 (*windowp)++; 3746 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3747 (*windowp)->wd_cookie_cnt++; 3748 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3749 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3750 coffset; 3751 (*windowp)->wd_trim.tr_first_size = trim_sz; 3752 } 3753 3754 return (DDI_SUCCESS); 3755 } 3756 3757 3758 /* 3759 * rootnex_copybuf_window_boundary() 3760 * Called in bind slowpath when we get to a window boundary because we used 3761 * up all the copy buffer that we have. 3762 */ 3763 static int 3764 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3765 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3766 size_t *copybuf_used) 3767 { 3768 rootnex_sglinfo_t *sinfo; 3769 off_t new_offset; 3770 size_t trim_sz; 3771 paddr_t paddr; 3772 off_t coffset; 3773 uint_t pidx; 3774 off_t poff; 3775 3776 3777 sinfo = &dma->dp_sglinfo; 3778 3779 /* 3780 * the copy buffer should be a whole multiple of page size. We know that 3781 * this cookie is <= MMU_PAGESIZE. 3782 */ 3783 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3784 3785 /* 3786 * from now on, all new windows in this bind need to be re-mapped during 3787 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3788 * space... 3789 */ 3790 #if !defined(__amd64) 3791 dma->dp_cb_remaping = B_TRUE; 3792 #endif 3793 3794 /* reset copybuf used */ 3795 *copybuf_used = 0; 3796 3797 /* 3798 * if we don't have to trim (since granularity is set to 1), go to the 3799 * next window and add the current cookie to it. We know the current 3800 * cookie uses the copy buffer since we're in this code path. 3801 */ 3802 if (!dma->dp_trim_required) { 3803 (*windowp)++; 3804 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3805 3806 /* Add this cookie to the new window */ 3807 (*windowp)->wd_cookie_cnt++; 3808 (*windowp)->wd_size += cookie->dmac_size; 3809 *copybuf_used += MMU_PAGESIZE; 3810 return (DDI_SUCCESS); 3811 } 3812 3813 /* 3814 * *** may need to trim, figure it out. 3815 */ 3816 3817 /* figure out how much we need to trim from the window */ 3818 if (dma->dp_granularity_power_2) { 3819 trim_sz = (*windowp)->wd_size & 3820 (hp->dmai_attr.dma_attr_granular - 1); 3821 } else { 3822 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3823 } 3824 3825 /* 3826 * if the window's a whole multiple of granularity, go to the next 3827 * window, init it, then add in the current cookie. We know the current 3828 * cookie uses the copy buffer since we're in this code path. 3829 */ 3830 if (trim_sz == 0) { 3831 (*windowp)++; 3832 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3833 3834 /* Add this cookie to the new window */ 3835 (*windowp)->wd_cookie_cnt++; 3836 (*windowp)->wd_size += cookie->dmac_size; 3837 *copybuf_used += MMU_PAGESIZE; 3838 return (DDI_SUCCESS); 3839 } 3840 3841 /* 3842 * *** We figured it out, we definitly need to trim 3843 */ 3844 3845 /* 3846 * make sure the driver isn't making us do something bad... 3847 * Trimming and sgllen == 1 don't go together. 3848 */ 3849 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3850 return (DDI_DMA_NOMAPPING); 3851 } 3852 3853 /* 3854 * first, setup the current window to account for the trim. Need to go 3855 * back to the last cookie for this. Some of the last cookie will be in 3856 * the current window, and some of the last cookie will be in the new 3857 * window. All of the current cookie will be in the new window. 3858 */ 3859 cookie--; 3860 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3861 (*windowp)->wd_trim.tr_last_cookie = cookie; 3862 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3863 ASSERT(cookie->dmac_size > trim_sz); 3864 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3865 (*windowp)->wd_size -= trim_sz; 3866 3867 /* 3868 * we're trimming the last cookie (not the current cookie). So that 3869 * last cookie may have or may not have been using the copy buffer ( 3870 * we know the cookie passed in uses the copy buffer since we're in 3871 * this code path). 3872 * 3873 * If the last cookie doesn't use the copy buffer, nothing special to 3874 * do. However, if it does uses the copy buffer, it will be both the 3875 * last page in the current window and the first page in the next 3876 * window. Since we are reusing the copy buffer (and KVA space on the 3877 * 32-bit kernel), this page will use the end of the copy buffer in the 3878 * current window, and the start of the copy buffer in the next window. 3879 * Track that info... The cookie physical address was already set to 3880 * the copy buffer physical address in setup_cookie.. 3881 */ 3882 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3883 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3884 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3885 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3886 (*windowp)->wd_trim.tr_last_pidx = pidx; 3887 (*windowp)->wd_trim.tr_last_cbaddr = 3888 dma->dp_pgmap[pidx].pm_cbaddr; 3889 #if !defined(__amd64) 3890 (*windowp)->wd_trim.tr_last_kaddr = 3891 dma->dp_pgmap[pidx].pm_kaddr; 3892 #endif 3893 } 3894 3895 /* save the buffer offsets for the next window */ 3896 coffset = cookie->dmac_size - trim_sz; 3897 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3898 3899 /* 3900 * set this now in case this is the first window. all other cases are 3901 * set in dma_win() 3902 */ 3903 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3904 3905 /* 3906 * initialize the next window using what's left over in the previous 3907 * cookie. 3908 */ 3909 (*windowp)++; 3910 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3911 (*windowp)->wd_cookie_cnt++; 3912 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3913 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3914 (*windowp)->wd_trim.tr_first_size = trim_sz; 3915 3916 /* 3917 * again, we're tracking if the last cookie uses the copy buffer. 3918 * read the comment above for more info on why we need to track 3919 * additional state. 3920 * 3921 * For the first cookie in the new window, we need reset the physical 3922 * address to DMA into to the start of the copy buffer plus any 3923 * initial page offset which may be present. 3924 */ 3925 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3926 (*windowp)->wd_dosync = B_TRUE; 3927 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3928 (*windowp)->wd_trim.tr_first_pidx = pidx; 3929 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3930 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3931 3932 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3933 poff; 3934 #ifdef __xpv 3935 /* 3936 * If we're dom0, we're using a real device so we need to load 3937 * the cookies with MAs instead of PAs. 3938 */ 3939 (*windowp)->wd_trim.tr_first_paddr = 3940 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3941 #else 3942 (*windowp)->wd_trim.tr_first_paddr = paddr; 3943 #endif 3944 3945 #if !defined(__amd64) 3946 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3947 #endif 3948 /* account for the cookie copybuf usage in the new window */ 3949 *copybuf_used += MMU_PAGESIZE; 3950 3951 /* 3952 * every piece of code has to have a hack, and here is this 3953 * ones :-) 3954 * 3955 * There is a complex interaction between setup_cookie and the 3956 * copybuf window boundary. The complexity had to be in either 3957 * the maxxfer window, or the copybuf window, and I chose the 3958 * copybuf code. 3959 * 3960 * So in this code path, we have taken the last cookie, 3961 * virtually broken it in half due to the trim, and it happens 3962 * to use the copybuf which further complicates life. At the 3963 * same time, we have already setup the current cookie, which 3964 * is now wrong. More background info: the current cookie uses 3965 * the copybuf, so it is only a page long max. So we need to 3966 * fix the current cookies copy buffer address, physical 3967 * address, and kva for the 32-bit kernel. We due this by 3968 * bumping them by page size (of course, we can't due this on 3969 * the physical address since the copy buffer may not be 3970 * physically contiguous). 3971 */ 3972 cookie++; 3973 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3974 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3975 3976 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3977 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3978 #ifdef __xpv 3979 /* 3980 * If we're dom0, we're using a real device so we need to load 3981 * the cookies with MAs instead of PAs. 3982 */ 3983 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3984 #else 3985 cookie->dmac_laddress = paddr; 3986 #endif 3987 3988 #if !defined(__amd64) 3989 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3990 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3991 #endif 3992 } else { 3993 /* go back to the current cookie */ 3994 cookie++; 3995 } 3996 3997 /* 3998 * add the current cookie to the new window. set the new window size to 3999 * the what was left over from the previous cookie and what's in the 4000 * current cookie. 4001 */ 4002 (*windowp)->wd_cookie_cnt++; 4003 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4004 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4005 4006 /* 4007 * we know that the cookie passed in always uses the copy buffer. We 4008 * wouldn't be here if it didn't. 4009 */ 4010 *copybuf_used += MMU_PAGESIZE; 4011 4012 return (DDI_SUCCESS); 4013 } 4014 4015 4016 /* 4017 * rootnex_maxxfer_window_boundary() 4018 * Called in bind slowpath when we get to a window boundary because we will 4019 * go over maxxfer. 4020 */ 4021 static int 4022 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4023 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4024 { 4025 size_t dmac_size; 4026 off_t new_offset; 4027 size_t trim_sz; 4028 off_t coffset; 4029 4030 4031 /* 4032 * calculate how much we have to trim off of the current cookie to equal 4033 * maxxfer. We don't have to account for granularity here since our 4034 * maxxfer already takes that into account. 4035 */ 4036 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4037 ASSERT(trim_sz <= cookie->dmac_size); 4038 ASSERT(trim_sz <= dma->dp_maxxfer); 4039 4040 /* save cookie size since we need it later and we might change it */ 4041 dmac_size = cookie->dmac_size; 4042 4043 /* 4044 * if we're not trimming the entire cookie, setup the current window to 4045 * account for the trim. 4046 */ 4047 if (trim_sz < cookie->dmac_size) { 4048 (*windowp)->wd_cookie_cnt++; 4049 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4050 (*windowp)->wd_trim.tr_last_cookie = cookie; 4051 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4052 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4053 (*windowp)->wd_size = dma->dp_maxxfer; 4054 4055 /* 4056 * set the adjusted cookie size now in case this is the first 4057 * window. All other windows are taken care of in get win 4058 */ 4059 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4060 } 4061 4062 /* 4063 * coffset is the current offset within the cookie, new_offset is the 4064 * current offset with the entire buffer. 4065 */ 4066 coffset = dmac_size - trim_sz; 4067 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4068 4069 /* initialize the next window */ 4070 (*windowp)++; 4071 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4072 (*windowp)->wd_cookie_cnt++; 4073 (*windowp)->wd_size = trim_sz; 4074 if (trim_sz < dmac_size) { 4075 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4076 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4077 coffset; 4078 (*windowp)->wd_trim.tr_first_size = trim_sz; 4079 } 4080 4081 return (DDI_SUCCESS); 4082 } 4083 4084 4085 /*ARGSUSED*/ 4086 static int 4087 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4088 off_t off, size_t len, uint_t cache_flags) 4089 { 4090 rootnex_sglinfo_t *sinfo; 4091 rootnex_pgmap_t *cbpage; 4092 rootnex_window_t *win; 4093 ddi_dma_impl_t *hp; 4094 rootnex_dma_t *dma; 4095 caddr_t fromaddr; 4096 caddr_t toaddr; 4097 uint_t psize; 4098 off_t offset; 4099 uint_t pidx; 4100 size_t size; 4101 off_t poff; 4102 int e; 4103 4104 4105 hp = (ddi_dma_impl_t *)handle; 4106 dma = (rootnex_dma_t *)hp->dmai_private; 4107 sinfo = &dma->dp_sglinfo; 4108 4109 /* 4110 * if we don't have any windows, we don't need to sync. A copybuf 4111 * will cause us to have at least one window. 4112 */ 4113 if (dma->dp_window == NULL) { 4114 return (DDI_SUCCESS); 4115 } 4116 4117 /* This window may not need to be sync'd */ 4118 win = &dma->dp_window[dma->dp_current_win]; 4119 if (!win->wd_dosync) { 4120 return (DDI_SUCCESS); 4121 } 4122 4123 /* handle off and len special cases */ 4124 if ((off == 0) || (rootnex_sync_ignore_params)) { 4125 offset = win->wd_offset; 4126 } else { 4127 offset = off; 4128 } 4129 if ((len == 0) || (rootnex_sync_ignore_params)) { 4130 size = win->wd_size; 4131 } else { 4132 size = len; 4133 } 4134 4135 /* check the sync args to make sure they make a little sense */ 4136 if (rootnex_sync_check_parms) { 4137 e = rootnex_valid_sync_parms(hp, win, offset, size, 4138 cache_flags); 4139 if (e != DDI_SUCCESS) { 4140 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4141 return (DDI_FAILURE); 4142 } 4143 } 4144 4145 /* 4146 * special case the first page to handle the offset into the page. The 4147 * offset to the current page for our buffer is the offset into the 4148 * first page of the buffer plus our current offset into the buffer 4149 * itself, masked of course. 4150 */ 4151 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4152 psize = MIN((MMU_PAGESIZE - poff), size); 4153 4154 /* go through all the pages that we want to sync */ 4155 while (size > 0) { 4156 /* 4157 * Calculate the page index relative to the start of the buffer. 4158 * The index to the current page for our buffer is the offset 4159 * into the first page of the buffer plus our current offset 4160 * into the buffer itself, shifted of course... 4161 */ 4162 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4163 ASSERT(pidx < sinfo->si_max_pages); 4164 4165 /* 4166 * if this page uses the copy buffer, we need to sync it, 4167 * otherwise, go on to the next page. 4168 */ 4169 cbpage = &dma->dp_pgmap[pidx]; 4170 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4171 (cbpage->pm_uses_copybuf == B_FALSE)); 4172 if (cbpage->pm_uses_copybuf) { 4173 /* cbaddr and kaddr should be page aligned */ 4174 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4175 MMU_PAGEOFFSET) == 0); 4176 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4177 MMU_PAGEOFFSET) == 0); 4178 4179 /* 4180 * if we're copying for the device, we are going to 4181 * copy from the drivers buffer and to the rootnex 4182 * allocated copy buffer. 4183 */ 4184 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4185 fromaddr = cbpage->pm_kaddr + poff; 4186 toaddr = cbpage->pm_cbaddr + poff; 4187 DTRACE_PROBE2(rootnex__sync__dev, 4188 dev_info_t *, dma->dp_dip, size_t, psize); 4189 4190 /* 4191 * if we're copying for the cpu/kernel, we are going to 4192 * copy from the rootnex allocated copy buffer to the 4193 * drivers buffer. 4194 */ 4195 } else { 4196 fromaddr = cbpage->pm_cbaddr + poff; 4197 toaddr = cbpage->pm_kaddr + poff; 4198 DTRACE_PROBE2(rootnex__sync__cpu, 4199 dev_info_t *, dma->dp_dip, size_t, psize); 4200 } 4201 4202 bcopy(fromaddr, toaddr, psize); 4203 } 4204 4205 /* 4206 * decrement size until we're done, update our offset into the 4207 * buffer, and get the next page size. 4208 */ 4209 size -= psize; 4210 offset += psize; 4211 psize = MIN(MMU_PAGESIZE, size); 4212 4213 /* page offset is zero for the rest of this loop */ 4214 poff = 0; 4215 } 4216 4217 return (DDI_SUCCESS); 4218 } 4219 4220 /* 4221 * rootnex_dma_sync() 4222 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4223 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4224 * is set, ddi_dma_sync() returns immediately passing back success. 4225 */ 4226 /*ARGSUSED*/ 4227 static int 4228 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4229 off_t off, size_t len, uint_t cache_flags) 4230 { 4231 #if !defined(__xpv) 4232 if (IOMMU_USED(rdip)) { 4233 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4234 cache_flags)); 4235 } 4236 #endif 4237 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4238 cache_flags)); 4239 } 4240 4241 /* 4242 * rootnex_valid_sync_parms() 4243 * checks the parameters passed to sync to verify they are correct. 4244 */ 4245 static int 4246 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4247 off_t offset, size_t size, uint_t cache_flags) 4248 { 4249 off_t woffset; 4250 4251 4252 /* 4253 * the first part of the test to make sure the offset passed in is 4254 * within the window. 4255 */ 4256 if (offset < win->wd_offset) { 4257 return (DDI_FAILURE); 4258 } 4259 4260 /* 4261 * second and last part of the test to make sure the offset and length 4262 * passed in is within the window. 4263 */ 4264 woffset = offset - win->wd_offset; 4265 if ((woffset + size) > win->wd_size) { 4266 return (DDI_FAILURE); 4267 } 4268 4269 /* 4270 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4271 * be set too. 4272 */ 4273 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4274 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4275 return (DDI_SUCCESS); 4276 } 4277 4278 /* 4279 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4280 * should be set. Also DDI_DMA_READ should be set in the flags. 4281 */ 4282 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4283 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4284 (hp->dmai_rflags & DDI_DMA_READ)) { 4285 return (DDI_SUCCESS); 4286 } 4287 4288 return (DDI_FAILURE); 4289 } 4290 4291 4292 /*ARGSUSED*/ 4293 static int 4294 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4295 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4296 uint_t *ccountp) 4297 { 4298 rootnex_window_t *window; 4299 rootnex_trim_t *trim; 4300 ddi_dma_impl_t *hp; 4301 rootnex_dma_t *dma; 4302 #if !defined(__amd64) 4303 rootnex_sglinfo_t *sinfo; 4304 rootnex_pgmap_t *pmap; 4305 uint_t pidx; 4306 uint_t pcnt; 4307 off_t poff; 4308 int i; 4309 #endif 4310 4311 4312 hp = (ddi_dma_impl_t *)handle; 4313 dma = (rootnex_dma_t *)hp->dmai_private; 4314 #if !defined(__amd64) 4315 sinfo = &dma->dp_sglinfo; 4316 #endif 4317 4318 /* If we try and get a window which doesn't exist, return failure */ 4319 if (win >= hp->dmai_nwin) { 4320 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4321 return (DDI_FAILURE); 4322 } 4323 4324 /* 4325 * if we don't have any windows, and they're asking for the first 4326 * window, setup the cookie pointer to the first cookie in the bind. 4327 * setup our return values, then increment the cookie since we return 4328 * the first cookie on the stack. 4329 */ 4330 if (dma->dp_window == NULL) { 4331 if (win != 0) { 4332 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4333 return (DDI_FAILURE); 4334 } 4335 hp->dmai_cookie = dma->dp_cookies; 4336 *offp = 0; 4337 *lenp = dma->dp_dma.dmao_size; 4338 *ccountp = dma->dp_sglinfo.si_sgl_size; 4339 *cookiep = hp->dmai_cookie[0]; 4340 hp->dmai_cookie++; 4341 return (DDI_SUCCESS); 4342 } 4343 4344 /* sync the old window before moving on to the new one */ 4345 window = &dma->dp_window[dma->dp_current_win]; 4346 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4347 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4348 DDI_DMA_SYNC_FORCPU); 4349 } 4350 4351 #if !defined(__amd64) 4352 /* 4353 * before we move to the next window, if we need to re-map, unmap all 4354 * the pages in this window. 4355 */ 4356 if (dma->dp_cb_remaping) { 4357 /* 4358 * If we switch to this window again, we'll need to map in 4359 * on the fly next time. 4360 */ 4361 window->wd_remap_copybuf = B_TRUE; 4362 4363 /* 4364 * calculate the page index into the buffer where this window 4365 * starts, and the number of pages this window takes up. 4366 */ 4367 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4368 MMU_PAGESHIFT; 4369 poff = (sinfo->si_buf_offset + window->wd_offset) & 4370 MMU_PAGEOFFSET; 4371 pcnt = mmu_btopr(window->wd_size + poff); 4372 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4373 4374 /* unmap pages which are currently mapped in this window */ 4375 for (i = 0; i < pcnt; i++) { 4376 if (dma->dp_pgmap[pidx].pm_mapped) { 4377 hat_unload(kas.a_hat, 4378 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4379 HAT_UNLOAD); 4380 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4381 } 4382 pidx++; 4383 } 4384 } 4385 #endif 4386 4387 /* 4388 * Move to the new window. 4389 * NOTE: current_win must be set for sync to work right 4390 */ 4391 dma->dp_current_win = win; 4392 window = &dma->dp_window[win]; 4393 4394 /* if needed, adjust the first and/or last cookies for trim */ 4395 trim = &window->wd_trim; 4396 if (trim->tr_trim_first) { 4397 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4398 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4399 #if !defined(__amd64) 4400 window->wd_first_cookie->dmac_type = 4401 (window->wd_first_cookie->dmac_type & 4402 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4403 #endif 4404 if (trim->tr_first_copybuf_win) { 4405 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4406 trim->tr_first_cbaddr; 4407 #if !defined(__amd64) 4408 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4409 trim->tr_first_kaddr; 4410 #endif 4411 } 4412 } 4413 if (trim->tr_trim_last) { 4414 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4415 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4416 if (trim->tr_last_copybuf_win) { 4417 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4418 trim->tr_last_cbaddr; 4419 #if !defined(__amd64) 4420 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4421 trim->tr_last_kaddr; 4422 #endif 4423 } 4424 } 4425 4426 /* 4427 * setup the cookie pointer to the first cookie in the window. setup 4428 * our return values, then increment the cookie since we return the 4429 * first cookie on the stack. 4430 */ 4431 hp->dmai_cookie = window->wd_first_cookie; 4432 *offp = window->wd_offset; 4433 *lenp = window->wd_size; 4434 *ccountp = window->wd_cookie_cnt; 4435 *cookiep = hp->dmai_cookie[0]; 4436 hp->dmai_cookie++; 4437 4438 #if !defined(__amd64) 4439 /* re-map copybuf if required for this window */ 4440 if (dma->dp_cb_remaping) { 4441 /* 4442 * calculate the page index into the buffer where this 4443 * window starts. 4444 */ 4445 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4446 MMU_PAGESHIFT; 4447 ASSERT(pidx < sinfo->si_max_pages); 4448 4449 /* 4450 * the first page can get unmapped if it's shared with the 4451 * previous window. Even if the rest of this window is already 4452 * mapped in, we need to still check this one. 4453 */ 4454 pmap = &dma->dp_pgmap[pidx]; 4455 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4456 if (pmap->pm_pp != NULL) { 4457 pmap->pm_mapped = B_TRUE; 4458 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4459 } else if (pmap->pm_vaddr != NULL) { 4460 pmap->pm_mapped = B_TRUE; 4461 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4462 pmap->pm_kaddr); 4463 } 4464 } 4465 pidx++; 4466 4467 /* map in the rest of the pages if required */ 4468 if (window->wd_remap_copybuf) { 4469 window->wd_remap_copybuf = B_FALSE; 4470 4471 /* figure out many pages this window takes up */ 4472 poff = (sinfo->si_buf_offset + window->wd_offset) & 4473 MMU_PAGEOFFSET; 4474 pcnt = mmu_btopr(window->wd_size + poff); 4475 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4476 4477 /* map pages which require it */ 4478 for (i = 1; i < pcnt; i++) { 4479 pmap = &dma->dp_pgmap[pidx]; 4480 if (pmap->pm_uses_copybuf) { 4481 ASSERT(pmap->pm_mapped == B_FALSE); 4482 if (pmap->pm_pp != NULL) { 4483 pmap->pm_mapped = B_TRUE; 4484 i86_pp_map(pmap->pm_pp, 4485 pmap->pm_kaddr); 4486 } else if (pmap->pm_vaddr != NULL) { 4487 pmap->pm_mapped = B_TRUE; 4488 i86_va_map(pmap->pm_vaddr, 4489 sinfo->si_asp, 4490 pmap->pm_kaddr); 4491 } 4492 } 4493 pidx++; 4494 } 4495 } 4496 } 4497 #endif 4498 4499 /* if the new window uses the copy buffer, sync it for the device */ 4500 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4501 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4502 DDI_DMA_SYNC_FORDEV); 4503 } 4504 4505 return (DDI_SUCCESS); 4506 } 4507 4508 /* 4509 * rootnex_dma_win() 4510 * called from ddi_dma_getwin() 4511 */ 4512 /*ARGSUSED*/ 4513 static int 4514 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4515 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4516 uint_t *ccountp) 4517 { 4518 #if !defined(__xpv) 4519 if (IOMMU_USED(rdip)) { 4520 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4521 cookiep, ccountp)); 4522 } 4523 #endif 4524 4525 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4526 cookiep, ccountp)); 4527 } 4528 4529 /* 4530 * ************************ 4531 * obsoleted dma routines 4532 * ************************ 4533 */ 4534 4535 /* 4536 * rootnex_dma_map() 4537 * called from ddi_dma_setup() 4538 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 4539 */ 4540 /* ARGSUSED */ 4541 static int 4542 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4543 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4544 { 4545 #if defined(__amd64) 4546 /* 4547 * this interface is not supported in 64-bit x86 kernel. See comment in 4548 * rootnex_dma_mctl() 4549 */ 4550 return (DDI_DMA_NORESOURCES); 4551 4552 #else /* 32-bit x86 kernel */ 4553 ddi_dma_handle_t *lhandlep; 4554 ddi_dma_handle_t lhandle; 4555 ddi_dma_cookie_t cookie; 4556 ddi_dma_attr_t dma_attr; 4557 ddi_dma_lim_t *dma_lim; 4558 uint_t ccnt; 4559 int e; 4560 4561 4562 /* 4563 * if the driver is just testing to see if it's possible to do the bind, 4564 * we'll use local state. Otherwise, use the handle pointer passed in. 4565 */ 4566 if (handlep == NULL) { 4567 lhandlep = &lhandle; 4568 } else { 4569 lhandlep = handlep; 4570 } 4571 4572 /* convert the limit structure to a dma_attr one */ 4573 dma_lim = dmareq->dmar_limits; 4574 dma_attr.dma_attr_version = DMA_ATTR_V0; 4575 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4576 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4577 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4578 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4579 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4580 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4581 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4582 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4583 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4584 dma_attr.dma_attr_align = MMU_PAGESIZE; 4585 dma_attr.dma_attr_flags = 0; 4586 4587 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4588 dmareq->dmar_arg, lhandlep); 4589 if (e != DDI_SUCCESS) { 4590 return (e); 4591 } 4592 4593 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4594 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4595 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4596 return (e); 4597 } 4598 4599 /* 4600 * if the driver is just testing to see if it's possible to do the bind, 4601 * free up the local state and return the result. 4602 */ 4603 if (handlep == NULL) { 4604 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4605 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4606 if (e == DDI_DMA_MAPPED) { 4607 return (DDI_DMA_MAPOK); 4608 } else { 4609 return (DDI_DMA_NOMAPPING); 4610 } 4611 } 4612 4613 return (e); 4614 #endif /* defined(__amd64) */ 4615 } 4616 4617 /* 4618 * rootnex_dma_mctl() 4619 * 4620 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 4621 */ 4622 /* ARGSUSED */ 4623 static int 4624 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4625 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4626 uint_t cache_flags) 4627 { 4628 #if defined(__amd64) 4629 /* 4630 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4631 * common implementation in genunix, so they no longer have x86 4632 * specific functionality which called into dma_ctl. 4633 * 4634 * The rest of the obsoleted interfaces were never supported in the 4635 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4636 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4637 * implementation issues. 4638 * 4639 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4640 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4641 * reflect that now too... 4642 * 4643 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4644 * not going to put this functionality into the 64-bit x86 kernel now. 4645 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4646 * that in a future release. 4647 */ 4648 return (DDI_FAILURE); 4649 4650 #else /* 32-bit x86 kernel */ 4651 ddi_dma_cookie_t lcookie; 4652 ddi_dma_cookie_t *cookie; 4653 rootnex_window_t *window; 4654 ddi_dma_impl_t *hp; 4655 rootnex_dma_t *dma; 4656 uint_t nwin; 4657 uint_t ccnt; 4658 size_t len; 4659 off_t off; 4660 int e; 4661 4662 4663 /* 4664 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4665 * hacky since were optimizing for the current interfaces and so we can 4666 * cleanup the mess in genunix. Hopefully we will remove the this 4667 * obsoleted routines someday soon. 4668 */ 4669 4670 switch (request) { 4671 4672 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4673 hp = (ddi_dma_impl_t *)handle; 4674 cookie = (ddi_dma_cookie_t *)objpp; 4675 4676 /* 4677 * convert segment to cookie. We don't distinguish between the 4678 * two :-) 4679 */ 4680 *cookie = *hp->dmai_cookie; 4681 *lenp = cookie->dmac_size; 4682 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4683 return (DDI_SUCCESS); 4684 4685 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4686 hp = (ddi_dma_impl_t *)handle; 4687 dma = (rootnex_dma_t *)hp->dmai_private; 4688 4689 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4690 return (DDI_DMA_STALE); 4691 } 4692 4693 /* handle the case where we don't have any windows */ 4694 if (dma->dp_window == NULL) { 4695 /* 4696 * if seg == NULL, and we don't have any windows, 4697 * return the first cookie in the sgl. 4698 */ 4699 if (*lenp == NULL) { 4700 dma->dp_current_cookie = 0; 4701 hp->dmai_cookie = dma->dp_cookies; 4702 *objpp = (caddr_t)handle; 4703 return (DDI_SUCCESS); 4704 4705 /* if we have more cookies, go to the next cookie */ 4706 } else { 4707 if ((dma->dp_current_cookie + 1) >= 4708 dma->dp_sglinfo.si_sgl_size) { 4709 return (DDI_DMA_DONE); 4710 } 4711 dma->dp_current_cookie++; 4712 hp->dmai_cookie++; 4713 return (DDI_SUCCESS); 4714 } 4715 } 4716 4717 /* We have one or more windows */ 4718 window = &dma->dp_window[dma->dp_current_win]; 4719 4720 /* 4721 * if seg == NULL, return the first cookie in the current 4722 * window 4723 */ 4724 if (*lenp == NULL) { 4725 dma->dp_current_cookie = 0; 4726 hp->dmai_cookie = window->wd_first_cookie; 4727 4728 /* 4729 * go to the next cookie in the window then see if we done with 4730 * this window. 4731 */ 4732 } else { 4733 if ((dma->dp_current_cookie + 1) >= 4734 window->wd_cookie_cnt) { 4735 return (DDI_DMA_DONE); 4736 } 4737 dma->dp_current_cookie++; 4738 hp->dmai_cookie++; 4739 } 4740 *objpp = (caddr_t)handle; 4741 return (DDI_SUCCESS); 4742 4743 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4744 hp = (ddi_dma_impl_t *)handle; 4745 dma = (rootnex_dma_t *)hp->dmai_private; 4746 4747 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4748 return (DDI_DMA_STALE); 4749 } 4750 4751 /* if win == NULL, return the first window in the bind */ 4752 if (*offp == NULL) { 4753 nwin = 0; 4754 4755 /* 4756 * else, go to the next window then see if we're done with all 4757 * the windows. 4758 */ 4759 } else { 4760 nwin = dma->dp_current_win + 1; 4761 if (nwin >= hp->dmai_nwin) { 4762 return (DDI_DMA_DONE); 4763 } 4764 } 4765 4766 /* switch to the next window */ 4767 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4768 &lcookie, &ccnt); 4769 ASSERT(e == DDI_SUCCESS); 4770 if (e != DDI_SUCCESS) { 4771 return (DDI_DMA_STALE); 4772 } 4773 4774 /* reset the cookie back to the first cookie in the window */ 4775 if (dma->dp_window != NULL) { 4776 window = &dma->dp_window[dma->dp_current_win]; 4777 hp->dmai_cookie = window->wd_first_cookie; 4778 } else { 4779 hp->dmai_cookie = dma->dp_cookies; 4780 } 4781 4782 *objpp = (caddr_t)handle; 4783 return (DDI_SUCCESS); 4784 4785 case DDI_DMA_FREE: /* ddi_dma_free() */ 4786 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4787 (void) rootnex_dma_freehdl(dip, rdip, handle); 4788 if (rootnex_state->r_dvma_call_list_id) { 4789 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4790 } 4791 return (DDI_SUCCESS); 4792 4793 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4794 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4795 /* should never get here, handled in genunix */ 4796 ASSERT(0); 4797 return (DDI_FAILURE); 4798 4799 case DDI_DMA_KVADDR: 4800 case DDI_DMA_GETERR: 4801 case DDI_DMA_COFF: 4802 return (DDI_FAILURE); 4803 } 4804 4805 return (DDI_FAILURE); 4806 #endif /* defined(__amd64) */ 4807 } 4808 4809 /* 4810 * ********* 4811 * FMA Code 4812 * ********* 4813 */ 4814 4815 /* 4816 * rootnex_fm_init() 4817 * FMA init busop 4818 */ 4819 /* ARGSUSED */ 4820 static int 4821 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4822 ddi_iblock_cookie_t *ibc) 4823 { 4824 *ibc = rootnex_state->r_err_ibc; 4825 4826 return (ddi_system_fmcap); 4827 } 4828 4829 /* 4830 * rootnex_dma_check() 4831 * Function called after a dma fault occurred to find out whether the 4832 * fault address is associated with a driver that is able to handle faults 4833 * and recover from faults. 4834 */ 4835 /* ARGSUSED */ 4836 static int 4837 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4838 const void *not_used) 4839 { 4840 rootnex_window_t *window; 4841 uint64_t start_addr; 4842 uint64_t fault_addr; 4843 ddi_dma_impl_t *hp; 4844 rootnex_dma_t *dma; 4845 uint64_t end_addr; 4846 size_t csize; 4847 int i; 4848 int j; 4849 4850 4851 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4852 hp = (ddi_dma_impl_t *)handle; 4853 ASSERT(hp); 4854 4855 dma = (rootnex_dma_t *)hp->dmai_private; 4856 4857 /* Get the address that we need to search for */ 4858 fault_addr = *(uint64_t *)addr; 4859 4860 /* 4861 * if we don't have any windows, we can just walk through all the 4862 * cookies. 4863 */ 4864 if (dma->dp_window == NULL) { 4865 /* for each cookie */ 4866 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4867 /* 4868 * if the faulted address is within the physical address 4869 * range of the cookie, return DDI_FM_NONFATAL. 4870 */ 4871 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4872 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4873 dma->dp_cookies[i].dmac_size))) { 4874 return (DDI_FM_NONFATAL); 4875 } 4876 } 4877 4878 /* fault_addr not within this DMA handle */ 4879 return (DDI_FM_UNKNOWN); 4880 } 4881 4882 /* we have mutiple windows, walk through each window */ 4883 for (i = 0; i < hp->dmai_nwin; i++) { 4884 window = &dma->dp_window[i]; 4885 4886 /* Go through all the cookies in the window */ 4887 for (j = 0; j < window->wd_cookie_cnt; j++) { 4888 4889 start_addr = window->wd_first_cookie[j].dmac_laddress; 4890 csize = window->wd_first_cookie[j].dmac_size; 4891 4892 /* 4893 * if we are trimming the first cookie in the window, 4894 * and this is the first cookie, adjust the start 4895 * address and size of the cookie to account for the 4896 * trim. 4897 */ 4898 if (window->wd_trim.tr_trim_first && (j == 0)) { 4899 start_addr = window->wd_trim.tr_first_paddr; 4900 csize = window->wd_trim.tr_first_size; 4901 } 4902 4903 /* 4904 * if we are trimming the last cookie in the window, 4905 * and this is the last cookie, adjust the start 4906 * address and size of the cookie to account for the 4907 * trim. 4908 */ 4909 if (window->wd_trim.tr_trim_last && 4910 (j == (window->wd_cookie_cnt - 1))) { 4911 start_addr = window->wd_trim.tr_last_paddr; 4912 csize = window->wd_trim.tr_last_size; 4913 } 4914 4915 end_addr = start_addr + csize; 4916 4917 /* 4918 * if the faulted address is within the physical address 4919 * range of the cookie, return DDI_FM_NONFATAL. 4920 */ 4921 if ((fault_addr >= start_addr) && 4922 (fault_addr <= end_addr)) { 4923 return (DDI_FM_NONFATAL); 4924 } 4925 } 4926 } 4927 4928 /* fault_addr not within this DMA handle */ 4929 return (DDI_FM_UNKNOWN); 4930 } 4931