1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #endif 71 72 #if defined(__amd64) && !defined(__xpv) 73 #include <sys/immu.h> 74 #endif 75 76 77 /* 78 * enable/disable extra checking of function parameters. Useful for debugging 79 * drivers. 80 */ 81 #ifdef DEBUG 82 int rootnex_alloc_check_parms = 1; 83 int rootnex_bind_check_parms = 1; 84 int rootnex_bind_check_inuse = 1; 85 int rootnex_unbind_verify_buffer = 0; 86 int rootnex_sync_check_parms = 1; 87 #else 88 int rootnex_alloc_check_parms = 0; 89 int rootnex_bind_check_parms = 0; 90 int rootnex_bind_check_inuse = 0; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 0; 93 #endif 94 95 boolean_t rootnex_dmar_not_setup; 96 97 /* Master Abort and Target Abort panic flag */ 98 int rootnex_fm_ma_ta_panic_flag = 0; 99 100 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 101 int rootnex_bind_fail = 1; 102 int rootnex_bind_warn = 1; 103 uint8_t *rootnex_warn_list; 104 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 105 #define ROOTNEX_BIND_WARNING (0x1 << 0) 106 107 /* 108 * revert back to old broken behavior of always sync'ing entire copy buffer. 109 * This is useful if be have a buggy driver which doesn't correctly pass in 110 * the offset and size into ddi_dma_sync(). 111 */ 112 int rootnex_sync_ignore_params = 0; 113 114 /* 115 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 116 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 117 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 118 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 119 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 120 * (< 8K). We will still need to allocate the copy buffer during bind though 121 * (if we need one). These can only be modified in /etc/system before rootnex 122 * attach. 123 */ 124 #if defined(__amd64) 125 int rootnex_prealloc_cookies = 65; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #else 129 int rootnex_prealloc_cookies = 33; 130 int rootnex_prealloc_windows = 4; 131 int rootnex_prealloc_copybuf = 2; 132 #endif 133 134 /* driver global state */ 135 static rootnex_state_t *rootnex_state; 136 137 /* shortcut to rootnex counters */ 138 static uint64_t *rootnex_cnt; 139 140 /* 141 * XXX - does x86 even need these or are they left over from the SPARC days? 142 */ 143 /* statically defined integer/boolean properties for the root node */ 144 static rootnex_intprop_t rootnex_intprp[] = { 145 { "PAGESIZE", PAGESIZE }, 146 { "MMU_PAGESIZE", MMU_PAGESIZE }, 147 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 148 { DDI_RELATIVE_ADDRESSING, 1 }, 149 }; 150 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 151 152 #ifdef __xpv 153 typedef maddr_t rootnex_addr_t; 154 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 155 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 156 #else 157 typedef paddr_t rootnex_addr_t; 158 #endif 159 160 #if !defined(__xpv) 161 char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica"; 162 #endif 163 164 static struct cb_ops rootnex_cb_ops = { 165 nodev, /* open */ 166 nodev, /* close */ 167 nodev, /* strategy */ 168 nodev, /* print */ 169 nodev, /* dump */ 170 nodev, /* read */ 171 nodev, /* write */ 172 nodev, /* ioctl */ 173 nodev, /* devmap */ 174 nodev, /* mmap */ 175 nodev, /* segmap */ 176 nochpoll, /* chpoll */ 177 ddi_prop_op, /* cb_prop_op */ 178 NULL, /* struct streamtab */ 179 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 180 CB_REV, /* Rev */ 181 nodev, /* cb_aread */ 182 nodev /* cb_awrite */ 183 }; 184 185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 186 off_t offset, off_t len, caddr_t *vaddrp); 187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 188 struct hat *hat, struct seg *seg, caddr_t addr, 189 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 190 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 191 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 192 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 193 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 194 ddi_dma_handle_t *handlep); 195 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 196 ddi_dma_handle_t handle); 197 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 198 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 199 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 200 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle); 202 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 204 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 207 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 209 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 210 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 211 ddi_ctl_enum_t ctlop, void *arg, void *result); 212 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 213 ddi_iblock_cookie_t *ibc); 214 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 215 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 216 217 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 218 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 219 ddi_dma_handle_t *handlep); 220 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 221 ddi_dma_handle_t handle); 222 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 223 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 224 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 225 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 226 ddi_dma_handle_t handle); 227 #if defined(__amd64) && !defined(__xpv) 228 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 229 ddi_dma_handle_t handle); 230 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 231 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 232 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 233 ddi_dma_cookie_t *cookiep, uint_t ccount); 234 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 235 ddi_dma_handle_t handle); 236 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 237 #endif 238 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 239 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 240 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 241 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 242 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 243 244 static struct bus_ops rootnex_bus_ops = { 245 BUSO_REV, 246 rootnex_map, 247 NULL, 248 NULL, 249 NULL, 250 rootnex_map_fault, 251 rootnex_dma_map, 252 rootnex_dma_allochdl, 253 rootnex_dma_freehdl, 254 rootnex_dma_bindhdl, 255 rootnex_dma_unbindhdl, 256 rootnex_dma_sync, 257 rootnex_dma_win, 258 rootnex_dma_mctl, 259 rootnex_ctlops, 260 ddi_bus_prop_op, 261 i_ddi_rootnex_get_eventcookie, 262 i_ddi_rootnex_add_eventcall, 263 i_ddi_rootnex_remove_eventcall, 264 i_ddi_rootnex_post_event, 265 0, /* bus_intr_ctl */ 266 0, /* bus_config */ 267 0, /* bus_unconfig */ 268 rootnex_fm_init, /* bus_fm_init */ 269 NULL, /* bus_fm_fini */ 270 NULL, /* bus_fm_access_enter */ 271 NULL, /* bus_fm_access_exit */ 272 NULL, /* bus_powr */ 273 rootnex_intr_ops /* bus_intr_op */ 274 }; 275 276 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 277 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 278 static int rootnex_quiesce(dev_info_t *dip); 279 280 static struct dev_ops rootnex_ops = { 281 DEVO_REV, 282 0, 283 ddi_no_info, 284 nulldev, 285 nulldev, 286 rootnex_attach, 287 rootnex_detach, 288 nulldev, 289 &rootnex_cb_ops, 290 &rootnex_bus_ops, 291 NULL, 292 rootnex_quiesce, /* quiesce */ 293 }; 294 295 static struct modldrv rootnex_modldrv = { 296 &mod_driverops, 297 "i86pc root nexus", 298 &rootnex_ops 299 }; 300 301 static struct modlinkage rootnex_modlinkage = { 302 MODREV_1, 303 (void *)&rootnex_modldrv, 304 NULL 305 }; 306 307 #if defined(__amd64) && !defined(__xpv) 308 static iommulib_nexops_t iommulib_nexops = { 309 IOMMU_NEXOPS_VERSION, 310 "Rootnex IOMMU ops Vers 1.1", 311 NULL, 312 rootnex_coredma_allochdl, 313 rootnex_coredma_freehdl, 314 rootnex_coredma_bindhdl, 315 rootnex_coredma_unbindhdl, 316 rootnex_coredma_reset_cookies, 317 rootnex_coredma_get_cookies, 318 rootnex_coredma_set_cookies, 319 rootnex_coredma_clear_cookies, 320 rootnex_coredma_get_sleep_flags, 321 rootnex_coredma_sync, 322 rootnex_coredma_win, 323 rootnex_dma_map, 324 rootnex_dma_mctl 325 }; 326 #endif 327 328 /* 329 * extern hacks 330 */ 331 extern struct seg_ops segdev_ops; 332 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 333 #ifdef DDI_MAP_DEBUG 334 extern int ddi_map_debug_flag; 335 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 336 #endif 337 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 338 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 339 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 340 psm_intr_op_t, int *); 341 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 342 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 343 344 /* 345 * Use device arena to use for device control register mappings. 346 * Various kernel memory walkers (debugger, dtrace) need to know 347 * to avoid this address range to prevent undesired device activity. 348 */ 349 extern void *device_arena_alloc(size_t size, int vm_flag); 350 extern void device_arena_free(void * vaddr, size_t size); 351 352 353 /* 354 * Internal functions 355 */ 356 static int rootnex_dma_init(); 357 static void rootnex_add_props(dev_info_t *); 358 static int rootnex_ctl_reportdev(dev_info_t *dip); 359 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 360 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 361 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 362 static int rootnex_map_handle(ddi_map_req_t *mp); 363 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 364 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 365 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 366 ddi_dma_attr_t *attr); 367 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 368 rootnex_sglinfo_t *sglinfo); 369 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 370 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 371 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 372 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 373 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 374 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 375 ddi_dma_attr_t *attr, int kmflag); 376 static void rootnex_teardown_windows(rootnex_dma_t *dma); 377 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 378 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 379 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 380 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 381 size_t *copybuf_used, page_t **cur_pp); 382 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 383 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 384 ddi_dma_attr_t *attr, off_t cur_offset); 385 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 386 rootnex_dma_t *dma, rootnex_window_t **windowp, 387 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 388 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 389 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 390 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 391 off_t offset, size_t size, uint_t cache_flags); 392 static int rootnex_verify_buffer(rootnex_dma_t *dma); 393 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 394 const void *comp_addr, const void *not_used); 395 396 /* 397 * _init() 398 * 399 */ 400 int 401 _init(void) 402 { 403 404 rootnex_state = NULL; 405 return (mod_install(&rootnex_modlinkage)); 406 } 407 408 409 /* 410 * _info() 411 * 412 */ 413 int 414 _info(struct modinfo *modinfop) 415 { 416 return (mod_info(&rootnex_modlinkage, modinfop)); 417 } 418 419 420 /* 421 * _fini() 422 * 423 */ 424 int 425 _fini(void) 426 { 427 return (EBUSY); 428 } 429 430 431 /* 432 * rootnex_attach() 433 * 434 */ 435 static int 436 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 437 { 438 int fmcap; 439 int e; 440 441 switch (cmd) { 442 case DDI_ATTACH: 443 break; 444 case DDI_RESUME: 445 #if defined(__amd64) && !defined(__xpv) 446 return (immu_unquiesce()); 447 #else 448 return (DDI_SUCCESS); 449 #endif 450 default: 451 return (DDI_FAILURE); 452 } 453 454 /* 455 * We should only have one instance of rootnex. Save it away since we 456 * don't have an easy way to get it back later. 457 */ 458 ASSERT(rootnex_state == NULL); 459 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 460 461 rootnex_state->r_dip = dip; 462 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 463 rootnex_state->r_reserved_msg_printed = B_FALSE; 464 rootnex_cnt = &rootnex_state->r_counters[0]; 465 466 /* 467 * Set minimum fm capability level for i86pc platforms and then 468 * initialize error handling. Since we're the rootnex, we don't 469 * care what's returned in the fmcap field. 470 */ 471 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 472 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 473 fmcap = ddi_system_fmcap; 474 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 475 476 /* initialize DMA related state */ 477 e = rootnex_dma_init(); 478 if (e != DDI_SUCCESS) { 479 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 480 return (DDI_FAILURE); 481 } 482 483 /* Add static root node properties */ 484 rootnex_add_props(dip); 485 486 /* since we can't call ddi_report_dev() */ 487 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 488 489 /* Initialize rootnex event handle */ 490 i_ddi_rootnex_init_events(dip); 491 492 #if defined(__amd64) && !defined(__xpv) 493 e = iommulib_nexus_register(dip, &iommulib_nexops, 494 &rootnex_state->r_iommulib_handle); 495 496 ASSERT(e == DDI_SUCCESS); 497 #endif 498 499 return (DDI_SUCCESS); 500 } 501 502 503 /* 504 * rootnex_detach() 505 * 506 */ 507 /*ARGSUSED*/ 508 static int 509 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 510 { 511 switch (cmd) { 512 case DDI_SUSPEND: 513 #if defined(__amd64) && !defined(__xpv) 514 return (immu_quiesce()); 515 #else 516 return (DDI_SUCCESS); 517 #endif 518 default: 519 return (DDI_FAILURE); 520 } 521 /*NOTREACHED*/ 522 523 } 524 525 526 /* 527 * rootnex_dma_init() 528 * 529 */ 530 /*ARGSUSED*/ 531 static int 532 rootnex_dma_init() 533 { 534 size_t bufsize; 535 536 537 /* 538 * size of our cookie/window/copybuf state needed in dma bind that we 539 * pre-alloc in dma_alloc_handle 540 */ 541 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 542 rootnex_state->r_prealloc_size = 543 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 544 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 545 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 546 547 /* 548 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 549 * allocate 16 extra bytes for struct pointer alignment 550 * (p->dmai_private & dma->dp_prealloc_buffer) 551 */ 552 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 553 rootnex_state->r_prealloc_size + 0x10; 554 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 555 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 556 if (rootnex_state->r_dmahdl_cache == NULL) { 557 return (DDI_FAILURE); 558 } 559 560 /* 561 * allocate array to track which major numbers we have printed warnings 562 * for. 563 */ 564 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 565 KM_SLEEP); 566 567 return (DDI_SUCCESS); 568 } 569 570 571 /* 572 * rootnex_add_props() 573 * 574 */ 575 static void 576 rootnex_add_props(dev_info_t *dip) 577 { 578 rootnex_intprop_t *rpp; 579 int i; 580 581 /* Add static integer/boolean properties to the root node */ 582 rpp = rootnex_intprp; 583 for (i = 0; i < NROOT_INTPROPS; i++) { 584 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 585 rpp[i].prop_name, rpp[i].prop_value); 586 } 587 } 588 589 590 591 /* 592 * ************************* 593 * ctlops related routines 594 * ************************* 595 */ 596 597 /* 598 * rootnex_ctlops() 599 * 600 */ 601 /*ARGSUSED*/ 602 static int 603 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 604 void *arg, void *result) 605 { 606 int n, *ptr; 607 struct ddi_parent_private_data *pdp; 608 609 switch (ctlop) { 610 case DDI_CTLOPS_DMAPMAPC: 611 /* 612 * Return 'partial' to indicate that dma mapping 613 * has to be done in the main MMU. 614 */ 615 return (DDI_DMA_PARTIAL); 616 617 case DDI_CTLOPS_BTOP: 618 /* 619 * Convert byte count input to physical page units. 620 * (byte counts that are not a page-size multiple 621 * are rounded down) 622 */ 623 *(ulong_t *)result = btop(*(ulong_t *)arg); 624 return (DDI_SUCCESS); 625 626 case DDI_CTLOPS_PTOB: 627 /* 628 * Convert size in physical pages to bytes 629 */ 630 *(ulong_t *)result = ptob(*(ulong_t *)arg); 631 return (DDI_SUCCESS); 632 633 case DDI_CTLOPS_BTOPR: 634 /* 635 * Convert byte count input to physical page units 636 * (byte counts that are not a page-size multiple 637 * are rounded up) 638 */ 639 *(ulong_t *)result = btopr(*(ulong_t *)arg); 640 return (DDI_SUCCESS); 641 642 case DDI_CTLOPS_INITCHILD: 643 return (impl_ddi_sunbus_initchild(arg)); 644 645 case DDI_CTLOPS_UNINITCHILD: 646 impl_ddi_sunbus_removechild(arg); 647 return (DDI_SUCCESS); 648 649 case DDI_CTLOPS_REPORTDEV: 650 return (rootnex_ctl_reportdev(rdip)); 651 652 case DDI_CTLOPS_IOMIN: 653 /* 654 * Nothing to do here but reflect back.. 655 */ 656 return (DDI_SUCCESS); 657 658 case DDI_CTLOPS_REGSIZE: 659 case DDI_CTLOPS_NREGS: 660 break; 661 662 case DDI_CTLOPS_SIDDEV: 663 if (ndi_dev_is_prom_node(rdip)) 664 return (DDI_SUCCESS); 665 if (ndi_dev_is_persistent_node(rdip)) 666 return (DDI_SUCCESS); 667 return (DDI_FAILURE); 668 669 case DDI_CTLOPS_POWER: 670 return ((*pm_platform_power)((power_req_t *)arg)); 671 672 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 673 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 674 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 675 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 676 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 677 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 678 if (!rootnex_state->r_reserved_msg_printed) { 679 rootnex_state->r_reserved_msg_printed = B_TRUE; 680 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 681 "1 or more reserved/obsolete operations."); 682 } 683 return (DDI_FAILURE); 684 685 default: 686 return (DDI_FAILURE); 687 } 688 /* 689 * The rest are for "hardware" properties 690 */ 691 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 692 return (DDI_FAILURE); 693 694 if (ctlop == DDI_CTLOPS_NREGS) { 695 ptr = (int *)result; 696 *ptr = pdp->par_nreg; 697 } else { 698 off_t *size = (off_t *)result; 699 700 ptr = (int *)arg; 701 n = *ptr; 702 if (n >= pdp->par_nreg) { 703 return (DDI_FAILURE); 704 } 705 *size = (off_t)pdp->par_reg[n].regspec_size; 706 } 707 return (DDI_SUCCESS); 708 } 709 710 711 /* 712 * rootnex_ctl_reportdev() 713 * 714 */ 715 static int 716 rootnex_ctl_reportdev(dev_info_t *dev) 717 { 718 int i, n, len, f_len = 0; 719 char *buf; 720 721 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 722 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 723 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 724 len = strlen(buf); 725 726 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 727 728 struct regspec *rp = sparc_pd_getreg(dev, i); 729 730 if (i == 0) 731 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 732 ": "); 733 else 734 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 735 " and "); 736 len = strlen(buf); 737 738 switch (rp->regspec_bustype) { 739 740 case BTEISA: 741 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 742 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 743 break; 744 745 case BTISA: 746 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 747 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 748 break; 749 750 default: 751 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 752 "space %x offset %x", 753 rp->regspec_bustype, rp->regspec_addr); 754 break; 755 } 756 len = strlen(buf); 757 } 758 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 759 int pri; 760 761 if (i != 0) { 762 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 763 ","); 764 len = strlen(buf); 765 } 766 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 767 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 768 " sparc ipl %d", pri); 769 len = strlen(buf); 770 } 771 #ifdef DEBUG 772 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 773 cmn_err(CE_NOTE, "next message is truncated: " 774 "printed length 1024, real length %d", f_len); 775 } 776 #endif /* DEBUG */ 777 cmn_err(CE_CONT, "?%s\n", buf); 778 kmem_free(buf, REPORTDEV_BUFSIZE); 779 return (DDI_SUCCESS); 780 } 781 782 783 /* 784 * ****************** 785 * map related code 786 * ****************** 787 */ 788 789 /* 790 * rootnex_map() 791 * 792 */ 793 static int 794 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 795 off_t len, caddr_t *vaddrp) 796 { 797 struct regspec *rp, tmp_reg; 798 ddi_map_req_t mr = *mp; /* Get private copy of request */ 799 int error; 800 801 mp = &mr; 802 803 switch (mp->map_op) { 804 case DDI_MO_MAP_LOCKED: 805 case DDI_MO_UNMAP: 806 case DDI_MO_MAP_HANDLE: 807 break; 808 default: 809 #ifdef DDI_MAP_DEBUG 810 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 811 mp->map_op); 812 #endif /* DDI_MAP_DEBUG */ 813 return (DDI_ME_UNIMPLEMENTED); 814 } 815 816 if (mp->map_flags & DDI_MF_USER_MAPPING) { 817 #ifdef DDI_MAP_DEBUG 818 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 819 #endif /* DDI_MAP_DEBUG */ 820 return (DDI_ME_UNIMPLEMENTED); 821 } 822 823 /* 824 * First, if given an rnumber, convert it to a regspec... 825 * (Presumably, this is on behalf of a child of the root node?) 826 */ 827 828 if (mp->map_type == DDI_MT_RNUMBER) { 829 830 int rnumber = mp->map_obj.rnumber; 831 #ifdef DDI_MAP_DEBUG 832 static char *out_of_range = 833 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 834 #endif /* DDI_MAP_DEBUG */ 835 836 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 837 if (rp == NULL) { 838 #ifdef DDI_MAP_DEBUG 839 cmn_err(CE_WARN, out_of_range, rnumber, 840 ddi_get_name(rdip)); 841 #endif /* DDI_MAP_DEBUG */ 842 return (DDI_ME_RNUMBER_RANGE); 843 } 844 845 /* 846 * Convert the given ddi_map_req_t from rnumber to regspec... 847 */ 848 849 mp->map_type = DDI_MT_REGSPEC; 850 mp->map_obj.rp = rp; 851 } 852 853 /* 854 * Adjust offset and length correspnding to called values... 855 * XXX: A non-zero length means override the one in the regspec 856 * XXX: (regardless of what's in the parent's range?) 857 */ 858 859 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 860 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 861 862 #ifdef DDI_MAP_DEBUG 863 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 864 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 865 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 866 len, mp->map_handlep); 867 #endif /* DDI_MAP_DEBUG */ 868 869 /* 870 * I/O or memory mapping: 871 * 872 * <bustype=0, addr=x, len=x>: memory 873 * <bustype=1, addr=x, len=x>: i/o 874 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 875 */ 876 877 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 878 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 879 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 880 ddi_get_name(rdip), rp->regspec_bustype, 881 rp->regspec_addr, rp->regspec_size); 882 return (DDI_ME_INVAL); 883 } 884 885 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 886 /* 887 * compatibility i/o mapping 888 */ 889 rp->regspec_bustype += (uint_t)offset; 890 } else { 891 /* 892 * Normal memory or i/o mapping 893 */ 894 rp->regspec_addr += (uint_t)offset; 895 } 896 897 if (len != 0) 898 rp->regspec_size = (uint_t)len; 899 900 #ifdef DDI_MAP_DEBUG 901 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 902 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 903 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 904 offset, len, mp->map_handlep); 905 #endif /* DDI_MAP_DEBUG */ 906 907 /* 908 * Apply any parent ranges at this level, if applicable. 909 * (This is where nexus specific regspec translation takes place. 910 * Use of this function is implicit agreement that translation is 911 * provided via ddi_apply_range.) 912 */ 913 914 #ifdef DDI_MAP_DEBUG 915 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 916 ddi_get_name(dip), ddi_get_name(rdip)); 917 #endif /* DDI_MAP_DEBUG */ 918 919 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 920 return (error); 921 922 switch (mp->map_op) { 923 case DDI_MO_MAP_LOCKED: 924 925 /* 926 * Set up the locked down kernel mapping to the regspec... 927 */ 928 929 return (rootnex_map_regspec(mp, vaddrp)); 930 931 case DDI_MO_UNMAP: 932 933 /* 934 * Release mapping... 935 */ 936 937 return (rootnex_unmap_regspec(mp, vaddrp)); 938 939 case DDI_MO_MAP_HANDLE: 940 941 return (rootnex_map_handle(mp)); 942 943 default: 944 return (DDI_ME_UNIMPLEMENTED); 945 } 946 } 947 948 949 /* 950 * rootnex_map_fault() 951 * 952 * fault in mappings for requestors 953 */ 954 /*ARGSUSED*/ 955 static int 956 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 957 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 958 uint_t lock) 959 { 960 961 #ifdef DDI_MAP_DEBUG 962 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 963 ddi_map_debug(" Seg <%s>\n", 964 seg->s_ops == &segdev_ops ? "segdev" : 965 seg == &kvseg ? "segkmem" : "NONE!"); 966 #endif /* DDI_MAP_DEBUG */ 967 968 /* 969 * This is all terribly broken, but it is a start 970 * 971 * XXX Note that this test means that segdev_ops 972 * must be exported from seg_dev.c. 973 * XXX What about devices with their own segment drivers? 974 */ 975 if (seg->s_ops == &segdev_ops) { 976 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 977 978 if (hat == NULL) { 979 /* 980 * This is one plausible interpretation of 981 * a null hat i.e. use the first hat on the 982 * address space hat list which by convention is 983 * the hat of the system MMU. At alternative 984 * would be to panic .. this might well be better .. 985 */ 986 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 987 hat = seg->s_as->a_hat; 988 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 989 } 990 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 991 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 992 } else if (seg == &kvseg && dp == NULL) { 993 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 994 HAT_LOAD_LOCK); 995 } else 996 return (DDI_FAILURE); 997 return (DDI_SUCCESS); 998 } 999 1000 1001 /* 1002 * rootnex_map_regspec() 1003 * we don't support mapping of I/O cards above 4Gb 1004 */ 1005 static int 1006 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1007 { 1008 rootnex_addr_t rbase; 1009 void *cvaddr; 1010 uint_t npages, pgoffset; 1011 struct regspec *rp; 1012 ddi_acc_hdl_t *hp; 1013 ddi_acc_impl_t *ap; 1014 uint_t hat_acc_flags; 1015 paddr_t pbase; 1016 1017 rp = mp->map_obj.rp; 1018 hp = mp->map_handlep; 1019 1020 #ifdef DDI_MAP_DEBUG 1021 ddi_map_debug( 1022 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1023 rp->regspec_bustype, rp->regspec_addr, 1024 rp->regspec_size, mp->map_handlep); 1025 #endif /* DDI_MAP_DEBUG */ 1026 1027 /* 1028 * I/O or memory mapping 1029 * 1030 * <bustype=0, addr=x, len=x>: memory 1031 * <bustype=1, addr=x, len=x>: i/o 1032 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1033 */ 1034 1035 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1036 cmn_err(CE_WARN, "rootnex: invalid register spec" 1037 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1038 rp->regspec_addr, rp->regspec_size); 1039 return (DDI_FAILURE); 1040 } 1041 1042 if (rp->regspec_bustype != 0) { 1043 /* 1044 * I/O space - needs a handle. 1045 */ 1046 if (hp == NULL) { 1047 return (DDI_FAILURE); 1048 } 1049 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1050 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1051 impl_acc_hdl_init(hp); 1052 1053 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1054 #ifdef DDI_MAP_DEBUG 1055 ddi_map_debug("rootnex_map_regspec: mmap() " 1056 "to I/O space is not supported.\n"); 1057 #endif /* DDI_MAP_DEBUG */ 1058 return (DDI_ME_INVAL); 1059 } else { 1060 /* 1061 * 1275-compliant vs. compatibility i/o mapping 1062 */ 1063 *vaddrp = 1064 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1065 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1066 ((caddr_t)(uintptr_t)rp->regspec_addr); 1067 #ifdef __xpv 1068 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1069 hp->ah_pfn = xen_assign_pfn( 1070 mmu_btop((ulong_t)rp->regspec_addr & 1071 MMU_PAGEMASK)); 1072 } else { 1073 hp->ah_pfn = mmu_btop( 1074 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1075 } 1076 #else 1077 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1078 MMU_PAGEMASK); 1079 #endif 1080 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1081 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1082 } 1083 1084 #ifdef DDI_MAP_DEBUG 1085 ddi_map_debug( 1086 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1087 rp->regspec_size, *vaddrp); 1088 #endif /* DDI_MAP_DEBUG */ 1089 return (DDI_SUCCESS); 1090 } 1091 1092 /* 1093 * Memory space 1094 */ 1095 1096 if (hp != NULL) { 1097 /* 1098 * hat layer ignores 1099 * hp->ah_acc.devacc_attr_endian_flags. 1100 */ 1101 switch (hp->ah_acc.devacc_attr_dataorder) { 1102 case DDI_STRICTORDER_ACC: 1103 hat_acc_flags = HAT_STRICTORDER; 1104 break; 1105 case DDI_UNORDERED_OK_ACC: 1106 hat_acc_flags = HAT_UNORDERED_OK; 1107 break; 1108 case DDI_MERGING_OK_ACC: 1109 hat_acc_flags = HAT_MERGING_OK; 1110 break; 1111 case DDI_LOADCACHING_OK_ACC: 1112 hat_acc_flags = HAT_LOADCACHING_OK; 1113 break; 1114 case DDI_STORECACHING_OK_ACC: 1115 hat_acc_flags = HAT_STORECACHING_OK; 1116 break; 1117 } 1118 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1119 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1120 impl_acc_hdl_init(hp); 1121 hp->ah_hat_flags = hat_acc_flags; 1122 } else { 1123 hat_acc_flags = HAT_STRICTORDER; 1124 } 1125 1126 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1127 #ifdef __xpv 1128 /* 1129 * If we're dom0, we're using a real device so we need to translate 1130 * the MA to a PA. 1131 */ 1132 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1133 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1134 } else { 1135 pbase = rbase; 1136 } 1137 #else 1138 pbase = rbase; 1139 #endif 1140 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1141 1142 if (rp->regspec_size == 0) { 1143 #ifdef DDI_MAP_DEBUG 1144 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1145 #endif /* DDI_MAP_DEBUG */ 1146 return (DDI_ME_INVAL); 1147 } 1148 1149 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1150 /* extra cast to make gcc happy */ 1151 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1152 } else { 1153 npages = mmu_btopr(rp->regspec_size + pgoffset); 1154 1155 #ifdef DDI_MAP_DEBUG 1156 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1157 "physical %llx", npages, pbase); 1158 #endif /* DDI_MAP_DEBUG */ 1159 1160 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1161 if (cvaddr == NULL) 1162 return (DDI_ME_NORESOURCES); 1163 1164 /* 1165 * Now map in the pages we've allocated... 1166 */ 1167 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1168 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1169 HAT_LOAD_LOCK); 1170 *vaddrp = (caddr_t)cvaddr + pgoffset; 1171 1172 /* save away pfn and npages for FMA */ 1173 hp = mp->map_handlep; 1174 if (hp) { 1175 hp->ah_pfn = mmu_btop(pbase); 1176 hp->ah_pnum = npages; 1177 } 1178 } 1179 1180 #ifdef DDI_MAP_DEBUG 1181 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1182 #endif /* DDI_MAP_DEBUG */ 1183 return (DDI_SUCCESS); 1184 } 1185 1186 1187 /* 1188 * rootnex_unmap_regspec() 1189 * 1190 */ 1191 static int 1192 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1193 { 1194 caddr_t addr = (caddr_t)*vaddrp; 1195 uint_t npages, pgoffset; 1196 struct regspec *rp; 1197 1198 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1199 return (0); 1200 1201 rp = mp->map_obj.rp; 1202 1203 if (rp->regspec_size == 0) { 1204 #ifdef DDI_MAP_DEBUG 1205 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1206 #endif /* DDI_MAP_DEBUG */ 1207 return (DDI_ME_INVAL); 1208 } 1209 1210 /* 1211 * I/O or memory mapping: 1212 * 1213 * <bustype=0, addr=x, len=x>: memory 1214 * <bustype=1, addr=x, len=x>: i/o 1215 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1216 */ 1217 if (rp->regspec_bustype != 0) { 1218 /* 1219 * This is I/O space, which requires no particular 1220 * processing on unmap since it isn't mapped in the 1221 * first place. 1222 */ 1223 return (DDI_SUCCESS); 1224 } 1225 1226 /* 1227 * Memory space 1228 */ 1229 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1230 npages = mmu_btopr(rp->regspec_size + pgoffset); 1231 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1232 device_arena_free(addr - pgoffset, ptob(npages)); 1233 1234 /* 1235 * Destroy the pointer - the mapping has logically gone 1236 */ 1237 *vaddrp = NULL; 1238 1239 return (DDI_SUCCESS); 1240 } 1241 1242 1243 /* 1244 * rootnex_map_handle() 1245 * 1246 */ 1247 static int 1248 rootnex_map_handle(ddi_map_req_t *mp) 1249 { 1250 rootnex_addr_t rbase; 1251 ddi_acc_hdl_t *hp; 1252 uint_t pgoffset; 1253 struct regspec *rp; 1254 paddr_t pbase; 1255 1256 rp = mp->map_obj.rp; 1257 1258 #ifdef DDI_MAP_DEBUG 1259 ddi_map_debug( 1260 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1261 rp->regspec_bustype, rp->regspec_addr, 1262 rp->regspec_size, mp->map_handlep); 1263 #endif /* DDI_MAP_DEBUG */ 1264 1265 /* 1266 * I/O or memory mapping: 1267 * 1268 * <bustype=0, addr=x, len=x>: memory 1269 * <bustype=1, addr=x, len=x>: i/o 1270 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1271 */ 1272 if (rp->regspec_bustype != 0) { 1273 /* 1274 * This refers to I/O space, and we don't support "mapping" 1275 * I/O space to a user. 1276 */ 1277 return (DDI_FAILURE); 1278 } 1279 1280 /* 1281 * Set up the hat_flags for the mapping. 1282 */ 1283 hp = mp->map_handlep; 1284 1285 switch (hp->ah_acc.devacc_attr_endian_flags) { 1286 case DDI_NEVERSWAP_ACC: 1287 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1288 break; 1289 case DDI_STRUCTURE_LE_ACC: 1290 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1291 break; 1292 case DDI_STRUCTURE_BE_ACC: 1293 return (DDI_FAILURE); 1294 default: 1295 return (DDI_REGS_ACC_CONFLICT); 1296 } 1297 1298 switch (hp->ah_acc.devacc_attr_dataorder) { 1299 case DDI_STRICTORDER_ACC: 1300 break; 1301 case DDI_UNORDERED_OK_ACC: 1302 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1303 break; 1304 case DDI_MERGING_OK_ACC: 1305 hp->ah_hat_flags |= HAT_MERGING_OK; 1306 break; 1307 case DDI_LOADCACHING_OK_ACC: 1308 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1309 break; 1310 case DDI_STORECACHING_OK_ACC: 1311 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1312 break; 1313 default: 1314 return (DDI_FAILURE); 1315 } 1316 1317 rbase = (rootnex_addr_t)rp->regspec_addr & 1318 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1319 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1320 1321 if (rp->regspec_size == 0) 1322 return (DDI_ME_INVAL); 1323 1324 #ifdef __xpv 1325 /* 1326 * If we're dom0, we're using a real device so we need to translate 1327 * the MA to a PA. 1328 */ 1329 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1330 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1331 (rbase & MMU_PAGEOFFSET); 1332 } else { 1333 pbase = rbase; 1334 } 1335 #else 1336 pbase = rbase; 1337 #endif 1338 1339 hp->ah_pfn = mmu_btop(pbase); 1340 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1341 1342 return (DDI_SUCCESS); 1343 } 1344 1345 1346 1347 /* 1348 * ************************ 1349 * interrupt related code 1350 * ************************ 1351 */ 1352 1353 /* 1354 * rootnex_intr_ops() 1355 * bus_intr_op() function for interrupt support 1356 */ 1357 /* ARGSUSED */ 1358 static int 1359 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1360 ddi_intr_handle_impl_t *hdlp, void *result) 1361 { 1362 struct intrspec *ispec; 1363 struct ddi_parent_private_data *pdp; 1364 1365 DDI_INTR_NEXDBG((CE_CONT, 1366 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1367 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1368 1369 /* Process the interrupt operation */ 1370 switch (intr_op) { 1371 case DDI_INTROP_GETCAP: 1372 /* First check with pcplusmp */ 1373 if (psm_intr_ops == NULL) 1374 return (DDI_FAILURE); 1375 1376 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1377 *(int *)result = 0; 1378 return (DDI_FAILURE); 1379 } 1380 break; 1381 case DDI_INTROP_SETCAP: 1382 if (psm_intr_ops == NULL) 1383 return (DDI_FAILURE); 1384 1385 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1386 return (DDI_FAILURE); 1387 break; 1388 case DDI_INTROP_ALLOC: 1389 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1390 return (DDI_FAILURE); 1391 hdlp->ih_pri = ispec->intrspec_pri; 1392 *(int *)result = hdlp->ih_scratch1; 1393 break; 1394 case DDI_INTROP_FREE: 1395 pdp = ddi_get_parent_data(rdip); 1396 /* 1397 * Special case for 'pcic' driver' only. 1398 * If an intrspec was created for it, clean it up here 1399 * See detailed comments on this in the function 1400 * rootnex_get_ispec(). 1401 */ 1402 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1403 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1404 pdp->par_nintr); 1405 /* 1406 * Set it to zero; so that 1407 * DDI framework doesn't free it again 1408 */ 1409 pdp->par_intr = NULL; 1410 pdp->par_nintr = 0; 1411 } 1412 break; 1413 case DDI_INTROP_GETPRI: 1414 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1415 return (DDI_FAILURE); 1416 *(int *)result = ispec->intrspec_pri; 1417 break; 1418 case DDI_INTROP_SETPRI: 1419 /* Validate the interrupt priority passed to us */ 1420 if (*(int *)result > LOCK_LEVEL) 1421 return (DDI_FAILURE); 1422 1423 /* Ensure that PSM is all initialized and ispec is ok */ 1424 if ((psm_intr_ops == NULL) || 1425 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1426 return (DDI_FAILURE); 1427 1428 /* Change the priority */ 1429 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1430 PSM_FAILURE) 1431 return (DDI_FAILURE); 1432 1433 /* update the ispec with the new priority */ 1434 ispec->intrspec_pri = *(int *)result; 1435 break; 1436 case DDI_INTROP_ADDISR: 1437 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1438 return (DDI_FAILURE); 1439 ispec->intrspec_func = hdlp->ih_cb_func; 1440 break; 1441 case DDI_INTROP_REMISR: 1442 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1443 return (DDI_FAILURE); 1444 ispec->intrspec_func = (uint_t (*)()) 0; 1445 break; 1446 case DDI_INTROP_ENABLE: 1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1448 return (DDI_FAILURE); 1449 1450 /* Call psmi to translate irq with the dip */ 1451 if (psm_intr_ops == NULL) 1452 return (DDI_FAILURE); 1453 1454 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1455 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1456 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1457 return (DDI_FAILURE); 1458 1459 /* Add the interrupt handler */ 1460 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1461 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1462 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1463 return (DDI_FAILURE); 1464 break; 1465 case DDI_INTROP_DISABLE: 1466 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1467 return (DDI_FAILURE); 1468 1469 /* Call psm_ops() to translate irq with the dip */ 1470 if (psm_intr_ops == NULL) 1471 return (DDI_FAILURE); 1472 1473 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1474 (void) (*psm_intr_ops)(rdip, hdlp, 1475 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1476 1477 /* Remove the interrupt handler */ 1478 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1479 hdlp->ih_cb_func, hdlp->ih_vector); 1480 break; 1481 case DDI_INTROP_SETMASK: 1482 if (psm_intr_ops == NULL) 1483 return (DDI_FAILURE); 1484 1485 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1486 return (DDI_FAILURE); 1487 break; 1488 case DDI_INTROP_CLRMASK: 1489 if (psm_intr_ops == NULL) 1490 return (DDI_FAILURE); 1491 1492 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1493 return (DDI_FAILURE); 1494 break; 1495 case DDI_INTROP_GETPENDING: 1496 if (psm_intr_ops == NULL) 1497 return (DDI_FAILURE); 1498 1499 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1500 result)) { 1501 *(int *)result = 0; 1502 return (DDI_FAILURE); 1503 } 1504 break; 1505 case DDI_INTROP_NAVAIL: 1506 case DDI_INTROP_NINTRS: 1507 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1508 if (*(int *)result == 0) { 1509 /* 1510 * Special case for 'pcic' driver' only. This driver 1511 * driver is a child of 'isa' and 'rootnex' drivers. 1512 * 1513 * See detailed comments on this in the function 1514 * rootnex_get_ispec(). 1515 * 1516 * Children of 'pcic' send 'NINITR' request all the 1517 * way to rootnex driver. But, the 'pdp->par_nintr' 1518 * field may not initialized. So, we fake it here 1519 * to return 1 (a la what PCMCIA nexus does). 1520 */ 1521 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1522 *(int *)result = 1; 1523 else 1524 return (DDI_FAILURE); 1525 } 1526 break; 1527 case DDI_INTROP_SUPPORTED_TYPES: 1528 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1529 break; 1530 default: 1531 return (DDI_FAILURE); 1532 } 1533 1534 return (DDI_SUCCESS); 1535 } 1536 1537 1538 /* 1539 * rootnex_get_ispec() 1540 * convert an interrupt number to an interrupt specification. 1541 * The interrupt number determines which interrupt spec will be 1542 * returned if more than one exists. 1543 * 1544 * Look into the parent private data area of the 'rdip' to find out 1545 * the interrupt specification. First check to make sure there is 1546 * one that matchs "inumber" and then return a pointer to it. 1547 * 1548 * Return NULL if one could not be found. 1549 * 1550 * NOTE: This is needed for rootnex_intr_ops() 1551 */ 1552 static struct intrspec * 1553 rootnex_get_ispec(dev_info_t *rdip, int inum) 1554 { 1555 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1556 1557 /* 1558 * Special case handling for drivers that provide their own 1559 * intrspec structures instead of relying on the DDI framework. 1560 * 1561 * A broken hardware driver in ON could potentially provide its 1562 * own intrspec structure, instead of relying on the hardware. 1563 * If these drivers are children of 'rootnex' then we need to 1564 * continue to provide backward compatibility to them here. 1565 * 1566 * Following check is a special case for 'pcic' driver which 1567 * was found to have broken hardwre andby provides its own intrspec. 1568 * 1569 * Verbatim comments from this driver are shown here: 1570 * "Don't use the ddi_add_intr since we don't have a 1571 * default intrspec in all cases." 1572 * 1573 * Since an 'ispec' may not be always created for it, 1574 * check for that and create one if so. 1575 * 1576 * NOTE: Currently 'pcic' is the only driver found to do this. 1577 */ 1578 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1579 pdp->par_nintr = 1; 1580 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1581 pdp->par_nintr, KM_SLEEP); 1582 } 1583 1584 /* Validate the interrupt number */ 1585 if (inum >= pdp->par_nintr) 1586 return (NULL); 1587 1588 /* Get the interrupt structure pointer and return that */ 1589 return ((struct intrspec *)&pdp->par_intr[inum]); 1590 } 1591 1592 1593 /* 1594 * ****************** 1595 * dma related code 1596 * ****************** 1597 */ 1598 1599 /*ARGSUSED*/ 1600 static int 1601 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1602 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1603 ddi_dma_handle_t *handlep) 1604 { 1605 uint64_t maxsegmentsize_ll; 1606 uint_t maxsegmentsize; 1607 ddi_dma_impl_t *hp; 1608 rootnex_dma_t *dma; 1609 uint64_t count_max; 1610 uint64_t seg; 1611 int kmflag; 1612 int e; 1613 1614 1615 /* convert our sleep flags */ 1616 if (waitfp == DDI_DMA_SLEEP) { 1617 kmflag = KM_SLEEP; 1618 } else { 1619 kmflag = KM_NOSLEEP; 1620 } 1621 1622 /* 1623 * We try to do only one memory allocation here. We'll do a little 1624 * pointer manipulation later. If the bind ends up taking more than 1625 * our prealloc's space, we'll have to allocate more memory in the 1626 * bind operation. Not great, but much better than before and the 1627 * best we can do with the current bind interfaces. 1628 */ 1629 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1630 if (hp == NULL) { 1631 if (waitfp != DDI_DMA_DONTWAIT) { 1632 ddi_set_callback(waitfp, arg, 1633 &rootnex_state->r_dvma_call_list_id); 1634 } 1635 return (DDI_DMA_NORESOURCES); 1636 } 1637 1638 /* Do our pointer manipulation now, align the structures */ 1639 hp->dmai_private = (void *)(((uintptr_t)hp + 1640 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1641 dma = (rootnex_dma_t *)hp->dmai_private; 1642 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1643 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1644 1645 /* setup the handle */ 1646 rootnex_clean_dmahdl(hp); 1647 dma->dp_dip = rdip; 1648 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1649 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1650 hp->dmai_minxfer = attr->dma_attr_minxfer; 1651 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1652 hp->dmai_rdip = rdip; 1653 hp->dmai_attr = *attr; 1654 1655 /* we don't need to worry about the SPL since we do a tryenter */ 1656 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1657 1658 /* 1659 * Figure out our maximum segment size. If the segment size is greater 1660 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1661 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1662 * dma_attr_count_max are size-1 type values. 1663 * 1664 * Maximum segment size is the largest physically contiguous chunk of 1665 * memory that we can return from a bind (i.e. the maximum size of a 1666 * single cookie). 1667 */ 1668 1669 /* handle the rollover cases */ 1670 seg = attr->dma_attr_seg + 1; 1671 if (seg < attr->dma_attr_seg) { 1672 seg = attr->dma_attr_seg; 1673 } 1674 count_max = attr->dma_attr_count_max + 1; 1675 if (count_max < attr->dma_attr_count_max) { 1676 count_max = attr->dma_attr_count_max; 1677 } 1678 1679 /* 1680 * granularity may or may not be a power of two. If it isn't, we can't 1681 * use a simple mask. 1682 */ 1683 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1684 dma->dp_granularity_power_2 = B_FALSE; 1685 } else { 1686 dma->dp_granularity_power_2 = B_TRUE; 1687 } 1688 1689 /* 1690 * maxxfer should be a whole multiple of granularity. If we're going to 1691 * break up a window because we're greater than maxxfer, we might as 1692 * well make sure it's maxxfer is a whole multiple so we don't have to 1693 * worry about triming the window later on for this case. 1694 */ 1695 if (attr->dma_attr_granular > 1) { 1696 if (dma->dp_granularity_power_2) { 1697 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1698 (attr->dma_attr_maxxfer & 1699 (attr->dma_attr_granular - 1)); 1700 } else { 1701 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1702 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1703 } 1704 } else { 1705 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1706 } 1707 1708 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1709 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1710 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1711 maxsegmentsize = 0xFFFFFFFF; 1712 } else { 1713 maxsegmentsize = maxsegmentsize_ll; 1714 } 1715 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1716 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1717 1718 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1719 if (rootnex_alloc_check_parms) { 1720 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1721 if (e != DDI_SUCCESS) { 1722 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1723 (void) rootnex_dma_freehdl(dip, rdip, 1724 (ddi_dma_handle_t)hp); 1725 return (e); 1726 } 1727 } 1728 1729 *handlep = (ddi_dma_handle_t)hp; 1730 1731 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1732 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1733 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1734 1735 return (DDI_SUCCESS); 1736 } 1737 1738 1739 /* 1740 * rootnex_dma_allochdl() 1741 * called from ddi_dma_alloc_handle(). 1742 */ 1743 static int 1744 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1745 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1746 { 1747 #if defined(__amd64) && !defined(__xpv) 1748 uint_t error = ENOTSUP; 1749 int retval; 1750 1751 retval = iommulib_nex_open(rdip, &error); 1752 1753 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1754 /* No IOMMU */ 1755 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1756 handlep)); 1757 } else if (retval != DDI_SUCCESS) { 1758 return (DDI_FAILURE); 1759 } 1760 1761 ASSERT(IOMMU_USED(rdip)); 1762 1763 /* has an IOMMU */ 1764 return (iommulib_nexdma_allochdl(dip, rdip, attr, 1765 waitfp, arg, handlep)); 1766 #else 1767 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1768 handlep)); 1769 #endif 1770 } 1771 1772 /*ARGSUSED*/ 1773 static int 1774 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1775 ddi_dma_handle_t handle) 1776 { 1777 ddi_dma_impl_t *hp; 1778 rootnex_dma_t *dma; 1779 1780 1781 hp = (ddi_dma_impl_t *)handle; 1782 dma = (rootnex_dma_t *)hp->dmai_private; 1783 1784 /* unbind should have been called first */ 1785 ASSERT(!dma->dp_inuse); 1786 1787 mutex_destroy(&dma->dp_mutex); 1788 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1789 1790 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1791 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1792 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1793 1794 if (rootnex_state->r_dvma_call_list_id) 1795 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1796 1797 return (DDI_SUCCESS); 1798 } 1799 1800 /* 1801 * rootnex_dma_freehdl() 1802 * called from ddi_dma_free_handle(). 1803 */ 1804 static int 1805 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1806 { 1807 #if defined(__amd64) && !defined(__xpv) 1808 if (IOMMU_USED(rdip)) { 1809 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1810 } 1811 #endif 1812 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1813 } 1814 1815 /*ARGSUSED*/ 1816 static int 1817 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1818 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1819 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1820 { 1821 rootnex_sglinfo_t *sinfo; 1822 ddi_dma_attr_t *attr; 1823 ddi_dma_impl_t *hp; 1824 rootnex_dma_t *dma; 1825 int kmflag; 1826 int e; 1827 1828 hp = (ddi_dma_impl_t *)handle; 1829 dma = (rootnex_dma_t *)hp->dmai_private; 1830 sinfo = &dma->dp_sglinfo; 1831 attr = &hp->dmai_attr; 1832 1833 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1834 dma->dp_sleep_flags = KM_SLEEP; 1835 } else { 1836 dma->dp_sleep_flags = KM_NOSLEEP; 1837 } 1838 1839 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1840 1841 /* 1842 * This is useful for debugging a driver. Not as useful in a production 1843 * system. The only time this will fail is if you have a driver bug. 1844 */ 1845 if (rootnex_bind_check_inuse) { 1846 /* 1847 * No one else should ever have this lock unless someone else 1848 * is trying to use this handle. So contention on the lock 1849 * is the same as inuse being set. 1850 */ 1851 e = mutex_tryenter(&dma->dp_mutex); 1852 if (e == 0) { 1853 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1854 return (DDI_DMA_INUSE); 1855 } 1856 if (dma->dp_inuse) { 1857 mutex_exit(&dma->dp_mutex); 1858 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1859 return (DDI_DMA_INUSE); 1860 } 1861 dma->dp_inuse = B_TRUE; 1862 mutex_exit(&dma->dp_mutex); 1863 } 1864 1865 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1866 if (rootnex_bind_check_parms) { 1867 e = rootnex_valid_bind_parms(dmareq, attr); 1868 if (e != DDI_SUCCESS) { 1869 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1870 rootnex_clean_dmahdl(hp); 1871 return (e); 1872 } 1873 } 1874 1875 /* save away the original bind info */ 1876 dma->dp_dma = dmareq->dmar_object; 1877 1878 #if defined(__amd64) && !defined(__xpv) 1879 e = immu_map_sgl(hp, dmareq, rootnex_prealloc_cookies, rdip); 1880 switch (e) { 1881 case DDI_DMA_MAPPED: 1882 goto out; 1883 case DDI_DMA_USE_PHYSICAL: 1884 break; 1885 case DDI_DMA_PARTIAL: 1886 ddi_err(DER_PANIC, rdip, "Partial DVMA map"); 1887 e = DDI_DMA_NORESOURCES; 1888 /*FALLTHROUGH*/ 1889 default: 1890 ddi_err(DER_MODE, rdip, "DVMA map failed"); 1891 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1892 rootnex_clean_dmahdl(hp); 1893 return (e); 1894 } 1895 #endif 1896 1897 /* 1898 * Figure out a rough estimate of what maximum number of pages this 1899 * buffer could use (a high estimate of course). 1900 */ 1901 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1902 1903 /* 1904 * We'll use the pre-allocated cookies for any bind that will *always* 1905 * fit (more important to be consistent, we don't want to create 1906 * additional degenerate cases). 1907 */ 1908 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1909 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1910 dma->dp_need_to_free_cookie = B_FALSE; 1911 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1912 uint_t, sinfo->si_max_pages); 1913 1914 /* 1915 * For anything larger than that, we'll go ahead and allocate the 1916 * maximum number of pages we expect to see. Hopefuly, we won't be 1917 * seeing this path in the fast path for high performance devices very 1918 * frequently. 1919 * 1920 * a ddi bind interface that allowed the driver to provide storage to 1921 * the bind interface would speed this case up. 1922 */ 1923 } else { 1924 /* convert the sleep flags */ 1925 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1926 kmflag = KM_SLEEP; 1927 } else { 1928 kmflag = KM_NOSLEEP; 1929 } 1930 1931 /* 1932 * Save away how much memory we allocated. If we're doing a 1933 * nosleep, the alloc could fail... 1934 */ 1935 dma->dp_cookie_size = sinfo->si_max_pages * 1936 sizeof (ddi_dma_cookie_t); 1937 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1938 if (dma->dp_cookies == NULL) { 1939 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1940 rootnex_clean_dmahdl(hp); 1941 return (DDI_DMA_NORESOURCES); 1942 } 1943 dma->dp_need_to_free_cookie = B_TRUE; 1944 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1945 sinfo->si_max_pages); 1946 } 1947 hp->dmai_cookie = dma->dp_cookies; 1948 1949 /* 1950 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1951 * looking at the constraints in the dma structure. It will then put 1952 * some additional state about the sgl in the dma struct (i.e. is 1953 * the sgl clean, or do we need to do some munging; how many pages 1954 * need to be copied, etc.) 1955 */ 1956 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1957 &dma->dp_sglinfo); 1958 1959 out: 1960 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1961 /* if we don't need a copy buffer, we don't need to sync */ 1962 if (sinfo->si_copybuf_req == 0) { 1963 hp->dmai_rflags |= DMP_NOSYNC; 1964 } 1965 1966 /* 1967 * if we don't need the copybuf and we don't need to do a partial, we 1968 * hit the fast path. All the high performance devices should be trying 1969 * to hit this path. To hit this path, a device should be able to reach 1970 * all of memory, shouldn't try to bind more than it can transfer, and 1971 * the buffer shouldn't require more cookies than the driver/device can 1972 * handle [sgllen]). 1973 */ 1974 if ((sinfo->si_copybuf_req == 0) && 1975 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1976 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1977 /* 1978 * If the driver supports FMA, insert the handle in the FMA DMA 1979 * handle cache. 1980 */ 1981 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1982 hp->dmai_error.err_cf = rootnex_dma_check; 1983 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1984 } 1985 1986 /* 1987 * copy out the first cookie and ccountp, set the cookie 1988 * pointer to the second cookie. The first cookie is passed 1989 * back on the stack. Additional cookies are accessed via 1990 * ddi_dma_nextcookie() 1991 */ 1992 *cookiep = dma->dp_cookies[0]; 1993 *ccountp = sinfo->si_sgl_size; 1994 hp->dmai_cookie++; 1995 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1996 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1997 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, 1998 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 1999 uint_t, dma->dp_dma.dmao_size); 2000 2001 2002 return (DDI_DMA_MAPPED); 2003 } 2004 2005 /* 2006 * go to the slow path, we may need to alloc more memory, create 2007 * multiple windows, and munge up a sgl to make the device happy. 2008 */ 2009 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2010 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2011 if (dma->dp_need_to_free_cookie) { 2012 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2013 } 2014 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2015 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2016 return (e); 2017 } 2018 2019 /* 2020 * If the driver supports FMA, insert the handle in the FMA DMA handle 2021 * cache. 2022 */ 2023 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 2024 hp->dmai_error.err_cf = rootnex_dma_check; 2025 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 2026 } 2027 2028 /* if the first window uses the copy buffer, sync it for the device */ 2029 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2030 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2031 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2032 DDI_DMA_SYNC_FORDEV); 2033 } 2034 2035 /* 2036 * copy out the first cookie and ccountp, set the cookie pointer to the 2037 * second cookie. Make sure the partial flag is set/cleared correctly. 2038 * If we have a partial map (i.e. multiple windows), the number of 2039 * cookies we return is the number of cookies in the first window. 2040 */ 2041 if (e == DDI_DMA_MAPPED) { 2042 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2043 *ccountp = sinfo->si_sgl_size; 2044 hp->dmai_nwin = 1; 2045 } else { 2046 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2047 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2048 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2049 } 2050 *cookiep = dma->dp_cookies[0]; 2051 hp->dmai_cookie++; 2052 2053 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2054 ROOTNEX_DPROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2055 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2056 dma->dp_dma.dmao_size); 2057 return (e); 2058 } 2059 2060 /* 2061 * rootnex_dma_bindhdl() 2062 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2063 */ 2064 static int 2065 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2066 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2067 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2068 { 2069 #if defined(__amd64) && !defined(__xpv) 2070 if (IOMMU_USED(rdip)) { 2071 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2072 cookiep, ccountp)); 2073 } 2074 #endif 2075 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2076 cookiep, ccountp)); 2077 } 2078 2079 2080 2081 /*ARGSUSED*/ 2082 static int 2083 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2084 ddi_dma_handle_t handle) 2085 { 2086 ddi_dma_impl_t *hp; 2087 rootnex_dma_t *dma; 2088 int e; 2089 2090 hp = (ddi_dma_impl_t *)handle; 2091 dma = (rootnex_dma_t *)hp->dmai_private; 2092 2093 /* make sure the buffer wasn't free'd before calling unbind */ 2094 if (rootnex_unbind_verify_buffer) { 2095 e = rootnex_verify_buffer(dma); 2096 if (e != DDI_SUCCESS) { 2097 ASSERT(0); 2098 return (DDI_FAILURE); 2099 } 2100 } 2101 2102 /* sync the current window before unbinding the buffer */ 2103 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2104 (hp->dmai_rflags & DDI_DMA_READ)) { 2105 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2106 DDI_DMA_SYNC_FORCPU); 2107 } 2108 2109 /* 2110 * If the driver supports FMA, remove the handle in the FMA DMA handle 2111 * cache. 2112 */ 2113 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 2114 if ((DEVI(rdip)->devi_fmhdl != NULL) && 2115 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 2116 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 2117 } 2118 } 2119 2120 /* 2121 * cleanup and copy buffer or window state. if we didn't use the copy 2122 * buffer or windows, there won't be much to do :-) 2123 */ 2124 rootnex_teardown_copybuf(dma); 2125 rootnex_teardown_windows(dma); 2126 2127 #if defined(__amd64) && !defined(__xpv) 2128 /* 2129 * Clean up the page tables and free the dvma 2130 */ 2131 e = immu_unmap_sgl(hp, rdip); 2132 if (e != DDI_DMA_USE_PHYSICAL && e != DDI_SUCCESS) { 2133 return (e); 2134 } 2135 #endif 2136 2137 /* 2138 * If we had to allocate space to for the worse case sgl (it didn't 2139 * fit into our pre-allocate buffer), free that up now 2140 */ 2141 if (dma->dp_need_to_free_cookie) { 2142 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2143 } 2144 2145 /* 2146 * clean up the handle so it's ready for the next bind (i.e. if the 2147 * handle is reused). 2148 */ 2149 rootnex_clean_dmahdl(hp); 2150 2151 if (rootnex_state->r_dvma_call_list_id) 2152 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2153 2154 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2155 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2156 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2157 2158 return (DDI_SUCCESS); 2159 } 2160 2161 /* 2162 * rootnex_dma_unbindhdl() 2163 * called from ddi_dma_unbind_handle() 2164 */ 2165 /*ARGSUSED*/ 2166 static int 2167 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2168 ddi_dma_handle_t handle) 2169 { 2170 #if defined(__amd64) && !defined(__xpv) 2171 if (IOMMU_USED(rdip)) { 2172 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2173 } 2174 #endif 2175 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2176 } 2177 2178 #if defined(__amd64) && !defined(__xpv) 2179 2180 static int 2181 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2182 { 2183 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2184 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2185 2186 if (dma->dp_sleep_flags != KM_SLEEP && 2187 dma->dp_sleep_flags != KM_NOSLEEP) 2188 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2189 return (dma->dp_sleep_flags); 2190 } 2191 /*ARGSUSED*/ 2192 static void 2193 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2194 { 2195 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2196 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2197 rootnex_window_t *window; 2198 2199 if (dma->dp_window) { 2200 window = &dma->dp_window[dma->dp_current_win]; 2201 hp->dmai_cookie = window->wd_first_cookie; 2202 } else { 2203 hp->dmai_cookie = dma->dp_cookies; 2204 } 2205 hp->dmai_cookie++; 2206 } 2207 2208 /*ARGSUSED*/ 2209 static int 2210 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2211 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2212 { 2213 int i; 2214 int km_flags; 2215 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2216 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2217 rootnex_window_t *window; 2218 ddi_dma_cookie_t *cp; 2219 ddi_dma_cookie_t *cookie; 2220 2221 ASSERT(*cookiepp == NULL); 2222 ASSERT(*ccountp == 0); 2223 2224 if (dma->dp_window) { 2225 window = &dma->dp_window[dma->dp_current_win]; 2226 cp = window->wd_first_cookie; 2227 *ccountp = window->wd_cookie_cnt; 2228 } else { 2229 cp = dma->dp_cookies; 2230 *ccountp = dma->dp_sglinfo.si_sgl_size; 2231 } 2232 2233 km_flags = rootnex_coredma_get_sleep_flags(handle); 2234 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2235 if (cookie == NULL) { 2236 return (DDI_DMA_NORESOURCES); 2237 } 2238 2239 for (i = 0; i < *ccountp; i++) { 2240 cookie[i].dmac_notused = cp[i].dmac_notused; 2241 cookie[i].dmac_type = cp[i].dmac_type; 2242 cookie[i].dmac_address = cp[i].dmac_address; 2243 cookie[i].dmac_size = cp[i].dmac_size; 2244 } 2245 2246 *cookiepp = cookie; 2247 2248 return (DDI_SUCCESS); 2249 } 2250 2251 /*ARGSUSED*/ 2252 static int 2253 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2254 ddi_dma_cookie_t *cookiep, uint_t ccount) 2255 { 2256 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2257 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2258 rootnex_window_t *window; 2259 ddi_dma_cookie_t *cur_cookiep; 2260 2261 ASSERT(cookiep); 2262 ASSERT(ccount != 0); 2263 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2264 2265 if (dma->dp_window) { 2266 window = &dma->dp_window[dma->dp_current_win]; 2267 dma->dp_saved_cookies = window->wd_first_cookie; 2268 window->wd_first_cookie = cookiep; 2269 ASSERT(ccount == window->wd_cookie_cnt); 2270 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2271 + window->wd_first_cookie; 2272 } else { 2273 dma->dp_saved_cookies = dma->dp_cookies; 2274 dma->dp_cookies = cookiep; 2275 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2276 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2277 + dma->dp_cookies; 2278 } 2279 2280 dma->dp_need_to_switch_cookies = B_TRUE; 2281 hp->dmai_cookie = cur_cookiep; 2282 2283 return (DDI_SUCCESS); 2284 } 2285 2286 /*ARGSUSED*/ 2287 static int 2288 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2289 { 2290 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2291 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2292 rootnex_window_t *window; 2293 ddi_dma_cookie_t *cur_cookiep; 2294 ddi_dma_cookie_t *cookie_array; 2295 uint_t ccount; 2296 2297 /* check if cookies have not been switched */ 2298 if (dma->dp_need_to_switch_cookies == B_FALSE) 2299 return (DDI_SUCCESS); 2300 2301 ASSERT(dma->dp_saved_cookies); 2302 2303 if (dma->dp_window) { 2304 window = &dma->dp_window[dma->dp_current_win]; 2305 cookie_array = window->wd_first_cookie; 2306 window->wd_first_cookie = dma->dp_saved_cookies; 2307 dma->dp_saved_cookies = NULL; 2308 ccount = window->wd_cookie_cnt; 2309 cur_cookiep = (hp->dmai_cookie - cookie_array) 2310 + window->wd_first_cookie; 2311 } else { 2312 cookie_array = dma->dp_cookies; 2313 dma->dp_cookies = dma->dp_saved_cookies; 2314 dma->dp_saved_cookies = NULL; 2315 ccount = dma->dp_sglinfo.si_sgl_size; 2316 cur_cookiep = (hp->dmai_cookie - cookie_array) 2317 + dma->dp_cookies; 2318 } 2319 2320 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2321 2322 hp->dmai_cookie = cur_cookiep; 2323 2324 dma->dp_need_to_switch_cookies = B_FALSE; 2325 2326 return (DDI_SUCCESS); 2327 } 2328 2329 #endif 2330 2331 /* 2332 * rootnex_verify_buffer() 2333 * verify buffer wasn't free'd 2334 */ 2335 static int 2336 rootnex_verify_buffer(rootnex_dma_t *dma) 2337 { 2338 page_t **pplist; 2339 caddr_t vaddr; 2340 uint_t pcnt; 2341 uint_t poff; 2342 page_t *pp; 2343 char b; 2344 int i; 2345 2346 /* Figure out how many pages this buffer occupies */ 2347 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2348 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2349 } else { 2350 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2351 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2352 } 2353 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2354 2355 switch (dma->dp_dma.dmao_type) { 2356 case DMA_OTYP_PAGES: 2357 /* 2358 * for a linked list of pp's walk through them to make sure 2359 * they're locked and not free. 2360 */ 2361 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2362 for (i = 0; i < pcnt; i++) { 2363 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2364 return (DDI_FAILURE); 2365 } 2366 pp = pp->p_next; 2367 } 2368 break; 2369 2370 case DMA_OTYP_VADDR: 2371 case DMA_OTYP_BUFVADDR: 2372 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2373 /* 2374 * for an array of pp's walk through them to make sure they're 2375 * not free. It's possible that they may not be locked. 2376 */ 2377 if (pplist) { 2378 for (i = 0; i < pcnt; i++) { 2379 if (PP_ISFREE(pplist[i])) { 2380 return (DDI_FAILURE); 2381 } 2382 } 2383 2384 /* For a virtual address, try to peek at each page */ 2385 } else { 2386 if (dma->dp_sglinfo.si_asp == &kas) { 2387 for (i = 0; i < pcnt; i++) { 2388 if (ddi_peek8(NULL, vaddr, &b) == 2389 DDI_FAILURE) 2390 return (DDI_FAILURE); 2391 vaddr += MMU_PAGESIZE; 2392 } 2393 } 2394 } 2395 break; 2396 2397 default: 2398 ASSERT(0); 2399 break; 2400 } 2401 2402 return (DDI_SUCCESS); 2403 } 2404 2405 2406 /* 2407 * rootnex_clean_dmahdl() 2408 * Clean the dma handle. This should be called on a handle alloc and an 2409 * unbind handle. Set the handle state to the default settings. 2410 */ 2411 static void 2412 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2413 { 2414 rootnex_dma_t *dma; 2415 2416 2417 dma = (rootnex_dma_t *)hp->dmai_private; 2418 2419 hp->dmai_nwin = 0; 2420 dma->dp_current_cookie = 0; 2421 dma->dp_copybuf_size = 0; 2422 dma->dp_window = NULL; 2423 dma->dp_cbaddr = NULL; 2424 dma->dp_inuse = B_FALSE; 2425 dma->dp_need_to_free_cookie = B_FALSE; 2426 dma->dp_need_to_switch_cookies = B_FALSE; 2427 dma->dp_saved_cookies = NULL; 2428 dma->dp_sleep_flags = KM_PANIC; 2429 dma->dp_need_to_free_window = B_FALSE; 2430 dma->dp_partial_required = B_FALSE; 2431 dma->dp_trim_required = B_FALSE; 2432 dma->dp_sglinfo.si_copybuf_req = 0; 2433 #if !defined(__amd64) 2434 dma->dp_cb_remaping = B_FALSE; 2435 dma->dp_kva = NULL; 2436 #endif 2437 2438 /* FMA related initialization */ 2439 hp->dmai_fault = 0; 2440 hp->dmai_fault_check = NULL; 2441 hp->dmai_fault_notify = NULL; 2442 hp->dmai_error.err_ena = 0; 2443 hp->dmai_error.err_status = DDI_FM_OK; 2444 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2445 hp->dmai_error.err_ontrap = NULL; 2446 hp->dmai_error.err_fep = NULL; 2447 hp->dmai_error.err_cf = NULL; 2448 } 2449 2450 2451 /* 2452 * rootnex_valid_alloc_parms() 2453 * Called in ddi_dma_alloc_handle path to validate its parameters. 2454 */ 2455 static int 2456 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2457 { 2458 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2459 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2460 (attr->dma_attr_granular > MMU_PAGESIZE) || 2461 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2462 return (DDI_DMA_BADATTR); 2463 } 2464 2465 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2466 return (DDI_DMA_BADATTR); 2467 } 2468 2469 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2470 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2471 attr->dma_attr_sgllen <= 0) { 2472 return (DDI_DMA_BADATTR); 2473 } 2474 2475 /* We should be able to DMA into every byte offset in a page */ 2476 if (maxsegmentsize < MMU_PAGESIZE) { 2477 return (DDI_DMA_BADATTR); 2478 } 2479 2480 return (DDI_SUCCESS); 2481 } 2482 2483 /* 2484 * rootnex_valid_bind_parms() 2485 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2486 */ 2487 /* ARGSUSED */ 2488 static int 2489 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2490 { 2491 #if !defined(__amd64) 2492 /* 2493 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2494 * we can track the offset for the obsoleted interfaces. 2495 */ 2496 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2497 return (DDI_DMA_TOOBIG); 2498 } 2499 #endif 2500 2501 return (DDI_SUCCESS); 2502 } 2503 2504 2505 /* 2506 * rootnex_get_sgl() 2507 * Called in bind fastpath to get the sgl. Most of this will be replaced 2508 * with a call to the vm layer when vm2.0 comes around... 2509 */ 2510 static void 2511 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2512 rootnex_sglinfo_t *sglinfo) 2513 { 2514 ddi_dma_atyp_t buftype; 2515 rootnex_addr_t raddr; 2516 uint64_t last_page; 2517 uint64_t offset; 2518 uint64_t addrhi; 2519 uint64_t addrlo; 2520 uint64_t maxseg; 2521 page_t **pplist; 2522 uint64_t paddr; 2523 uint32_t psize; 2524 uint32_t size; 2525 caddr_t vaddr; 2526 uint_t pcnt; 2527 page_t *pp; 2528 uint_t cnt; 2529 2530 2531 /* shortcuts */ 2532 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2533 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2534 maxseg = sglinfo->si_max_cookie_size; 2535 buftype = dmar_object->dmao_type; 2536 addrhi = sglinfo->si_max_addr; 2537 addrlo = sglinfo->si_min_addr; 2538 size = dmar_object->dmao_size; 2539 2540 pcnt = 0; 2541 cnt = 0; 2542 2543 /* 2544 * if we were passed down a linked list of pages, i.e. pointer to 2545 * page_t, use this to get our physical address and buf offset. 2546 */ 2547 if (buftype == DMA_OTYP_PAGES) { 2548 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2549 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2550 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2551 MMU_PAGEOFFSET; 2552 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2553 psize = MIN(size, (MMU_PAGESIZE - offset)); 2554 pp = pp->p_next; 2555 sglinfo->si_asp = NULL; 2556 2557 /* 2558 * We weren't passed down a linked list of pages, but if we were passed 2559 * down an array of pages, use this to get our physical address and buf 2560 * offset. 2561 */ 2562 } else if (pplist != NULL) { 2563 ASSERT((buftype == DMA_OTYP_VADDR) || 2564 (buftype == DMA_OTYP_BUFVADDR)); 2565 2566 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2567 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2568 if (sglinfo->si_asp == NULL) { 2569 sglinfo->si_asp = &kas; 2570 } 2571 2572 ASSERT(!PP_ISFREE(pplist[pcnt])); 2573 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2574 paddr += offset; 2575 psize = MIN(size, (MMU_PAGESIZE - offset)); 2576 pcnt++; 2577 2578 /* 2579 * All we have is a virtual address, we'll need to call into the VM 2580 * to get the physical address. 2581 */ 2582 } else { 2583 ASSERT((buftype == DMA_OTYP_VADDR) || 2584 (buftype == DMA_OTYP_BUFVADDR)); 2585 2586 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2587 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2588 if (sglinfo->si_asp == NULL) { 2589 sglinfo->si_asp = &kas; 2590 } 2591 2592 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2593 paddr += offset; 2594 psize = MIN(size, (MMU_PAGESIZE - offset)); 2595 vaddr += psize; 2596 } 2597 2598 #ifdef __xpv 2599 /* 2600 * If we're dom0, we're using a real device so we need to load 2601 * the cookies with MFNs instead of PFNs. 2602 */ 2603 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2604 #else 2605 raddr = paddr; 2606 #endif 2607 2608 /* 2609 * Setup the first cookie with the physical address of the page and the 2610 * size of the page (which takes into account the initial offset into 2611 * the page. 2612 */ 2613 sgl[cnt].dmac_laddress = raddr; 2614 sgl[cnt].dmac_size = psize; 2615 sgl[cnt].dmac_type = 0; 2616 2617 /* 2618 * Save away the buffer offset into the page. We'll need this later in 2619 * the copy buffer code to help figure out the page index within the 2620 * buffer and the offset into the current page. 2621 */ 2622 sglinfo->si_buf_offset = offset; 2623 2624 /* 2625 * If the DMA engine can't reach the physical address, increase how 2626 * much copy buffer we need. We always increase by pagesize so we don't 2627 * have to worry about converting offsets. Set a flag in the cookies 2628 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2629 * last cookie, go to the next cookie (since we separate each page which 2630 * uses the copy buffer in case the copy buffer is not physically 2631 * contiguous. 2632 */ 2633 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2634 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2635 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2636 if ((cnt + 1) < sglinfo->si_max_pages) { 2637 cnt++; 2638 sgl[cnt].dmac_laddress = 0; 2639 sgl[cnt].dmac_size = 0; 2640 sgl[cnt].dmac_type = 0; 2641 } 2642 } 2643 2644 /* 2645 * save this page's physical address so we can figure out if the next 2646 * page is physically contiguous. Keep decrementing size until we are 2647 * done with the buffer. 2648 */ 2649 last_page = raddr & MMU_PAGEMASK; 2650 size -= psize; 2651 2652 while (size > 0) { 2653 /* Get the size for this page (i.e. partial or full page) */ 2654 psize = MIN(size, MMU_PAGESIZE); 2655 2656 if (buftype == DMA_OTYP_PAGES) { 2657 /* get the paddr from the page_t */ 2658 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2659 paddr = pfn_to_pa(pp->p_pagenum); 2660 pp = pp->p_next; 2661 } else if (pplist != NULL) { 2662 /* index into the array of page_t's to get the paddr */ 2663 ASSERT(!PP_ISFREE(pplist[pcnt])); 2664 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2665 pcnt++; 2666 } else { 2667 /* call into the VM to get the paddr */ 2668 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2669 vaddr)); 2670 vaddr += psize; 2671 } 2672 2673 #ifdef __xpv 2674 /* 2675 * If we're dom0, we're using a real device so we need to load 2676 * the cookies with MFNs instead of PFNs. 2677 */ 2678 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2679 #else 2680 raddr = paddr; 2681 #endif 2682 /* check to see if this page needs the copy buffer */ 2683 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2684 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2685 2686 /* 2687 * if there is something in the current cookie, go to 2688 * the next one. We only want one page in a cookie which 2689 * uses the copybuf since the copybuf doesn't have to 2690 * be physically contiguous. 2691 */ 2692 if (sgl[cnt].dmac_size != 0) { 2693 cnt++; 2694 } 2695 sgl[cnt].dmac_laddress = raddr; 2696 sgl[cnt].dmac_size = psize; 2697 #if defined(__amd64) 2698 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2699 #else 2700 /* 2701 * save the buf offset for 32-bit kernel. used in the 2702 * obsoleted interfaces. 2703 */ 2704 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2705 (dmar_object->dmao_size - size); 2706 #endif 2707 /* if this isn't the last cookie, go to the next one */ 2708 if ((cnt + 1) < sglinfo->si_max_pages) { 2709 cnt++; 2710 sgl[cnt].dmac_laddress = 0; 2711 sgl[cnt].dmac_size = 0; 2712 sgl[cnt].dmac_type = 0; 2713 } 2714 2715 /* 2716 * this page didn't need the copy buffer, if it's not physically 2717 * contiguous, or it would put us over a segment boundary, or it 2718 * puts us over the max cookie size, or the current sgl doesn't 2719 * have anything in it. 2720 */ 2721 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2722 !(raddr & sglinfo->si_segmask) || 2723 ((sgl[cnt].dmac_size + psize) > maxseg) || 2724 (sgl[cnt].dmac_size == 0)) { 2725 /* 2726 * if we're not already in a new cookie, go to the next 2727 * cookie. 2728 */ 2729 if (sgl[cnt].dmac_size != 0) { 2730 cnt++; 2731 } 2732 2733 /* save the cookie information */ 2734 sgl[cnt].dmac_laddress = raddr; 2735 sgl[cnt].dmac_size = psize; 2736 #if defined(__amd64) 2737 sgl[cnt].dmac_type = 0; 2738 #else 2739 /* 2740 * save the buf offset for 32-bit kernel. used in the 2741 * obsoleted interfaces. 2742 */ 2743 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2744 #endif 2745 2746 /* 2747 * this page didn't need the copy buffer, it is physically 2748 * contiguous with the last page, and it's <= the max cookie 2749 * size. 2750 */ 2751 } else { 2752 sgl[cnt].dmac_size += psize; 2753 2754 /* 2755 * if this exactly == the maximum cookie size, and 2756 * it isn't the last cookie, go to the next cookie. 2757 */ 2758 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2759 ((cnt + 1) < sglinfo->si_max_pages)) { 2760 cnt++; 2761 sgl[cnt].dmac_laddress = 0; 2762 sgl[cnt].dmac_size = 0; 2763 sgl[cnt].dmac_type = 0; 2764 } 2765 } 2766 2767 /* 2768 * save this page's physical address so we can figure out if the 2769 * next page is physically contiguous. Keep decrementing size 2770 * until we are done with the buffer. 2771 */ 2772 last_page = raddr; 2773 size -= psize; 2774 } 2775 2776 /* we're done, save away how many cookies the sgl has */ 2777 if (sgl[cnt].dmac_size == 0) { 2778 ASSERT(cnt < sglinfo->si_max_pages); 2779 sglinfo->si_sgl_size = cnt; 2780 } else { 2781 sglinfo->si_sgl_size = cnt + 1; 2782 } 2783 } 2784 2785 /* 2786 * rootnex_bind_slowpath() 2787 * Call in the bind path if the calling driver can't use the sgl without 2788 * modifying it. We either need to use the copy buffer and/or we will end up 2789 * with a partial bind. 2790 */ 2791 static int 2792 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2793 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2794 { 2795 rootnex_sglinfo_t *sinfo; 2796 rootnex_window_t *window; 2797 ddi_dma_cookie_t *cookie; 2798 size_t copybuf_used; 2799 size_t dmac_size; 2800 boolean_t partial; 2801 off_t cur_offset; 2802 page_t *cur_pp; 2803 major_t mnum; 2804 int e; 2805 int i; 2806 2807 2808 sinfo = &dma->dp_sglinfo; 2809 copybuf_used = 0; 2810 partial = B_FALSE; 2811 2812 /* 2813 * If we're using the copybuf, set the copybuf state in dma struct. 2814 * Needs to be first since it sets the copy buffer size. 2815 */ 2816 if (sinfo->si_copybuf_req != 0) { 2817 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2818 if (e != DDI_SUCCESS) { 2819 return (e); 2820 } 2821 } else { 2822 dma->dp_copybuf_size = 0; 2823 } 2824 2825 /* 2826 * Figure out if we need to do a partial mapping. If so, figure out 2827 * if we need to trim the buffers when we munge the sgl. 2828 */ 2829 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2830 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2831 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2832 dma->dp_partial_required = B_TRUE; 2833 if (attr->dma_attr_granular != 1) { 2834 dma->dp_trim_required = B_TRUE; 2835 } 2836 } else { 2837 dma->dp_partial_required = B_FALSE; 2838 dma->dp_trim_required = B_FALSE; 2839 } 2840 2841 /* If we need to do a partial bind, make sure the driver supports it */ 2842 if (dma->dp_partial_required && 2843 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2844 2845 mnum = ddi_driver_major(dma->dp_dip); 2846 /* 2847 * patchable which allows us to print one warning per major 2848 * number. 2849 */ 2850 if ((rootnex_bind_warn) && 2851 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2852 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2853 cmn_err(CE_WARN, "!%s: coding error detected, the " 2854 "driver is using ddi_dma_attr(9S) incorrectly. " 2855 "There is a small risk of data corruption in " 2856 "particular with large I/Os. The driver should be " 2857 "replaced with a corrected version for proper " 2858 "system operation. To disable this warning, add " 2859 "'set rootnex:rootnex_bind_warn=0' to " 2860 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2861 } 2862 return (DDI_DMA_TOOBIG); 2863 } 2864 2865 /* 2866 * we might need multiple windows, setup state to handle them. In this 2867 * code path, we will have at least one window. 2868 */ 2869 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2870 if (e != DDI_SUCCESS) { 2871 rootnex_teardown_copybuf(dma); 2872 return (e); 2873 } 2874 2875 window = &dma->dp_window[0]; 2876 cookie = &dma->dp_cookies[0]; 2877 cur_offset = 0; 2878 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2879 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2880 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2881 } 2882 2883 /* loop though all the cookies we got back from get_sgl() */ 2884 for (i = 0; i < sinfo->si_sgl_size; i++) { 2885 /* 2886 * If we're using the copy buffer, check this cookie and setup 2887 * its associated copy buffer state. If this cookie uses the 2888 * copy buffer, make sure we sync this window during dma_sync. 2889 */ 2890 if (dma->dp_copybuf_size > 0) { 2891 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2892 cur_offset, ©buf_used, &cur_pp); 2893 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2894 window->wd_dosync = B_TRUE; 2895 } 2896 } 2897 2898 /* 2899 * save away the cookie size, since it could be modified in 2900 * the windowing code. 2901 */ 2902 dmac_size = cookie->dmac_size; 2903 2904 /* if we went over max copybuf size */ 2905 if (dma->dp_copybuf_size && 2906 (copybuf_used > dma->dp_copybuf_size)) { 2907 partial = B_TRUE; 2908 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2909 cookie, cur_offset, ©buf_used); 2910 if (e != DDI_SUCCESS) { 2911 rootnex_teardown_copybuf(dma); 2912 rootnex_teardown_windows(dma); 2913 return (e); 2914 } 2915 2916 /* 2917 * if the coookie uses the copy buffer, make sure the 2918 * new window we just moved to is set to sync. 2919 */ 2920 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2921 window->wd_dosync = B_TRUE; 2922 } 2923 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2924 dma->dp_dip); 2925 2926 /* if the cookie cnt == max sgllen, move to the next window */ 2927 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2928 partial = B_TRUE; 2929 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2930 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2931 cookie, attr, cur_offset); 2932 if (e != DDI_SUCCESS) { 2933 rootnex_teardown_copybuf(dma); 2934 rootnex_teardown_windows(dma); 2935 return (e); 2936 } 2937 2938 /* 2939 * if the coookie uses the copy buffer, make sure the 2940 * new window we just moved to is set to sync. 2941 */ 2942 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2943 window->wd_dosync = B_TRUE; 2944 } 2945 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2946 dma->dp_dip); 2947 2948 /* else if we will be over maxxfer */ 2949 } else if ((window->wd_size + dmac_size) > 2950 dma->dp_maxxfer) { 2951 partial = B_TRUE; 2952 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2953 cookie); 2954 if (e != DDI_SUCCESS) { 2955 rootnex_teardown_copybuf(dma); 2956 rootnex_teardown_windows(dma); 2957 return (e); 2958 } 2959 2960 /* 2961 * if the coookie uses the copy buffer, make sure the 2962 * new window we just moved to is set to sync. 2963 */ 2964 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2965 window->wd_dosync = B_TRUE; 2966 } 2967 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2968 dma->dp_dip); 2969 2970 /* else this cookie fits in the current window */ 2971 } else { 2972 window->wd_cookie_cnt++; 2973 window->wd_size += dmac_size; 2974 } 2975 2976 /* track our offset into the buffer, go to the next cookie */ 2977 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2978 ASSERT(cookie->dmac_size <= dmac_size); 2979 cur_offset += dmac_size; 2980 cookie++; 2981 } 2982 2983 /* if we ended up with a zero sized window in the end, clean it up */ 2984 if (window->wd_size == 0) { 2985 hp->dmai_nwin--; 2986 window--; 2987 } 2988 2989 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2990 2991 if (!partial) { 2992 return (DDI_DMA_MAPPED); 2993 } 2994 2995 ASSERT(dma->dp_partial_required); 2996 return (DDI_DMA_PARTIAL_MAP); 2997 } 2998 2999 3000 /* 3001 * rootnex_setup_copybuf() 3002 * Called in bind slowpath. Figures out if we're going to use the copy 3003 * buffer, and if we do, sets up the basic state to handle it. 3004 */ 3005 static int 3006 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3007 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3008 { 3009 rootnex_sglinfo_t *sinfo; 3010 ddi_dma_attr_t lattr; 3011 size_t max_copybuf; 3012 int cansleep; 3013 int e; 3014 #if !defined(__amd64) 3015 int vmflag; 3016 #endif 3017 3018 3019 sinfo = &dma->dp_sglinfo; 3020 3021 /* read this first so it's consistent through the routine */ 3022 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3023 3024 /* We need to call into the rootnex on ddi_dma_sync() */ 3025 hp->dmai_rflags &= ~DMP_NOSYNC; 3026 3027 /* make sure the copybuf size <= the max size */ 3028 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3029 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3030 3031 #if !defined(__amd64) 3032 /* 3033 * if we don't have kva space to copy to/from, allocate the KVA space 3034 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3035 * the 64-bit kernel. 3036 */ 3037 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3038 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3039 3040 /* convert the sleep flags */ 3041 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3042 vmflag = VM_SLEEP; 3043 } else { 3044 vmflag = VM_NOSLEEP; 3045 } 3046 3047 /* allocate Kernel VA space that we can bcopy to/from */ 3048 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3049 vmflag); 3050 if (dma->dp_kva == NULL) { 3051 return (DDI_DMA_NORESOURCES); 3052 } 3053 } 3054 #endif 3055 3056 /* convert the sleep flags */ 3057 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3058 cansleep = 1; 3059 } else { 3060 cansleep = 0; 3061 } 3062 3063 /* 3064 * Allocate the actual copy buffer. This needs to fit within the DMA 3065 * engine limits, so we can't use kmem_alloc... We don't need 3066 * contiguous memory (sgllen) since we will be forcing windows on 3067 * sgllen anyway. 3068 */ 3069 lattr = *attr; 3070 lattr.dma_attr_align = MMU_PAGESIZE; 3071 /* 3072 * this should be < 0 to indicate no limit, but due to a bug in 3073 * the rootnex, we'll set it to the maximum positive int. 3074 */ 3075 lattr.dma_attr_sgllen = 0x7fffffff; 3076 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3077 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3078 if (e != DDI_SUCCESS) { 3079 #if !defined(__amd64) 3080 if (dma->dp_kva != NULL) { 3081 vmem_free(heap_arena, dma->dp_kva, 3082 dma->dp_copybuf_size); 3083 } 3084 #endif 3085 return (DDI_DMA_NORESOURCES); 3086 } 3087 3088 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3089 size_t, dma->dp_copybuf_size); 3090 3091 return (DDI_SUCCESS); 3092 } 3093 3094 3095 /* 3096 * rootnex_setup_windows() 3097 * Called in bind slowpath to setup the window state. We always have windows 3098 * in the slowpath. Even if the window count = 1. 3099 */ 3100 static int 3101 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3102 ddi_dma_attr_t *attr, int kmflag) 3103 { 3104 rootnex_window_t *windowp; 3105 rootnex_sglinfo_t *sinfo; 3106 size_t copy_state_size; 3107 size_t win_state_size; 3108 size_t state_available; 3109 size_t space_needed; 3110 uint_t copybuf_win; 3111 uint_t maxxfer_win; 3112 size_t space_used; 3113 uint_t sglwin; 3114 3115 3116 sinfo = &dma->dp_sglinfo; 3117 3118 dma->dp_current_win = 0; 3119 hp->dmai_nwin = 0; 3120 3121 /* If we don't need to do a partial, we only have one window */ 3122 if (!dma->dp_partial_required) { 3123 dma->dp_max_win = 1; 3124 3125 /* 3126 * we need multiple windows, need to figure out the worse case number 3127 * of windows. 3128 */ 3129 } else { 3130 /* 3131 * if we need windows because we need more copy buffer that 3132 * we allow, the worse case number of windows we could need 3133 * here would be (copybuf space required / copybuf space that 3134 * we have) plus one for remainder, and plus 2 to handle the 3135 * extra pages on the trim for the first and last pages of the 3136 * buffer (a page is the minimum window size so under the right 3137 * attr settings, you could have a window for each page). 3138 * The last page will only be hit here if the size is not a 3139 * multiple of the granularity (which theoretically shouldn't 3140 * be the case but never has been enforced, so we could have 3141 * broken things without it). 3142 */ 3143 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3144 ASSERT(dma->dp_copybuf_size > 0); 3145 copybuf_win = (sinfo->si_copybuf_req / 3146 dma->dp_copybuf_size) + 1 + 2; 3147 } else { 3148 copybuf_win = 0; 3149 } 3150 3151 /* 3152 * if we need windows because we have more cookies than the H/W 3153 * can handle, the number of windows we would need here would 3154 * be (cookie count / cookies count H/W supports) plus one for 3155 * remainder, and plus 2 to handle the extra pages on the trim 3156 * (see above comment about trim) 3157 */ 3158 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3159 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 3160 + 1) + 2; 3161 } else { 3162 sglwin = 0; 3163 } 3164 3165 /* 3166 * if we need windows because we're binding more memory than the 3167 * H/W can transfer at once, the number of windows we would need 3168 * here would be (xfer count / max xfer H/W supports) plus one 3169 * for remainder, and plus 2 to handle the extra pages on the 3170 * trim (see above comment about trim) 3171 */ 3172 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3173 maxxfer_win = (dma->dp_dma.dmao_size / 3174 dma->dp_maxxfer) + 1 + 2; 3175 } else { 3176 maxxfer_win = 0; 3177 } 3178 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3179 ASSERT(dma->dp_max_win > 0); 3180 } 3181 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3182 3183 /* 3184 * Get space for window and potential copy buffer state. Before we 3185 * go and allocate memory, see if we can get away with using what's 3186 * left in the pre-allocted state or the dynamically allocated sgl. 3187 */ 3188 space_used = (uintptr_t)(sinfo->si_sgl_size * 3189 sizeof (ddi_dma_cookie_t)); 3190 3191 /* if we dynamically allocated space for the cookies */ 3192 if (dma->dp_need_to_free_cookie) { 3193 /* if we have more space in the pre-allocted buffer, use it */ 3194 ASSERT(space_used <= dma->dp_cookie_size); 3195 if ((dma->dp_cookie_size - space_used) <= 3196 rootnex_state->r_prealloc_size) { 3197 state_available = rootnex_state->r_prealloc_size; 3198 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3199 3200 /* 3201 * else, we have more free space in the dynamically allocated 3202 * buffer, i.e. the buffer wasn't worse case fragmented so we 3203 * didn't need a lot of cookies. 3204 */ 3205 } else { 3206 state_available = dma->dp_cookie_size - space_used; 3207 windowp = (rootnex_window_t *) 3208 &dma->dp_cookies[sinfo->si_sgl_size]; 3209 } 3210 3211 /* we used the pre-alloced buffer */ 3212 } else { 3213 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3214 state_available = rootnex_state->r_prealloc_size - space_used; 3215 windowp = (rootnex_window_t *) 3216 &dma->dp_cookies[sinfo->si_sgl_size]; 3217 } 3218 3219 /* 3220 * figure out how much state we need to track the copy buffer. Add an 3221 * addition 8 bytes for pointer alignemnt later. 3222 */ 3223 if (dma->dp_copybuf_size > 0) { 3224 copy_state_size = sinfo->si_max_pages * 3225 sizeof (rootnex_pgmap_t); 3226 } else { 3227 copy_state_size = 0; 3228 } 3229 /* add an additional 8 bytes for pointer alignment */ 3230 space_needed = win_state_size + copy_state_size + 0x8; 3231 3232 /* if we have enough space already, use it */ 3233 if (state_available >= space_needed) { 3234 dma->dp_window = windowp; 3235 dma->dp_need_to_free_window = B_FALSE; 3236 3237 /* not enough space, need to allocate more. */ 3238 } else { 3239 dma->dp_window = kmem_alloc(space_needed, kmflag); 3240 if (dma->dp_window == NULL) { 3241 return (DDI_DMA_NORESOURCES); 3242 } 3243 dma->dp_need_to_free_window = B_TRUE; 3244 dma->dp_window_size = space_needed; 3245 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3246 dma->dp_dip, size_t, space_needed); 3247 } 3248 3249 /* 3250 * we allocate copy buffer state and window state at the same time. 3251 * setup our copy buffer state pointers. Make sure it's aligned. 3252 */ 3253 if (dma->dp_copybuf_size > 0) { 3254 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3255 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3256 3257 #if !defined(__amd64) 3258 /* 3259 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3260 * false/NULL. Should be quicker to bzero vs loop and set. 3261 */ 3262 bzero(dma->dp_pgmap, copy_state_size); 3263 #endif 3264 } else { 3265 dma->dp_pgmap = NULL; 3266 } 3267 3268 return (DDI_SUCCESS); 3269 } 3270 3271 3272 /* 3273 * rootnex_teardown_copybuf() 3274 * cleans up after rootnex_setup_copybuf() 3275 */ 3276 static void 3277 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3278 { 3279 #if !defined(__amd64) 3280 int i; 3281 3282 /* 3283 * if we allocated kernel heap VMEM space, go through all the pages and 3284 * map out any of the ones that we're mapped into the kernel heap VMEM 3285 * arena. Then free the VMEM space. 3286 */ 3287 if (dma->dp_kva != NULL) { 3288 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3289 if (dma->dp_pgmap[i].pm_mapped) { 3290 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3291 MMU_PAGESIZE, HAT_UNLOAD); 3292 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3293 } 3294 } 3295 3296 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3297 } 3298 3299 #endif 3300 3301 /* if we allocated a copy buffer, free it */ 3302 if (dma->dp_cbaddr != NULL) { 3303 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3304 } 3305 } 3306 3307 3308 /* 3309 * rootnex_teardown_windows() 3310 * cleans up after rootnex_setup_windows() 3311 */ 3312 static void 3313 rootnex_teardown_windows(rootnex_dma_t *dma) 3314 { 3315 /* 3316 * if we had to allocate window state on the last bind (because we 3317 * didn't have enough pre-allocated space in the handle), free it. 3318 */ 3319 if (dma->dp_need_to_free_window) { 3320 kmem_free(dma->dp_window, dma->dp_window_size); 3321 } 3322 } 3323 3324 3325 /* 3326 * rootnex_init_win() 3327 * Called in bind slow path during creation of a new window. Initializes 3328 * window state to default values. 3329 */ 3330 /*ARGSUSED*/ 3331 static void 3332 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3333 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3334 { 3335 hp->dmai_nwin++; 3336 window->wd_dosync = B_FALSE; 3337 window->wd_offset = cur_offset; 3338 window->wd_size = 0; 3339 window->wd_first_cookie = cookie; 3340 window->wd_cookie_cnt = 0; 3341 window->wd_trim.tr_trim_first = B_FALSE; 3342 window->wd_trim.tr_trim_last = B_FALSE; 3343 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3344 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3345 #if !defined(__amd64) 3346 window->wd_remap_copybuf = dma->dp_cb_remaping; 3347 #endif 3348 } 3349 3350 3351 /* 3352 * rootnex_setup_cookie() 3353 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3354 * the sgl uses the copy buffer, we need to go through each cookie, figure 3355 * out if it uses the copy buffer, and if it does, save away everything we'll 3356 * need during sync. 3357 */ 3358 static void 3359 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3360 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3361 page_t **cur_pp) 3362 { 3363 boolean_t copybuf_sz_power_2; 3364 rootnex_sglinfo_t *sinfo; 3365 paddr_t paddr; 3366 uint_t pidx; 3367 uint_t pcnt; 3368 off_t poff; 3369 #if defined(__amd64) 3370 pfn_t pfn; 3371 #else 3372 page_t **pplist; 3373 #endif 3374 3375 sinfo = &dma->dp_sglinfo; 3376 3377 /* 3378 * Calculate the page index relative to the start of the buffer. The 3379 * index to the current page for our buffer is the offset into the 3380 * first page of the buffer plus our current offset into the buffer 3381 * itself, shifted of course... 3382 */ 3383 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3384 ASSERT(pidx < sinfo->si_max_pages); 3385 3386 /* if this cookie uses the copy buffer */ 3387 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3388 /* 3389 * NOTE: we know that since this cookie uses the copy buffer, it 3390 * is <= MMU_PAGESIZE. 3391 */ 3392 3393 /* 3394 * get the offset into the page. For the 64-bit kernel, get the 3395 * pfn which we'll use with seg kpm. 3396 */ 3397 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3398 #if defined(__amd64) 3399 /* mfn_to_pfn() is a NOP on i86pc */ 3400 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3401 #endif /* __amd64 */ 3402 3403 /* figure out if the copybuf size is a power of 2 */ 3404 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3405 copybuf_sz_power_2 = B_FALSE; 3406 } else { 3407 copybuf_sz_power_2 = B_TRUE; 3408 } 3409 3410 /* This page uses the copy buffer */ 3411 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3412 3413 /* 3414 * save the copy buffer KVA that we'll use with this page. 3415 * if we still fit within the copybuf, it's a simple add. 3416 * otherwise, we need to wrap over using & or % accordingly. 3417 */ 3418 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3419 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3420 *copybuf_used; 3421 } else { 3422 if (copybuf_sz_power_2) { 3423 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3424 (uintptr_t)dma->dp_cbaddr + 3425 (*copybuf_used & 3426 (dma->dp_copybuf_size - 1))); 3427 } else { 3428 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3429 (uintptr_t)dma->dp_cbaddr + 3430 (*copybuf_used % dma->dp_copybuf_size)); 3431 } 3432 } 3433 3434 /* 3435 * over write the cookie physical address with the address of 3436 * the physical address of the copy buffer page that we will 3437 * use. 3438 */ 3439 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3440 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3441 3442 #ifdef __xpv 3443 /* 3444 * If we're dom0, we're using a real device so we need to load 3445 * the cookies with MAs instead of PAs. 3446 */ 3447 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3448 #else 3449 cookie->dmac_laddress = paddr; 3450 #endif 3451 3452 /* if we have a kernel VA, it's easy, just save that address */ 3453 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3454 (sinfo->si_asp == &kas)) { 3455 /* 3456 * save away the page aligned virtual address of the 3457 * driver buffer. Offsets are handled in the sync code. 3458 */ 3459 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3460 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3461 & MMU_PAGEMASK); 3462 #if !defined(__amd64) 3463 /* 3464 * we didn't need to, and will never need to map this 3465 * page. 3466 */ 3467 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3468 #endif 3469 3470 /* we don't have a kernel VA. We need one for the bcopy. */ 3471 } else { 3472 #if defined(__amd64) 3473 /* 3474 * for the 64-bit kernel, it's easy. We use seg kpm to 3475 * get a Kernel VA for the corresponding pfn. 3476 */ 3477 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3478 #else 3479 /* 3480 * for the 32-bit kernel, this is a pain. First we'll 3481 * save away the page_t or user VA for this page. This 3482 * is needed in rootnex_dma_win() when we switch to a 3483 * new window which requires us to re-map the copy 3484 * buffer. 3485 */ 3486 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3487 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3488 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3489 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3490 } else if (pplist != NULL) { 3491 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3492 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3493 } else { 3494 dma->dp_pgmap[pidx].pm_pp = NULL; 3495 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3496 (((uintptr_t) 3497 dmar_object->dmao_obj.virt_obj.v_addr + 3498 cur_offset) & MMU_PAGEMASK); 3499 } 3500 3501 /* 3502 * save away the page aligned virtual address which was 3503 * allocated from the kernel heap arena (taking into 3504 * account if we need more copy buffer than we alloced 3505 * and use multiple windows to handle this, i.e. &,%). 3506 * NOTE: there isn't and physical memory backing up this 3507 * virtual address space currently. 3508 */ 3509 if ((*copybuf_used + MMU_PAGESIZE) <= 3510 dma->dp_copybuf_size) { 3511 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3512 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3513 MMU_PAGEMASK); 3514 } else { 3515 if (copybuf_sz_power_2) { 3516 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3517 (((uintptr_t)dma->dp_kva + 3518 (*copybuf_used & 3519 (dma->dp_copybuf_size - 1))) & 3520 MMU_PAGEMASK); 3521 } else { 3522 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3523 (((uintptr_t)dma->dp_kva + 3524 (*copybuf_used % 3525 dma->dp_copybuf_size)) & 3526 MMU_PAGEMASK); 3527 } 3528 } 3529 3530 /* 3531 * if we haven't used up the available copy buffer yet, 3532 * map the kva to the physical page. 3533 */ 3534 if (!dma->dp_cb_remaping && ((*copybuf_used + 3535 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3536 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3537 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3538 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3539 dma->dp_pgmap[pidx].pm_kaddr); 3540 } else { 3541 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3542 sinfo->si_asp, 3543 dma->dp_pgmap[pidx].pm_kaddr); 3544 } 3545 3546 /* 3547 * we've used up the available copy buffer, this page 3548 * will have to be mapped during rootnex_dma_win() when 3549 * we switch to a new window which requires a re-map 3550 * the copy buffer. (32-bit kernel only) 3551 */ 3552 } else { 3553 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3554 } 3555 #endif 3556 /* go to the next page_t */ 3557 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3558 *cur_pp = (*cur_pp)->p_next; 3559 } 3560 } 3561 3562 /* add to the copy buffer count */ 3563 *copybuf_used += MMU_PAGESIZE; 3564 3565 /* 3566 * This cookie doesn't use the copy buffer. Walk through the pages this 3567 * cookie occupies to reflect this. 3568 */ 3569 } else { 3570 /* 3571 * figure out how many pages the cookie occupies. We need to 3572 * use the original page offset of the buffer and the cookies 3573 * offset in the buffer to do this. 3574 */ 3575 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3576 pcnt = mmu_btopr(cookie->dmac_size + poff); 3577 3578 while (pcnt > 0) { 3579 #if !defined(__amd64) 3580 /* 3581 * the 32-bit kernel doesn't have seg kpm, so we need 3582 * to map in the driver buffer (if it didn't come down 3583 * with a kernel VA) on the fly. Since this page doesn't 3584 * use the copy buffer, it's not, or will it ever, have 3585 * to be mapped in. 3586 */ 3587 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3588 #endif 3589 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3590 3591 /* 3592 * we need to update pidx and cur_pp or we'll loose 3593 * track of where we are. 3594 */ 3595 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3596 *cur_pp = (*cur_pp)->p_next; 3597 } 3598 pidx++; 3599 pcnt--; 3600 } 3601 } 3602 } 3603 3604 3605 /* 3606 * rootnex_sgllen_window_boundary() 3607 * Called in the bind slow path when the next cookie causes us to exceed (in 3608 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3609 * length supported by the DMA H/W. 3610 */ 3611 static int 3612 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3613 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3614 off_t cur_offset) 3615 { 3616 off_t new_offset; 3617 size_t trim_sz; 3618 off_t coffset; 3619 3620 3621 /* 3622 * if we know we'll never have to trim, it's pretty easy. Just move to 3623 * the next window and init it. We're done. 3624 */ 3625 if (!dma->dp_trim_required) { 3626 (*windowp)++; 3627 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3628 (*windowp)->wd_cookie_cnt++; 3629 (*windowp)->wd_size = cookie->dmac_size; 3630 return (DDI_SUCCESS); 3631 } 3632 3633 /* figure out how much we need to trim from the window */ 3634 ASSERT(attr->dma_attr_granular != 0); 3635 if (dma->dp_granularity_power_2) { 3636 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3637 } else { 3638 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3639 } 3640 3641 /* The window's a whole multiple of granularity. We're done */ 3642 if (trim_sz == 0) { 3643 (*windowp)++; 3644 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3645 (*windowp)->wd_cookie_cnt++; 3646 (*windowp)->wd_size = cookie->dmac_size; 3647 return (DDI_SUCCESS); 3648 } 3649 3650 /* 3651 * The window's not a whole multiple of granularity, since we know this 3652 * is due to the sgllen, we need to go back to the last cookie and trim 3653 * that one, add the left over part of the old cookie into the new 3654 * window, and then add in the new cookie into the new window. 3655 */ 3656 3657 /* 3658 * make sure the driver isn't making us do something bad... Trimming and 3659 * sgllen == 1 don't go together. 3660 */ 3661 if (attr->dma_attr_sgllen == 1) { 3662 return (DDI_DMA_NOMAPPING); 3663 } 3664 3665 /* 3666 * first, setup the current window to account for the trim. Need to go 3667 * back to the last cookie for this. 3668 */ 3669 cookie--; 3670 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3671 (*windowp)->wd_trim.tr_last_cookie = cookie; 3672 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3673 ASSERT(cookie->dmac_size > trim_sz); 3674 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3675 (*windowp)->wd_size -= trim_sz; 3676 3677 /* save the buffer offsets for the next window */ 3678 coffset = cookie->dmac_size - trim_sz; 3679 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3680 3681 /* 3682 * set this now in case this is the first window. all other cases are 3683 * set in dma_win() 3684 */ 3685 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3686 3687 /* 3688 * initialize the next window using what's left over in the previous 3689 * cookie. 3690 */ 3691 (*windowp)++; 3692 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3693 (*windowp)->wd_cookie_cnt++; 3694 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3695 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3696 (*windowp)->wd_trim.tr_first_size = trim_sz; 3697 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3698 (*windowp)->wd_dosync = B_TRUE; 3699 } 3700 3701 /* 3702 * now go back to the current cookie and add it to the new window. set 3703 * the new window size to the what was left over from the previous 3704 * cookie and what's in the current cookie. 3705 */ 3706 cookie++; 3707 (*windowp)->wd_cookie_cnt++; 3708 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3709 3710 /* 3711 * trim plus the next cookie could put us over maxxfer (a cookie can be 3712 * a max size of maxxfer). Handle that case. 3713 */ 3714 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3715 /* 3716 * maxxfer is already a whole multiple of granularity, and this 3717 * trim will be <= the previous trim (since a cookie can't be 3718 * larger than maxxfer). Make things simple here. 3719 */ 3720 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3721 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3722 (*windowp)->wd_trim.tr_last_cookie = cookie; 3723 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3724 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3725 (*windowp)->wd_size -= trim_sz; 3726 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3727 3728 /* save the buffer offsets for the next window */ 3729 coffset = cookie->dmac_size - trim_sz; 3730 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3731 3732 /* setup the next window */ 3733 (*windowp)++; 3734 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3735 (*windowp)->wd_cookie_cnt++; 3736 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3737 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3738 coffset; 3739 (*windowp)->wd_trim.tr_first_size = trim_sz; 3740 } 3741 3742 return (DDI_SUCCESS); 3743 } 3744 3745 3746 /* 3747 * rootnex_copybuf_window_boundary() 3748 * Called in bind slowpath when we get to a window boundary because we used 3749 * up all the copy buffer that we have. 3750 */ 3751 static int 3752 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3753 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3754 size_t *copybuf_used) 3755 { 3756 rootnex_sglinfo_t *sinfo; 3757 off_t new_offset; 3758 size_t trim_sz; 3759 paddr_t paddr; 3760 off_t coffset; 3761 uint_t pidx; 3762 off_t poff; 3763 3764 3765 sinfo = &dma->dp_sglinfo; 3766 3767 /* 3768 * the copy buffer should be a whole multiple of page size. We know that 3769 * this cookie is <= MMU_PAGESIZE. 3770 */ 3771 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3772 3773 /* 3774 * from now on, all new windows in this bind need to be re-mapped during 3775 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3776 * space... 3777 */ 3778 #if !defined(__amd64) 3779 dma->dp_cb_remaping = B_TRUE; 3780 #endif 3781 3782 /* reset copybuf used */ 3783 *copybuf_used = 0; 3784 3785 /* 3786 * if we don't have to trim (since granularity is set to 1), go to the 3787 * next window and add the current cookie to it. We know the current 3788 * cookie uses the copy buffer since we're in this code path. 3789 */ 3790 if (!dma->dp_trim_required) { 3791 (*windowp)++; 3792 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3793 3794 /* Add this cookie to the new window */ 3795 (*windowp)->wd_cookie_cnt++; 3796 (*windowp)->wd_size += cookie->dmac_size; 3797 *copybuf_used += MMU_PAGESIZE; 3798 return (DDI_SUCCESS); 3799 } 3800 3801 /* 3802 * *** may need to trim, figure it out. 3803 */ 3804 3805 /* figure out how much we need to trim from the window */ 3806 if (dma->dp_granularity_power_2) { 3807 trim_sz = (*windowp)->wd_size & 3808 (hp->dmai_attr.dma_attr_granular - 1); 3809 } else { 3810 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3811 } 3812 3813 /* 3814 * if the window's a whole multiple of granularity, go to the next 3815 * window, init it, then add in the current cookie. We know the current 3816 * cookie uses the copy buffer since we're in this code path. 3817 */ 3818 if (trim_sz == 0) { 3819 (*windowp)++; 3820 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3821 3822 /* Add this cookie to the new window */ 3823 (*windowp)->wd_cookie_cnt++; 3824 (*windowp)->wd_size += cookie->dmac_size; 3825 *copybuf_used += MMU_PAGESIZE; 3826 return (DDI_SUCCESS); 3827 } 3828 3829 /* 3830 * *** We figured it out, we definitly need to trim 3831 */ 3832 3833 /* 3834 * make sure the driver isn't making us do something bad... 3835 * Trimming and sgllen == 1 don't go together. 3836 */ 3837 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3838 return (DDI_DMA_NOMAPPING); 3839 } 3840 3841 /* 3842 * first, setup the current window to account for the trim. Need to go 3843 * back to the last cookie for this. Some of the last cookie will be in 3844 * the current window, and some of the last cookie will be in the new 3845 * window. All of the current cookie will be in the new window. 3846 */ 3847 cookie--; 3848 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3849 (*windowp)->wd_trim.tr_last_cookie = cookie; 3850 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3851 ASSERT(cookie->dmac_size > trim_sz); 3852 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3853 (*windowp)->wd_size -= trim_sz; 3854 3855 /* 3856 * we're trimming the last cookie (not the current cookie). So that 3857 * last cookie may have or may not have been using the copy buffer ( 3858 * we know the cookie passed in uses the copy buffer since we're in 3859 * this code path). 3860 * 3861 * If the last cookie doesn't use the copy buffer, nothing special to 3862 * do. However, if it does uses the copy buffer, it will be both the 3863 * last page in the current window and the first page in the next 3864 * window. Since we are reusing the copy buffer (and KVA space on the 3865 * 32-bit kernel), this page will use the end of the copy buffer in the 3866 * current window, and the start of the copy buffer in the next window. 3867 * Track that info... The cookie physical address was already set to 3868 * the copy buffer physical address in setup_cookie.. 3869 */ 3870 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3871 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3872 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3873 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3874 (*windowp)->wd_trim.tr_last_pidx = pidx; 3875 (*windowp)->wd_trim.tr_last_cbaddr = 3876 dma->dp_pgmap[pidx].pm_cbaddr; 3877 #if !defined(__amd64) 3878 (*windowp)->wd_trim.tr_last_kaddr = 3879 dma->dp_pgmap[pidx].pm_kaddr; 3880 #endif 3881 } 3882 3883 /* save the buffer offsets for the next window */ 3884 coffset = cookie->dmac_size - trim_sz; 3885 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3886 3887 /* 3888 * set this now in case this is the first window. all other cases are 3889 * set in dma_win() 3890 */ 3891 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3892 3893 /* 3894 * initialize the next window using what's left over in the previous 3895 * cookie. 3896 */ 3897 (*windowp)++; 3898 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3899 (*windowp)->wd_cookie_cnt++; 3900 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3901 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3902 (*windowp)->wd_trim.tr_first_size = trim_sz; 3903 3904 /* 3905 * again, we're tracking if the last cookie uses the copy buffer. 3906 * read the comment above for more info on why we need to track 3907 * additional state. 3908 * 3909 * For the first cookie in the new window, we need reset the physical 3910 * address to DMA into to the start of the copy buffer plus any 3911 * initial page offset which may be present. 3912 */ 3913 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3914 (*windowp)->wd_dosync = B_TRUE; 3915 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3916 (*windowp)->wd_trim.tr_first_pidx = pidx; 3917 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3918 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3919 3920 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3921 poff; 3922 #ifdef __xpv 3923 /* 3924 * If we're dom0, we're using a real device so we need to load 3925 * the cookies with MAs instead of PAs. 3926 */ 3927 (*windowp)->wd_trim.tr_first_paddr = 3928 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3929 #else 3930 (*windowp)->wd_trim.tr_first_paddr = paddr; 3931 #endif 3932 3933 #if !defined(__amd64) 3934 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3935 #endif 3936 /* account for the cookie copybuf usage in the new window */ 3937 *copybuf_used += MMU_PAGESIZE; 3938 3939 /* 3940 * every piece of code has to have a hack, and here is this 3941 * ones :-) 3942 * 3943 * There is a complex interaction between setup_cookie and the 3944 * copybuf window boundary. The complexity had to be in either 3945 * the maxxfer window, or the copybuf window, and I chose the 3946 * copybuf code. 3947 * 3948 * So in this code path, we have taken the last cookie, 3949 * virtually broken it in half due to the trim, and it happens 3950 * to use the copybuf which further complicates life. At the 3951 * same time, we have already setup the current cookie, which 3952 * is now wrong. More background info: the current cookie uses 3953 * the copybuf, so it is only a page long max. So we need to 3954 * fix the current cookies copy buffer address, physical 3955 * address, and kva for the 32-bit kernel. We due this by 3956 * bumping them by page size (of course, we can't due this on 3957 * the physical address since the copy buffer may not be 3958 * physically contiguous). 3959 */ 3960 cookie++; 3961 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3962 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3963 3964 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3965 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3966 #ifdef __xpv 3967 /* 3968 * If we're dom0, we're using a real device so we need to load 3969 * the cookies with MAs instead of PAs. 3970 */ 3971 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3972 #else 3973 cookie->dmac_laddress = paddr; 3974 #endif 3975 3976 #if !defined(__amd64) 3977 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3978 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3979 #endif 3980 } else { 3981 /* go back to the current cookie */ 3982 cookie++; 3983 } 3984 3985 /* 3986 * add the current cookie to the new window. set the new window size to 3987 * the what was left over from the previous cookie and what's in the 3988 * current cookie. 3989 */ 3990 (*windowp)->wd_cookie_cnt++; 3991 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3992 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3993 3994 /* 3995 * we know that the cookie passed in always uses the copy buffer. We 3996 * wouldn't be here if it didn't. 3997 */ 3998 *copybuf_used += MMU_PAGESIZE; 3999 4000 return (DDI_SUCCESS); 4001 } 4002 4003 4004 /* 4005 * rootnex_maxxfer_window_boundary() 4006 * Called in bind slowpath when we get to a window boundary because we will 4007 * go over maxxfer. 4008 */ 4009 static int 4010 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4011 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4012 { 4013 size_t dmac_size; 4014 off_t new_offset; 4015 size_t trim_sz; 4016 off_t coffset; 4017 4018 4019 /* 4020 * calculate how much we have to trim off of the current cookie to equal 4021 * maxxfer. We don't have to account for granularity here since our 4022 * maxxfer already takes that into account. 4023 */ 4024 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4025 ASSERT(trim_sz <= cookie->dmac_size); 4026 ASSERT(trim_sz <= dma->dp_maxxfer); 4027 4028 /* save cookie size since we need it later and we might change it */ 4029 dmac_size = cookie->dmac_size; 4030 4031 /* 4032 * if we're not trimming the entire cookie, setup the current window to 4033 * account for the trim. 4034 */ 4035 if (trim_sz < cookie->dmac_size) { 4036 (*windowp)->wd_cookie_cnt++; 4037 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4038 (*windowp)->wd_trim.tr_last_cookie = cookie; 4039 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4040 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4041 (*windowp)->wd_size = dma->dp_maxxfer; 4042 4043 /* 4044 * set the adjusted cookie size now in case this is the first 4045 * window. All other windows are taken care of in get win 4046 */ 4047 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4048 } 4049 4050 /* 4051 * coffset is the current offset within the cookie, new_offset is the 4052 * current offset with the entire buffer. 4053 */ 4054 coffset = dmac_size - trim_sz; 4055 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4056 4057 /* initialize the next window */ 4058 (*windowp)++; 4059 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4060 (*windowp)->wd_cookie_cnt++; 4061 (*windowp)->wd_size = trim_sz; 4062 if (trim_sz < dmac_size) { 4063 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4064 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4065 coffset; 4066 (*windowp)->wd_trim.tr_first_size = trim_sz; 4067 } 4068 4069 return (DDI_SUCCESS); 4070 } 4071 4072 4073 /*ARGSUSED*/ 4074 static int 4075 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4076 off_t off, size_t len, uint_t cache_flags) 4077 { 4078 rootnex_sglinfo_t *sinfo; 4079 rootnex_pgmap_t *cbpage; 4080 rootnex_window_t *win; 4081 ddi_dma_impl_t *hp; 4082 rootnex_dma_t *dma; 4083 caddr_t fromaddr; 4084 caddr_t toaddr; 4085 uint_t psize; 4086 off_t offset; 4087 uint_t pidx; 4088 size_t size; 4089 off_t poff; 4090 int e; 4091 4092 4093 hp = (ddi_dma_impl_t *)handle; 4094 dma = (rootnex_dma_t *)hp->dmai_private; 4095 sinfo = &dma->dp_sglinfo; 4096 4097 /* 4098 * if we don't have any windows, we don't need to sync. A copybuf 4099 * will cause us to have at least one window. 4100 */ 4101 if (dma->dp_window == NULL) { 4102 return (DDI_SUCCESS); 4103 } 4104 4105 /* This window may not need to be sync'd */ 4106 win = &dma->dp_window[dma->dp_current_win]; 4107 if (!win->wd_dosync) { 4108 return (DDI_SUCCESS); 4109 } 4110 4111 /* handle off and len special cases */ 4112 if ((off == 0) || (rootnex_sync_ignore_params)) { 4113 offset = win->wd_offset; 4114 } else { 4115 offset = off; 4116 } 4117 if ((len == 0) || (rootnex_sync_ignore_params)) { 4118 size = win->wd_size; 4119 } else { 4120 size = len; 4121 } 4122 4123 /* check the sync args to make sure they make a little sense */ 4124 if (rootnex_sync_check_parms) { 4125 e = rootnex_valid_sync_parms(hp, win, offset, size, 4126 cache_flags); 4127 if (e != DDI_SUCCESS) { 4128 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4129 return (DDI_FAILURE); 4130 } 4131 } 4132 4133 /* 4134 * special case the first page to handle the offset into the page. The 4135 * offset to the current page for our buffer is the offset into the 4136 * first page of the buffer plus our current offset into the buffer 4137 * itself, masked of course. 4138 */ 4139 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4140 psize = MIN((MMU_PAGESIZE - poff), size); 4141 4142 /* go through all the pages that we want to sync */ 4143 while (size > 0) { 4144 /* 4145 * Calculate the page index relative to the start of the buffer. 4146 * The index to the current page for our buffer is the offset 4147 * into the first page of the buffer plus our current offset 4148 * into the buffer itself, shifted of course... 4149 */ 4150 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4151 ASSERT(pidx < sinfo->si_max_pages); 4152 4153 /* 4154 * if this page uses the copy buffer, we need to sync it, 4155 * otherwise, go on to the next page. 4156 */ 4157 cbpage = &dma->dp_pgmap[pidx]; 4158 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4159 (cbpage->pm_uses_copybuf == B_FALSE)); 4160 if (cbpage->pm_uses_copybuf) { 4161 /* cbaddr and kaddr should be page aligned */ 4162 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4163 MMU_PAGEOFFSET) == 0); 4164 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4165 MMU_PAGEOFFSET) == 0); 4166 4167 /* 4168 * if we're copying for the device, we are going to 4169 * copy from the drivers buffer and to the rootnex 4170 * allocated copy buffer. 4171 */ 4172 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4173 fromaddr = cbpage->pm_kaddr + poff; 4174 toaddr = cbpage->pm_cbaddr + poff; 4175 DTRACE_PROBE2(rootnex__sync__dev, 4176 dev_info_t *, dma->dp_dip, size_t, psize); 4177 4178 /* 4179 * if we're copying for the cpu/kernel, we are going to 4180 * copy from the rootnex allocated copy buffer to the 4181 * drivers buffer. 4182 */ 4183 } else { 4184 fromaddr = cbpage->pm_cbaddr + poff; 4185 toaddr = cbpage->pm_kaddr + poff; 4186 DTRACE_PROBE2(rootnex__sync__cpu, 4187 dev_info_t *, dma->dp_dip, size_t, psize); 4188 } 4189 4190 bcopy(fromaddr, toaddr, psize); 4191 } 4192 4193 /* 4194 * decrement size until we're done, update our offset into the 4195 * buffer, and get the next page size. 4196 */ 4197 size -= psize; 4198 offset += psize; 4199 psize = MIN(MMU_PAGESIZE, size); 4200 4201 /* page offset is zero for the rest of this loop */ 4202 poff = 0; 4203 } 4204 4205 return (DDI_SUCCESS); 4206 } 4207 4208 /* 4209 * rootnex_dma_sync() 4210 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4211 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4212 * is set, ddi_dma_sync() returns immediately passing back success. 4213 */ 4214 /*ARGSUSED*/ 4215 static int 4216 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4217 off_t off, size_t len, uint_t cache_flags) 4218 { 4219 #if defined(__amd64) && !defined(__xpv) 4220 if (IOMMU_USED(rdip)) { 4221 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4222 cache_flags)); 4223 } 4224 #endif 4225 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4226 cache_flags)); 4227 } 4228 4229 /* 4230 * rootnex_valid_sync_parms() 4231 * checks the parameters passed to sync to verify they are correct. 4232 */ 4233 static int 4234 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4235 off_t offset, size_t size, uint_t cache_flags) 4236 { 4237 off_t woffset; 4238 4239 4240 /* 4241 * the first part of the test to make sure the offset passed in is 4242 * within the window. 4243 */ 4244 if (offset < win->wd_offset) { 4245 return (DDI_FAILURE); 4246 } 4247 4248 /* 4249 * second and last part of the test to make sure the offset and length 4250 * passed in is within the window. 4251 */ 4252 woffset = offset - win->wd_offset; 4253 if ((woffset + size) > win->wd_size) { 4254 return (DDI_FAILURE); 4255 } 4256 4257 /* 4258 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4259 * be set too. 4260 */ 4261 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4262 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4263 return (DDI_SUCCESS); 4264 } 4265 4266 /* 4267 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4268 * should be set. Also DDI_DMA_READ should be set in the flags. 4269 */ 4270 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4271 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4272 (hp->dmai_rflags & DDI_DMA_READ)) { 4273 return (DDI_SUCCESS); 4274 } 4275 4276 return (DDI_FAILURE); 4277 } 4278 4279 4280 /*ARGSUSED*/ 4281 static int 4282 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4283 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4284 uint_t *ccountp) 4285 { 4286 rootnex_window_t *window; 4287 rootnex_trim_t *trim; 4288 ddi_dma_impl_t *hp; 4289 rootnex_dma_t *dma; 4290 #if !defined(__amd64) 4291 rootnex_sglinfo_t *sinfo; 4292 rootnex_pgmap_t *pmap; 4293 uint_t pidx; 4294 uint_t pcnt; 4295 off_t poff; 4296 int i; 4297 #endif 4298 4299 4300 hp = (ddi_dma_impl_t *)handle; 4301 dma = (rootnex_dma_t *)hp->dmai_private; 4302 #if !defined(__amd64) 4303 sinfo = &dma->dp_sglinfo; 4304 #endif 4305 4306 /* If we try and get a window which doesn't exist, return failure */ 4307 if (win >= hp->dmai_nwin) { 4308 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4309 return (DDI_FAILURE); 4310 } 4311 4312 /* 4313 * if we don't have any windows, and they're asking for the first 4314 * window, setup the cookie pointer to the first cookie in the bind. 4315 * setup our return values, then increment the cookie since we return 4316 * the first cookie on the stack. 4317 */ 4318 if (dma->dp_window == NULL) { 4319 if (win != 0) { 4320 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4321 return (DDI_FAILURE); 4322 } 4323 hp->dmai_cookie = dma->dp_cookies; 4324 *offp = 0; 4325 *lenp = dma->dp_dma.dmao_size; 4326 *ccountp = dma->dp_sglinfo.si_sgl_size; 4327 *cookiep = hp->dmai_cookie[0]; 4328 hp->dmai_cookie++; 4329 return (DDI_SUCCESS); 4330 } 4331 4332 /* sync the old window before moving on to the new one */ 4333 window = &dma->dp_window[dma->dp_current_win]; 4334 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4335 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4336 DDI_DMA_SYNC_FORCPU); 4337 } 4338 4339 #if !defined(__amd64) 4340 /* 4341 * before we move to the next window, if we need to re-map, unmap all 4342 * the pages in this window. 4343 */ 4344 if (dma->dp_cb_remaping) { 4345 /* 4346 * If we switch to this window again, we'll need to map in 4347 * on the fly next time. 4348 */ 4349 window->wd_remap_copybuf = B_TRUE; 4350 4351 /* 4352 * calculate the page index into the buffer where this window 4353 * starts, and the number of pages this window takes up. 4354 */ 4355 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4356 MMU_PAGESHIFT; 4357 poff = (sinfo->si_buf_offset + window->wd_offset) & 4358 MMU_PAGEOFFSET; 4359 pcnt = mmu_btopr(window->wd_size + poff); 4360 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4361 4362 /* unmap pages which are currently mapped in this window */ 4363 for (i = 0; i < pcnt; i++) { 4364 if (dma->dp_pgmap[pidx].pm_mapped) { 4365 hat_unload(kas.a_hat, 4366 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4367 HAT_UNLOAD); 4368 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4369 } 4370 pidx++; 4371 } 4372 } 4373 #endif 4374 4375 /* 4376 * Move to the new window. 4377 * NOTE: current_win must be set for sync to work right 4378 */ 4379 dma->dp_current_win = win; 4380 window = &dma->dp_window[win]; 4381 4382 /* if needed, adjust the first and/or last cookies for trim */ 4383 trim = &window->wd_trim; 4384 if (trim->tr_trim_first) { 4385 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4386 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4387 #if !defined(__amd64) 4388 window->wd_first_cookie->dmac_type = 4389 (window->wd_first_cookie->dmac_type & 4390 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4391 #endif 4392 if (trim->tr_first_copybuf_win) { 4393 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4394 trim->tr_first_cbaddr; 4395 #if !defined(__amd64) 4396 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4397 trim->tr_first_kaddr; 4398 #endif 4399 } 4400 } 4401 if (trim->tr_trim_last) { 4402 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4403 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4404 if (trim->tr_last_copybuf_win) { 4405 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4406 trim->tr_last_cbaddr; 4407 #if !defined(__amd64) 4408 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4409 trim->tr_last_kaddr; 4410 #endif 4411 } 4412 } 4413 4414 /* 4415 * setup the cookie pointer to the first cookie in the window. setup 4416 * our return values, then increment the cookie since we return the 4417 * first cookie on the stack. 4418 */ 4419 hp->dmai_cookie = window->wd_first_cookie; 4420 *offp = window->wd_offset; 4421 *lenp = window->wd_size; 4422 *ccountp = window->wd_cookie_cnt; 4423 *cookiep = hp->dmai_cookie[0]; 4424 hp->dmai_cookie++; 4425 4426 #if !defined(__amd64) 4427 /* re-map copybuf if required for this window */ 4428 if (dma->dp_cb_remaping) { 4429 /* 4430 * calculate the page index into the buffer where this 4431 * window starts. 4432 */ 4433 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4434 MMU_PAGESHIFT; 4435 ASSERT(pidx < sinfo->si_max_pages); 4436 4437 /* 4438 * the first page can get unmapped if it's shared with the 4439 * previous window. Even if the rest of this window is already 4440 * mapped in, we need to still check this one. 4441 */ 4442 pmap = &dma->dp_pgmap[pidx]; 4443 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4444 if (pmap->pm_pp != NULL) { 4445 pmap->pm_mapped = B_TRUE; 4446 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4447 } else if (pmap->pm_vaddr != NULL) { 4448 pmap->pm_mapped = B_TRUE; 4449 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4450 pmap->pm_kaddr); 4451 } 4452 } 4453 pidx++; 4454 4455 /* map in the rest of the pages if required */ 4456 if (window->wd_remap_copybuf) { 4457 window->wd_remap_copybuf = B_FALSE; 4458 4459 /* figure out many pages this window takes up */ 4460 poff = (sinfo->si_buf_offset + window->wd_offset) & 4461 MMU_PAGEOFFSET; 4462 pcnt = mmu_btopr(window->wd_size + poff); 4463 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4464 4465 /* map pages which require it */ 4466 for (i = 1; i < pcnt; i++) { 4467 pmap = &dma->dp_pgmap[pidx]; 4468 if (pmap->pm_uses_copybuf) { 4469 ASSERT(pmap->pm_mapped == B_FALSE); 4470 if (pmap->pm_pp != NULL) { 4471 pmap->pm_mapped = B_TRUE; 4472 i86_pp_map(pmap->pm_pp, 4473 pmap->pm_kaddr); 4474 } else if (pmap->pm_vaddr != NULL) { 4475 pmap->pm_mapped = B_TRUE; 4476 i86_va_map(pmap->pm_vaddr, 4477 sinfo->si_asp, 4478 pmap->pm_kaddr); 4479 } 4480 } 4481 pidx++; 4482 } 4483 } 4484 } 4485 #endif 4486 4487 /* if the new window uses the copy buffer, sync it for the device */ 4488 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4489 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4490 DDI_DMA_SYNC_FORDEV); 4491 } 4492 4493 return (DDI_SUCCESS); 4494 } 4495 4496 /* 4497 * rootnex_dma_win() 4498 * called from ddi_dma_getwin() 4499 */ 4500 /*ARGSUSED*/ 4501 static int 4502 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4503 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4504 uint_t *ccountp) 4505 { 4506 #if defined(__amd64) && !defined(__xpv) 4507 if (IOMMU_USED(rdip)) { 4508 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4509 cookiep, ccountp)); 4510 } 4511 #endif 4512 4513 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4514 cookiep, ccountp)); 4515 } 4516 4517 /* 4518 * ************************ 4519 * obsoleted dma routines 4520 * ************************ 4521 */ 4522 4523 /* 4524 * rootnex_dma_map() 4525 * called from ddi_dma_setup() 4526 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 4527 */ 4528 /* ARGSUSED */ 4529 static int 4530 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4531 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4532 { 4533 #if defined(__amd64) 4534 /* 4535 * this interface is not supported in 64-bit x86 kernel. See comment in 4536 * rootnex_dma_mctl() 4537 */ 4538 return (DDI_DMA_NORESOURCES); 4539 4540 #else /* 32-bit x86 kernel */ 4541 ddi_dma_handle_t *lhandlep; 4542 ddi_dma_handle_t lhandle; 4543 ddi_dma_cookie_t cookie; 4544 ddi_dma_attr_t dma_attr; 4545 ddi_dma_lim_t *dma_lim; 4546 uint_t ccnt; 4547 int e; 4548 4549 4550 /* 4551 * if the driver is just testing to see if it's possible to do the bind, 4552 * we'll use local state. Otherwise, use the handle pointer passed in. 4553 */ 4554 if (handlep == NULL) { 4555 lhandlep = &lhandle; 4556 } else { 4557 lhandlep = handlep; 4558 } 4559 4560 /* convert the limit structure to a dma_attr one */ 4561 dma_lim = dmareq->dmar_limits; 4562 dma_attr.dma_attr_version = DMA_ATTR_V0; 4563 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4564 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4565 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4566 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4567 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4568 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4569 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4570 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4571 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4572 dma_attr.dma_attr_align = MMU_PAGESIZE; 4573 dma_attr.dma_attr_flags = 0; 4574 4575 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4576 dmareq->dmar_arg, lhandlep); 4577 if (e != DDI_SUCCESS) { 4578 return (e); 4579 } 4580 4581 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4582 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4583 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4584 return (e); 4585 } 4586 4587 /* 4588 * if the driver is just testing to see if it's possible to do the bind, 4589 * free up the local state and return the result. 4590 */ 4591 if (handlep == NULL) { 4592 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4593 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4594 if (e == DDI_DMA_MAPPED) { 4595 return (DDI_DMA_MAPOK); 4596 } else { 4597 return (DDI_DMA_NOMAPPING); 4598 } 4599 } 4600 4601 return (e); 4602 #endif /* defined(__amd64) */ 4603 } 4604 4605 /* 4606 * rootnex_dma_mctl() 4607 * 4608 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 4609 */ 4610 /* ARGSUSED */ 4611 static int 4612 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4613 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4614 uint_t cache_flags) 4615 { 4616 #if defined(__amd64) 4617 /* 4618 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4619 * common implementation in genunix, so they no longer have x86 4620 * specific functionality which called into dma_ctl. 4621 * 4622 * The rest of the obsoleted interfaces were never supported in the 4623 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4624 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4625 * implementation issues. 4626 * 4627 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4628 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4629 * reflect that now too... 4630 * 4631 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4632 * not going to put this functionality into the 64-bit x86 kernel now. 4633 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4634 * that in a future release. 4635 */ 4636 return (DDI_FAILURE); 4637 4638 #else /* 32-bit x86 kernel */ 4639 ddi_dma_cookie_t lcookie; 4640 ddi_dma_cookie_t *cookie; 4641 rootnex_window_t *window; 4642 ddi_dma_impl_t *hp; 4643 rootnex_dma_t *dma; 4644 uint_t nwin; 4645 uint_t ccnt; 4646 size_t len; 4647 off_t off; 4648 int e; 4649 4650 4651 /* 4652 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4653 * hacky since were optimizing for the current interfaces and so we can 4654 * cleanup the mess in genunix. Hopefully we will remove the this 4655 * obsoleted routines someday soon. 4656 */ 4657 4658 switch (request) { 4659 4660 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4661 hp = (ddi_dma_impl_t *)handle; 4662 cookie = (ddi_dma_cookie_t *)objpp; 4663 4664 /* 4665 * convert segment to cookie. We don't distinguish between the 4666 * two :-) 4667 */ 4668 *cookie = *hp->dmai_cookie; 4669 *lenp = cookie->dmac_size; 4670 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4671 return (DDI_SUCCESS); 4672 4673 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4674 hp = (ddi_dma_impl_t *)handle; 4675 dma = (rootnex_dma_t *)hp->dmai_private; 4676 4677 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4678 return (DDI_DMA_STALE); 4679 } 4680 4681 /* handle the case where we don't have any windows */ 4682 if (dma->dp_window == NULL) { 4683 /* 4684 * if seg == NULL, and we don't have any windows, 4685 * return the first cookie in the sgl. 4686 */ 4687 if (*lenp == NULL) { 4688 dma->dp_current_cookie = 0; 4689 hp->dmai_cookie = dma->dp_cookies; 4690 *objpp = (caddr_t)handle; 4691 return (DDI_SUCCESS); 4692 4693 /* if we have more cookies, go to the next cookie */ 4694 } else { 4695 if ((dma->dp_current_cookie + 1) >= 4696 dma->dp_sglinfo.si_sgl_size) { 4697 return (DDI_DMA_DONE); 4698 } 4699 dma->dp_current_cookie++; 4700 hp->dmai_cookie++; 4701 return (DDI_SUCCESS); 4702 } 4703 } 4704 4705 /* We have one or more windows */ 4706 window = &dma->dp_window[dma->dp_current_win]; 4707 4708 /* 4709 * if seg == NULL, return the first cookie in the current 4710 * window 4711 */ 4712 if (*lenp == NULL) { 4713 dma->dp_current_cookie = 0; 4714 hp->dmai_cookie = window->wd_first_cookie; 4715 4716 /* 4717 * go to the next cookie in the window then see if we done with 4718 * this window. 4719 */ 4720 } else { 4721 if ((dma->dp_current_cookie + 1) >= 4722 window->wd_cookie_cnt) { 4723 return (DDI_DMA_DONE); 4724 } 4725 dma->dp_current_cookie++; 4726 hp->dmai_cookie++; 4727 } 4728 *objpp = (caddr_t)handle; 4729 return (DDI_SUCCESS); 4730 4731 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4732 hp = (ddi_dma_impl_t *)handle; 4733 dma = (rootnex_dma_t *)hp->dmai_private; 4734 4735 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4736 return (DDI_DMA_STALE); 4737 } 4738 4739 /* if win == NULL, return the first window in the bind */ 4740 if (*offp == NULL) { 4741 nwin = 0; 4742 4743 /* 4744 * else, go to the next window then see if we're done with all 4745 * the windows. 4746 */ 4747 } else { 4748 nwin = dma->dp_current_win + 1; 4749 if (nwin >= hp->dmai_nwin) { 4750 return (DDI_DMA_DONE); 4751 } 4752 } 4753 4754 /* switch to the next window */ 4755 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4756 &lcookie, &ccnt); 4757 ASSERT(e == DDI_SUCCESS); 4758 if (e != DDI_SUCCESS) { 4759 return (DDI_DMA_STALE); 4760 } 4761 4762 /* reset the cookie back to the first cookie in the window */ 4763 if (dma->dp_window != NULL) { 4764 window = &dma->dp_window[dma->dp_current_win]; 4765 hp->dmai_cookie = window->wd_first_cookie; 4766 } else { 4767 hp->dmai_cookie = dma->dp_cookies; 4768 } 4769 4770 *objpp = (caddr_t)handle; 4771 return (DDI_SUCCESS); 4772 4773 case DDI_DMA_FREE: /* ddi_dma_free() */ 4774 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4775 (void) rootnex_dma_freehdl(dip, rdip, handle); 4776 if (rootnex_state->r_dvma_call_list_id) { 4777 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4778 } 4779 return (DDI_SUCCESS); 4780 4781 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4782 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4783 /* should never get here, handled in genunix */ 4784 ASSERT(0); 4785 return (DDI_FAILURE); 4786 4787 case DDI_DMA_KVADDR: 4788 case DDI_DMA_GETERR: 4789 case DDI_DMA_COFF: 4790 return (DDI_FAILURE); 4791 } 4792 4793 return (DDI_FAILURE); 4794 #endif /* defined(__amd64) */ 4795 } 4796 4797 /* 4798 * ********* 4799 * FMA Code 4800 * ********* 4801 */ 4802 4803 /* 4804 * rootnex_fm_init() 4805 * FMA init busop 4806 */ 4807 /* ARGSUSED */ 4808 static int 4809 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4810 ddi_iblock_cookie_t *ibc) 4811 { 4812 *ibc = rootnex_state->r_err_ibc; 4813 4814 return (ddi_system_fmcap); 4815 } 4816 4817 /* 4818 * rootnex_dma_check() 4819 * Function called after a dma fault occurred to find out whether the 4820 * fault address is associated with a driver that is able to handle faults 4821 * and recover from faults. 4822 */ 4823 /* ARGSUSED */ 4824 static int 4825 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4826 const void *not_used) 4827 { 4828 rootnex_window_t *window; 4829 uint64_t start_addr; 4830 uint64_t fault_addr; 4831 ddi_dma_impl_t *hp; 4832 rootnex_dma_t *dma; 4833 uint64_t end_addr; 4834 size_t csize; 4835 int i; 4836 int j; 4837 4838 4839 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4840 hp = (ddi_dma_impl_t *)handle; 4841 ASSERT(hp); 4842 4843 dma = (rootnex_dma_t *)hp->dmai_private; 4844 4845 /* Get the address that we need to search for */ 4846 fault_addr = *(uint64_t *)addr; 4847 4848 /* 4849 * if we don't have any windows, we can just walk through all the 4850 * cookies. 4851 */ 4852 if (dma->dp_window == NULL) { 4853 /* for each cookie */ 4854 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4855 /* 4856 * if the faulted address is within the physical address 4857 * range of the cookie, return DDI_FM_NONFATAL. 4858 */ 4859 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4860 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4861 dma->dp_cookies[i].dmac_size))) { 4862 return (DDI_FM_NONFATAL); 4863 } 4864 } 4865 4866 /* fault_addr not within this DMA handle */ 4867 return (DDI_FM_UNKNOWN); 4868 } 4869 4870 /* we have mutiple windows, walk through each window */ 4871 for (i = 0; i < hp->dmai_nwin; i++) { 4872 window = &dma->dp_window[i]; 4873 4874 /* Go through all the cookies in the window */ 4875 for (j = 0; j < window->wd_cookie_cnt; j++) { 4876 4877 start_addr = window->wd_first_cookie[j].dmac_laddress; 4878 csize = window->wd_first_cookie[j].dmac_size; 4879 4880 /* 4881 * if we are trimming the first cookie in the window, 4882 * and this is the first cookie, adjust the start 4883 * address and size of the cookie to account for the 4884 * trim. 4885 */ 4886 if (window->wd_trim.tr_trim_first && (j == 0)) { 4887 start_addr = window->wd_trim.tr_first_paddr; 4888 csize = window->wd_trim.tr_first_size; 4889 } 4890 4891 /* 4892 * if we are trimming the last cookie in the window, 4893 * and this is the last cookie, adjust the start 4894 * address and size of the cookie to account for the 4895 * trim. 4896 */ 4897 if (window->wd_trim.tr_trim_last && 4898 (j == (window->wd_cookie_cnt - 1))) { 4899 start_addr = window->wd_trim.tr_last_paddr; 4900 csize = window->wd_trim.tr_last_size; 4901 } 4902 4903 end_addr = start_addr + csize; 4904 4905 /* 4906 * if the faulted address is within the physical 4907 * address of the cookie, return DDI_FM_NONFATAL. 4908 */ 4909 if ((fault_addr >= start_addr) && 4910 (fault_addr <= end_addr)) { 4911 return (DDI_FM_NONFATAL); 4912 } 4913 } 4914 } 4915 4916 /* fault_addr not within this DMA handle */ 4917 return (DDI_FM_UNKNOWN); 4918 } 4919 4920 /*ARGSUSED*/ 4921 static int 4922 rootnex_quiesce(dev_info_t *dip) 4923 { 4924 #if defined(__amd64) && !defined(__xpv) 4925 return (immu_quiesce()); 4926 #else 4927 return (DDI_SUCCESS); 4928 #endif 4929 } 4930 4931 #if defined(__xpv) 4932 void 4933 immu_init(void) 4934 { 4935 ; 4936 } 4937 4938 void 4939 immu_startup(void) 4940 { 4941 ; 4942 } 4943 /*ARGSUSED*/ 4944 void 4945 immu_physmem_update(uint64_t addr, uint64_t size) 4946 { 4947 ; 4948 } 4949 #endif 4950