1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #endif 71 72 #if defined(__amd64) && !defined(__xpv) 73 #include <sys/immu.h> 74 #endif 75 76 77 /* 78 * enable/disable extra checking of function parameters. Useful for debugging 79 * drivers. 80 */ 81 #ifdef DEBUG 82 int rootnex_alloc_check_parms = 1; 83 int rootnex_bind_check_parms = 1; 84 int rootnex_bind_check_inuse = 1; 85 int rootnex_unbind_verify_buffer = 0; 86 int rootnex_sync_check_parms = 1; 87 #else 88 int rootnex_alloc_check_parms = 0; 89 int rootnex_bind_check_parms = 0; 90 int rootnex_bind_check_inuse = 0; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 0; 93 #endif 94 95 boolean_t rootnex_dmar_not_setup; 96 97 /* Master Abort and Target Abort panic flag */ 98 int rootnex_fm_ma_ta_panic_flag = 0; 99 100 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 101 int rootnex_bind_fail = 1; 102 int rootnex_bind_warn = 1; 103 uint8_t *rootnex_warn_list; 104 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 105 #define ROOTNEX_BIND_WARNING (0x1 << 0) 106 107 /* 108 * revert back to old broken behavior of always sync'ing entire copy buffer. 109 * This is useful if be have a buggy driver which doesn't correctly pass in 110 * the offset and size into ddi_dma_sync(). 111 */ 112 int rootnex_sync_ignore_params = 0; 113 114 /* 115 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 116 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 117 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 118 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 119 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 120 * (< 8K). We will still need to allocate the copy buffer during bind though 121 * (if we need one). These can only be modified in /etc/system before rootnex 122 * attach. 123 */ 124 #if defined(__amd64) 125 int rootnex_prealloc_cookies = 65; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #else 129 int rootnex_prealloc_cookies = 33; 130 int rootnex_prealloc_windows = 4; 131 int rootnex_prealloc_copybuf = 2; 132 #endif 133 134 /* driver global state */ 135 static rootnex_state_t *rootnex_state; 136 137 /* shortcut to rootnex counters */ 138 static uint64_t *rootnex_cnt; 139 140 /* 141 * XXX - does x86 even need these or are they left over from the SPARC days? 142 */ 143 /* statically defined integer/boolean properties for the root node */ 144 static rootnex_intprop_t rootnex_intprp[] = { 145 { "PAGESIZE", PAGESIZE }, 146 { "MMU_PAGESIZE", MMU_PAGESIZE }, 147 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 148 { DDI_RELATIVE_ADDRESSING, 1 }, 149 }; 150 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 151 152 #ifdef __xpv 153 typedef maddr_t rootnex_addr_t; 154 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 155 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 156 #else 157 typedef paddr_t rootnex_addr_t; 158 #endif 159 160 #if !defined(__xpv) 161 char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica"; 162 #endif 163 164 static struct cb_ops rootnex_cb_ops = { 165 nodev, /* open */ 166 nodev, /* close */ 167 nodev, /* strategy */ 168 nodev, /* print */ 169 nodev, /* dump */ 170 nodev, /* read */ 171 nodev, /* write */ 172 nodev, /* ioctl */ 173 nodev, /* devmap */ 174 nodev, /* mmap */ 175 nodev, /* segmap */ 176 nochpoll, /* chpoll */ 177 ddi_prop_op, /* cb_prop_op */ 178 NULL, /* struct streamtab */ 179 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 180 CB_REV, /* Rev */ 181 nodev, /* cb_aread */ 182 nodev /* cb_awrite */ 183 }; 184 185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 186 off_t offset, off_t len, caddr_t *vaddrp); 187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 188 struct hat *hat, struct seg *seg, caddr_t addr, 189 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 190 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 191 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 192 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 193 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 194 ddi_dma_handle_t *handlep); 195 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 196 ddi_dma_handle_t handle); 197 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 198 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 199 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 200 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle); 202 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 204 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 207 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 209 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 210 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 211 ddi_ctl_enum_t ctlop, void *arg, void *result); 212 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 213 ddi_iblock_cookie_t *ibc); 214 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 215 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 216 217 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 218 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 219 ddi_dma_handle_t *handlep); 220 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 221 ddi_dma_handle_t handle); 222 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 223 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 224 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 225 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 226 ddi_dma_handle_t handle); 227 #if defined(__amd64) && !defined(__xpv) 228 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 229 ddi_dma_handle_t handle); 230 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 231 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 232 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 233 ddi_dma_cookie_t *cookiep, uint_t ccount); 234 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 235 ddi_dma_handle_t handle); 236 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 237 #endif 238 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 239 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 240 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 241 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 242 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 243 244 static struct bus_ops rootnex_bus_ops = { 245 BUSO_REV, 246 rootnex_map, 247 NULL, 248 NULL, 249 NULL, 250 rootnex_map_fault, 251 rootnex_dma_map, 252 rootnex_dma_allochdl, 253 rootnex_dma_freehdl, 254 rootnex_dma_bindhdl, 255 rootnex_dma_unbindhdl, 256 rootnex_dma_sync, 257 rootnex_dma_win, 258 rootnex_dma_mctl, 259 rootnex_ctlops, 260 ddi_bus_prop_op, 261 i_ddi_rootnex_get_eventcookie, 262 i_ddi_rootnex_add_eventcall, 263 i_ddi_rootnex_remove_eventcall, 264 i_ddi_rootnex_post_event, 265 0, /* bus_intr_ctl */ 266 0, /* bus_config */ 267 0, /* bus_unconfig */ 268 rootnex_fm_init, /* bus_fm_init */ 269 NULL, /* bus_fm_fini */ 270 NULL, /* bus_fm_access_enter */ 271 NULL, /* bus_fm_access_exit */ 272 NULL, /* bus_powr */ 273 rootnex_intr_ops /* bus_intr_op */ 274 }; 275 276 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 277 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 278 static int rootnex_quiesce(dev_info_t *dip); 279 280 static struct dev_ops rootnex_ops = { 281 DEVO_REV, 282 0, 283 ddi_no_info, 284 nulldev, 285 nulldev, 286 rootnex_attach, 287 rootnex_detach, 288 nulldev, 289 &rootnex_cb_ops, 290 &rootnex_bus_ops, 291 NULL, 292 rootnex_quiesce, /* quiesce */ 293 }; 294 295 static struct modldrv rootnex_modldrv = { 296 &mod_driverops, 297 "i86pc root nexus", 298 &rootnex_ops 299 }; 300 301 static struct modlinkage rootnex_modlinkage = { 302 MODREV_1, 303 (void *)&rootnex_modldrv, 304 NULL 305 }; 306 307 #if defined(__amd64) && !defined(__xpv) 308 static iommulib_nexops_t iommulib_nexops = { 309 IOMMU_NEXOPS_VERSION, 310 "Rootnex IOMMU ops Vers 1.1", 311 NULL, 312 rootnex_coredma_allochdl, 313 rootnex_coredma_freehdl, 314 rootnex_coredma_bindhdl, 315 rootnex_coredma_unbindhdl, 316 rootnex_coredma_reset_cookies, 317 rootnex_coredma_get_cookies, 318 rootnex_coredma_set_cookies, 319 rootnex_coredma_clear_cookies, 320 rootnex_coredma_get_sleep_flags, 321 rootnex_coredma_sync, 322 rootnex_coredma_win, 323 rootnex_dma_map, 324 rootnex_dma_mctl 325 }; 326 #endif 327 328 /* 329 * extern hacks 330 */ 331 extern struct seg_ops segdev_ops; 332 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 333 #ifdef DDI_MAP_DEBUG 334 extern int ddi_map_debug_flag; 335 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 336 #endif 337 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 338 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 339 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 340 psm_intr_op_t, int *); 341 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 342 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 343 344 /* 345 * Use device arena to use for device control register mappings. 346 * Various kernel memory walkers (debugger, dtrace) need to know 347 * to avoid this address range to prevent undesired device activity. 348 */ 349 extern void *device_arena_alloc(size_t size, int vm_flag); 350 extern void device_arena_free(void * vaddr, size_t size); 351 352 353 /* 354 * Internal functions 355 */ 356 static int rootnex_dma_init(); 357 static void rootnex_add_props(dev_info_t *); 358 static int rootnex_ctl_reportdev(dev_info_t *dip); 359 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 360 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 361 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 362 static int rootnex_map_handle(ddi_map_req_t *mp); 363 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 364 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 365 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 366 ddi_dma_attr_t *attr); 367 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 368 rootnex_sglinfo_t *sglinfo); 369 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 370 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 371 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 372 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 373 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 374 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 375 ddi_dma_attr_t *attr, int kmflag); 376 static void rootnex_teardown_windows(rootnex_dma_t *dma); 377 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 378 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 379 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 380 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 381 size_t *copybuf_used, page_t **cur_pp); 382 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 383 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 384 ddi_dma_attr_t *attr, off_t cur_offset); 385 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 386 rootnex_dma_t *dma, rootnex_window_t **windowp, 387 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 388 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 389 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 390 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 391 off_t offset, size_t size, uint_t cache_flags); 392 static int rootnex_verify_buffer(rootnex_dma_t *dma); 393 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 394 const void *comp_addr, const void *not_used); 395 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 396 rootnex_sglinfo_t *sglinfo); 397 398 /* 399 * _init() 400 * 401 */ 402 int 403 _init(void) 404 { 405 406 rootnex_state = NULL; 407 return (mod_install(&rootnex_modlinkage)); 408 } 409 410 411 /* 412 * _info() 413 * 414 */ 415 int 416 _info(struct modinfo *modinfop) 417 { 418 return (mod_info(&rootnex_modlinkage, modinfop)); 419 } 420 421 422 /* 423 * _fini() 424 * 425 */ 426 int 427 _fini(void) 428 { 429 return (EBUSY); 430 } 431 432 433 /* 434 * rootnex_attach() 435 * 436 */ 437 static int 438 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 439 { 440 int fmcap; 441 int e; 442 443 switch (cmd) { 444 case DDI_ATTACH: 445 break; 446 case DDI_RESUME: 447 #if defined(__amd64) && !defined(__xpv) 448 return (immu_unquiesce()); 449 #else 450 return (DDI_SUCCESS); 451 #endif 452 default: 453 return (DDI_FAILURE); 454 } 455 456 /* 457 * We should only have one instance of rootnex. Save it away since we 458 * don't have an easy way to get it back later. 459 */ 460 ASSERT(rootnex_state == NULL); 461 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 462 463 rootnex_state->r_dip = dip; 464 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 465 rootnex_state->r_reserved_msg_printed = B_FALSE; 466 rootnex_cnt = &rootnex_state->r_counters[0]; 467 468 /* 469 * Set minimum fm capability level for i86pc platforms and then 470 * initialize error handling. Since we're the rootnex, we don't 471 * care what's returned in the fmcap field. 472 */ 473 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 474 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 475 fmcap = ddi_system_fmcap; 476 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 477 478 /* initialize DMA related state */ 479 e = rootnex_dma_init(); 480 if (e != DDI_SUCCESS) { 481 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 482 return (DDI_FAILURE); 483 } 484 485 /* Add static root node properties */ 486 rootnex_add_props(dip); 487 488 /* since we can't call ddi_report_dev() */ 489 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 490 491 /* Initialize rootnex event handle */ 492 i_ddi_rootnex_init_events(dip); 493 494 #if defined(__amd64) && !defined(__xpv) 495 e = iommulib_nexus_register(dip, &iommulib_nexops, 496 &rootnex_state->r_iommulib_handle); 497 498 ASSERT(e == DDI_SUCCESS); 499 #endif 500 501 return (DDI_SUCCESS); 502 } 503 504 505 /* 506 * rootnex_detach() 507 * 508 */ 509 /*ARGSUSED*/ 510 static int 511 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 512 { 513 switch (cmd) { 514 case DDI_SUSPEND: 515 #if defined(__amd64) && !defined(__xpv) 516 return (immu_quiesce()); 517 #else 518 return (DDI_SUCCESS); 519 #endif 520 default: 521 return (DDI_FAILURE); 522 } 523 /*NOTREACHED*/ 524 525 } 526 527 528 /* 529 * rootnex_dma_init() 530 * 531 */ 532 /*ARGSUSED*/ 533 static int 534 rootnex_dma_init() 535 { 536 size_t bufsize; 537 538 539 /* 540 * size of our cookie/window/copybuf state needed in dma bind that we 541 * pre-alloc in dma_alloc_handle 542 */ 543 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 544 rootnex_state->r_prealloc_size = 545 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 546 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 547 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 548 549 /* 550 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 551 * allocate 16 extra bytes for struct pointer alignment 552 * (p->dmai_private & dma->dp_prealloc_buffer) 553 */ 554 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 555 rootnex_state->r_prealloc_size + 0x10; 556 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 557 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 558 if (rootnex_state->r_dmahdl_cache == NULL) { 559 return (DDI_FAILURE); 560 } 561 562 /* 563 * allocate array to track which major numbers we have printed warnings 564 * for. 565 */ 566 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 567 KM_SLEEP); 568 569 return (DDI_SUCCESS); 570 } 571 572 573 /* 574 * rootnex_add_props() 575 * 576 */ 577 static void 578 rootnex_add_props(dev_info_t *dip) 579 { 580 rootnex_intprop_t *rpp; 581 int i; 582 583 /* Add static integer/boolean properties to the root node */ 584 rpp = rootnex_intprp; 585 for (i = 0; i < NROOT_INTPROPS; i++) { 586 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 587 rpp[i].prop_name, rpp[i].prop_value); 588 } 589 } 590 591 592 593 /* 594 * ************************* 595 * ctlops related routines 596 * ************************* 597 */ 598 599 /* 600 * rootnex_ctlops() 601 * 602 */ 603 /*ARGSUSED*/ 604 static int 605 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 606 void *arg, void *result) 607 { 608 int n, *ptr; 609 struct ddi_parent_private_data *pdp; 610 611 switch (ctlop) { 612 case DDI_CTLOPS_DMAPMAPC: 613 /* 614 * Return 'partial' to indicate that dma mapping 615 * has to be done in the main MMU. 616 */ 617 return (DDI_DMA_PARTIAL); 618 619 case DDI_CTLOPS_BTOP: 620 /* 621 * Convert byte count input to physical page units. 622 * (byte counts that are not a page-size multiple 623 * are rounded down) 624 */ 625 *(ulong_t *)result = btop(*(ulong_t *)arg); 626 return (DDI_SUCCESS); 627 628 case DDI_CTLOPS_PTOB: 629 /* 630 * Convert size in physical pages to bytes 631 */ 632 *(ulong_t *)result = ptob(*(ulong_t *)arg); 633 return (DDI_SUCCESS); 634 635 case DDI_CTLOPS_BTOPR: 636 /* 637 * Convert byte count input to physical page units 638 * (byte counts that are not a page-size multiple 639 * are rounded up) 640 */ 641 *(ulong_t *)result = btopr(*(ulong_t *)arg); 642 return (DDI_SUCCESS); 643 644 case DDI_CTLOPS_INITCHILD: 645 return (impl_ddi_sunbus_initchild(arg)); 646 647 case DDI_CTLOPS_UNINITCHILD: 648 impl_ddi_sunbus_removechild(arg); 649 return (DDI_SUCCESS); 650 651 case DDI_CTLOPS_REPORTDEV: 652 return (rootnex_ctl_reportdev(rdip)); 653 654 case DDI_CTLOPS_IOMIN: 655 /* 656 * Nothing to do here but reflect back.. 657 */ 658 return (DDI_SUCCESS); 659 660 case DDI_CTLOPS_REGSIZE: 661 case DDI_CTLOPS_NREGS: 662 break; 663 664 case DDI_CTLOPS_SIDDEV: 665 if (ndi_dev_is_prom_node(rdip)) 666 return (DDI_SUCCESS); 667 if (ndi_dev_is_persistent_node(rdip)) 668 return (DDI_SUCCESS); 669 return (DDI_FAILURE); 670 671 case DDI_CTLOPS_POWER: 672 return ((*pm_platform_power)((power_req_t *)arg)); 673 674 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 675 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 676 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 677 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 678 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 679 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 680 if (!rootnex_state->r_reserved_msg_printed) { 681 rootnex_state->r_reserved_msg_printed = B_TRUE; 682 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 683 "1 or more reserved/obsolete operations."); 684 } 685 return (DDI_FAILURE); 686 687 default: 688 return (DDI_FAILURE); 689 } 690 /* 691 * The rest are for "hardware" properties 692 */ 693 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 694 return (DDI_FAILURE); 695 696 if (ctlop == DDI_CTLOPS_NREGS) { 697 ptr = (int *)result; 698 *ptr = pdp->par_nreg; 699 } else { 700 off_t *size = (off_t *)result; 701 702 ptr = (int *)arg; 703 n = *ptr; 704 if (n >= pdp->par_nreg) { 705 return (DDI_FAILURE); 706 } 707 *size = (off_t)pdp->par_reg[n].regspec_size; 708 } 709 return (DDI_SUCCESS); 710 } 711 712 713 /* 714 * rootnex_ctl_reportdev() 715 * 716 */ 717 static int 718 rootnex_ctl_reportdev(dev_info_t *dev) 719 { 720 int i, n, len, f_len = 0; 721 char *buf; 722 723 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 724 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 725 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 726 len = strlen(buf); 727 728 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 729 730 struct regspec *rp = sparc_pd_getreg(dev, i); 731 732 if (i == 0) 733 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 734 ": "); 735 else 736 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 737 " and "); 738 len = strlen(buf); 739 740 switch (rp->regspec_bustype) { 741 742 case BTEISA: 743 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 744 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 745 break; 746 747 case BTISA: 748 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 749 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 750 break; 751 752 default: 753 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 754 "space %x offset %x", 755 rp->regspec_bustype, rp->regspec_addr); 756 break; 757 } 758 len = strlen(buf); 759 } 760 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 761 int pri; 762 763 if (i != 0) { 764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 765 ","); 766 len = strlen(buf); 767 } 768 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 769 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 770 " sparc ipl %d", pri); 771 len = strlen(buf); 772 } 773 #ifdef DEBUG 774 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 775 cmn_err(CE_NOTE, "next message is truncated: " 776 "printed length 1024, real length %d", f_len); 777 } 778 #endif /* DEBUG */ 779 cmn_err(CE_CONT, "?%s\n", buf); 780 kmem_free(buf, REPORTDEV_BUFSIZE); 781 return (DDI_SUCCESS); 782 } 783 784 785 /* 786 * ****************** 787 * map related code 788 * ****************** 789 */ 790 791 /* 792 * rootnex_map() 793 * 794 */ 795 static int 796 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 797 off_t len, caddr_t *vaddrp) 798 { 799 struct regspec *rp, tmp_reg; 800 ddi_map_req_t mr = *mp; /* Get private copy of request */ 801 int error; 802 803 mp = &mr; 804 805 switch (mp->map_op) { 806 case DDI_MO_MAP_LOCKED: 807 case DDI_MO_UNMAP: 808 case DDI_MO_MAP_HANDLE: 809 break; 810 default: 811 #ifdef DDI_MAP_DEBUG 812 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 813 mp->map_op); 814 #endif /* DDI_MAP_DEBUG */ 815 return (DDI_ME_UNIMPLEMENTED); 816 } 817 818 if (mp->map_flags & DDI_MF_USER_MAPPING) { 819 #ifdef DDI_MAP_DEBUG 820 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 821 #endif /* DDI_MAP_DEBUG */ 822 return (DDI_ME_UNIMPLEMENTED); 823 } 824 825 /* 826 * First, if given an rnumber, convert it to a regspec... 827 * (Presumably, this is on behalf of a child of the root node?) 828 */ 829 830 if (mp->map_type == DDI_MT_RNUMBER) { 831 832 int rnumber = mp->map_obj.rnumber; 833 #ifdef DDI_MAP_DEBUG 834 static char *out_of_range = 835 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 836 #endif /* DDI_MAP_DEBUG */ 837 838 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 839 if (rp == NULL) { 840 #ifdef DDI_MAP_DEBUG 841 cmn_err(CE_WARN, out_of_range, rnumber, 842 ddi_get_name(rdip)); 843 #endif /* DDI_MAP_DEBUG */ 844 return (DDI_ME_RNUMBER_RANGE); 845 } 846 847 /* 848 * Convert the given ddi_map_req_t from rnumber to regspec... 849 */ 850 851 mp->map_type = DDI_MT_REGSPEC; 852 mp->map_obj.rp = rp; 853 } 854 855 /* 856 * Adjust offset and length correspnding to called values... 857 * XXX: A non-zero length means override the one in the regspec 858 * XXX: (regardless of what's in the parent's range?) 859 */ 860 861 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 862 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 863 864 #ifdef DDI_MAP_DEBUG 865 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 866 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 867 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 868 len, mp->map_handlep); 869 #endif /* DDI_MAP_DEBUG */ 870 871 /* 872 * I/O or memory mapping: 873 * 874 * <bustype=0, addr=x, len=x>: memory 875 * <bustype=1, addr=x, len=x>: i/o 876 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 877 */ 878 879 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 880 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 881 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 882 ddi_get_name(rdip), rp->regspec_bustype, 883 rp->regspec_addr, rp->regspec_size); 884 return (DDI_ME_INVAL); 885 } 886 887 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 888 /* 889 * compatibility i/o mapping 890 */ 891 rp->regspec_bustype += (uint_t)offset; 892 } else { 893 /* 894 * Normal memory or i/o mapping 895 */ 896 rp->regspec_addr += (uint_t)offset; 897 } 898 899 if (len != 0) 900 rp->regspec_size = (uint_t)len; 901 902 #ifdef DDI_MAP_DEBUG 903 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 904 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 905 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 906 offset, len, mp->map_handlep); 907 #endif /* DDI_MAP_DEBUG */ 908 909 /* 910 * Apply any parent ranges at this level, if applicable. 911 * (This is where nexus specific regspec translation takes place. 912 * Use of this function is implicit agreement that translation is 913 * provided via ddi_apply_range.) 914 */ 915 916 #ifdef DDI_MAP_DEBUG 917 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 918 ddi_get_name(dip), ddi_get_name(rdip)); 919 #endif /* DDI_MAP_DEBUG */ 920 921 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 922 return (error); 923 924 switch (mp->map_op) { 925 case DDI_MO_MAP_LOCKED: 926 927 /* 928 * Set up the locked down kernel mapping to the regspec... 929 */ 930 931 return (rootnex_map_regspec(mp, vaddrp)); 932 933 case DDI_MO_UNMAP: 934 935 /* 936 * Release mapping... 937 */ 938 939 return (rootnex_unmap_regspec(mp, vaddrp)); 940 941 case DDI_MO_MAP_HANDLE: 942 943 return (rootnex_map_handle(mp)); 944 945 default: 946 return (DDI_ME_UNIMPLEMENTED); 947 } 948 } 949 950 951 /* 952 * rootnex_map_fault() 953 * 954 * fault in mappings for requestors 955 */ 956 /*ARGSUSED*/ 957 static int 958 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 959 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 960 uint_t lock) 961 { 962 963 #ifdef DDI_MAP_DEBUG 964 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 965 ddi_map_debug(" Seg <%s>\n", 966 seg->s_ops == &segdev_ops ? "segdev" : 967 seg == &kvseg ? "segkmem" : "NONE!"); 968 #endif /* DDI_MAP_DEBUG */ 969 970 /* 971 * This is all terribly broken, but it is a start 972 * 973 * XXX Note that this test means that segdev_ops 974 * must be exported from seg_dev.c. 975 * XXX What about devices with their own segment drivers? 976 */ 977 if (seg->s_ops == &segdev_ops) { 978 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 979 980 if (hat == NULL) { 981 /* 982 * This is one plausible interpretation of 983 * a null hat i.e. use the first hat on the 984 * address space hat list which by convention is 985 * the hat of the system MMU. At alternative 986 * would be to panic .. this might well be better .. 987 */ 988 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 989 hat = seg->s_as->a_hat; 990 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 991 } 992 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 993 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 994 } else if (seg == &kvseg && dp == NULL) { 995 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 996 HAT_LOAD_LOCK); 997 } else 998 return (DDI_FAILURE); 999 return (DDI_SUCCESS); 1000 } 1001 1002 1003 /* 1004 * rootnex_map_regspec() 1005 * we don't support mapping of I/O cards above 4Gb 1006 */ 1007 static int 1008 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1009 { 1010 rootnex_addr_t rbase; 1011 void *cvaddr; 1012 uint_t npages, pgoffset; 1013 struct regspec *rp; 1014 ddi_acc_hdl_t *hp; 1015 ddi_acc_impl_t *ap; 1016 uint_t hat_acc_flags; 1017 paddr_t pbase; 1018 1019 rp = mp->map_obj.rp; 1020 hp = mp->map_handlep; 1021 1022 #ifdef DDI_MAP_DEBUG 1023 ddi_map_debug( 1024 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1025 rp->regspec_bustype, rp->regspec_addr, 1026 rp->regspec_size, mp->map_handlep); 1027 #endif /* DDI_MAP_DEBUG */ 1028 1029 /* 1030 * I/O or memory mapping 1031 * 1032 * <bustype=0, addr=x, len=x>: memory 1033 * <bustype=1, addr=x, len=x>: i/o 1034 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1035 */ 1036 1037 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1038 cmn_err(CE_WARN, "rootnex: invalid register spec" 1039 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1040 rp->regspec_addr, rp->regspec_size); 1041 return (DDI_FAILURE); 1042 } 1043 1044 if (rp->regspec_bustype != 0) { 1045 /* 1046 * I/O space - needs a handle. 1047 */ 1048 if (hp == NULL) { 1049 return (DDI_FAILURE); 1050 } 1051 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1052 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1053 impl_acc_hdl_init(hp); 1054 1055 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1056 #ifdef DDI_MAP_DEBUG 1057 ddi_map_debug("rootnex_map_regspec: mmap() " 1058 "to I/O space is not supported.\n"); 1059 #endif /* DDI_MAP_DEBUG */ 1060 return (DDI_ME_INVAL); 1061 } else { 1062 /* 1063 * 1275-compliant vs. compatibility i/o mapping 1064 */ 1065 *vaddrp = 1066 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1067 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1068 ((caddr_t)(uintptr_t)rp->regspec_addr); 1069 #ifdef __xpv 1070 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1071 hp->ah_pfn = xen_assign_pfn( 1072 mmu_btop((ulong_t)rp->regspec_addr & 1073 MMU_PAGEMASK)); 1074 } else { 1075 hp->ah_pfn = mmu_btop( 1076 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1077 } 1078 #else 1079 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1080 MMU_PAGEMASK); 1081 #endif 1082 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1083 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1084 } 1085 1086 #ifdef DDI_MAP_DEBUG 1087 ddi_map_debug( 1088 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1089 rp->regspec_size, *vaddrp); 1090 #endif /* DDI_MAP_DEBUG */ 1091 return (DDI_SUCCESS); 1092 } 1093 1094 /* 1095 * Memory space 1096 */ 1097 1098 if (hp != NULL) { 1099 /* 1100 * hat layer ignores 1101 * hp->ah_acc.devacc_attr_endian_flags. 1102 */ 1103 switch (hp->ah_acc.devacc_attr_dataorder) { 1104 case DDI_STRICTORDER_ACC: 1105 hat_acc_flags = HAT_STRICTORDER; 1106 break; 1107 case DDI_UNORDERED_OK_ACC: 1108 hat_acc_flags = HAT_UNORDERED_OK; 1109 break; 1110 case DDI_MERGING_OK_ACC: 1111 hat_acc_flags = HAT_MERGING_OK; 1112 break; 1113 case DDI_LOADCACHING_OK_ACC: 1114 hat_acc_flags = HAT_LOADCACHING_OK; 1115 break; 1116 case DDI_STORECACHING_OK_ACC: 1117 hat_acc_flags = HAT_STORECACHING_OK; 1118 break; 1119 } 1120 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1121 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1122 impl_acc_hdl_init(hp); 1123 hp->ah_hat_flags = hat_acc_flags; 1124 } else { 1125 hat_acc_flags = HAT_STRICTORDER; 1126 } 1127 1128 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1129 #ifdef __xpv 1130 /* 1131 * If we're dom0, we're using a real device so we need to translate 1132 * the MA to a PA. 1133 */ 1134 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1135 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1136 } else { 1137 pbase = rbase; 1138 } 1139 #else 1140 pbase = rbase; 1141 #endif 1142 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1143 1144 if (rp->regspec_size == 0) { 1145 #ifdef DDI_MAP_DEBUG 1146 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1147 #endif /* DDI_MAP_DEBUG */ 1148 return (DDI_ME_INVAL); 1149 } 1150 1151 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1152 /* extra cast to make gcc happy */ 1153 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1154 } else { 1155 npages = mmu_btopr(rp->regspec_size + pgoffset); 1156 1157 #ifdef DDI_MAP_DEBUG 1158 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1159 "physical %llx", npages, pbase); 1160 #endif /* DDI_MAP_DEBUG */ 1161 1162 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1163 if (cvaddr == NULL) 1164 return (DDI_ME_NORESOURCES); 1165 1166 /* 1167 * Now map in the pages we've allocated... 1168 */ 1169 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1170 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1171 HAT_LOAD_LOCK); 1172 *vaddrp = (caddr_t)cvaddr + pgoffset; 1173 1174 /* save away pfn and npages for FMA */ 1175 hp = mp->map_handlep; 1176 if (hp) { 1177 hp->ah_pfn = mmu_btop(pbase); 1178 hp->ah_pnum = npages; 1179 } 1180 } 1181 1182 #ifdef DDI_MAP_DEBUG 1183 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1184 #endif /* DDI_MAP_DEBUG */ 1185 return (DDI_SUCCESS); 1186 } 1187 1188 1189 /* 1190 * rootnex_unmap_regspec() 1191 * 1192 */ 1193 static int 1194 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1195 { 1196 caddr_t addr = (caddr_t)*vaddrp; 1197 uint_t npages, pgoffset; 1198 struct regspec *rp; 1199 1200 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1201 return (0); 1202 1203 rp = mp->map_obj.rp; 1204 1205 if (rp->regspec_size == 0) { 1206 #ifdef DDI_MAP_DEBUG 1207 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1208 #endif /* DDI_MAP_DEBUG */ 1209 return (DDI_ME_INVAL); 1210 } 1211 1212 /* 1213 * I/O or memory mapping: 1214 * 1215 * <bustype=0, addr=x, len=x>: memory 1216 * <bustype=1, addr=x, len=x>: i/o 1217 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1218 */ 1219 if (rp->regspec_bustype != 0) { 1220 /* 1221 * This is I/O space, which requires no particular 1222 * processing on unmap since it isn't mapped in the 1223 * first place. 1224 */ 1225 return (DDI_SUCCESS); 1226 } 1227 1228 /* 1229 * Memory space 1230 */ 1231 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1232 npages = mmu_btopr(rp->regspec_size + pgoffset); 1233 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1234 device_arena_free(addr - pgoffset, ptob(npages)); 1235 1236 /* 1237 * Destroy the pointer - the mapping has logically gone 1238 */ 1239 *vaddrp = NULL; 1240 1241 return (DDI_SUCCESS); 1242 } 1243 1244 1245 /* 1246 * rootnex_map_handle() 1247 * 1248 */ 1249 static int 1250 rootnex_map_handle(ddi_map_req_t *mp) 1251 { 1252 rootnex_addr_t rbase; 1253 ddi_acc_hdl_t *hp; 1254 uint_t pgoffset; 1255 struct regspec *rp; 1256 paddr_t pbase; 1257 1258 rp = mp->map_obj.rp; 1259 1260 #ifdef DDI_MAP_DEBUG 1261 ddi_map_debug( 1262 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1263 rp->regspec_bustype, rp->regspec_addr, 1264 rp->regspec_size, mp->map_handlep); 1265 #endif /* DDI_MAP_DEBUG */ 1266 1267 /* 1268 * I/O or memory mapping: 1269 * 1270 * <bustype=0, addr=x, len=x>: memory 1271 * <bustype=1, addr=x, len=x>: i/o 1272 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1273 */ 1274 if (rp->regspec_bustype != 0) { 1275 /* 1276 * This refers to I/O space, and we don't support "mapping" 1277 * I/O space to a user. 1278 */ 1279 return (DDI_FAILURE); 1280 } 1281 1282 /* 1283 * Set up the hat_flags for the mapping. 1284 */ 1285 hp = mp->map_handlep; 1286 1287 switch (hp->ah_acc.devacc_attr_endian_flags) { 1288 case DDI_NEVERSWAP_ACC: 1289 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1290 break; 1291 case DDI_STRUCTURE_LE_ACC: 1292 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1293 break; 1294 case DDI_STRUCTURE_BE_ACC: 1295 return (DDI_FAILURE); 1296 default: 1297 return (DDI_REGS_ACC_CONFLICT); 1298 } 1299 1300 switch (hp->ah_acc.devacc_attr_dataorder) { 1301 case DDI_STRICTORDER_ACC: 1302 break; 1303 case DDI_UNORDERED_OK_ACC: 1304 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1305 break; 1306 case DDI_MERGING_OK_ACC: 1307 hp->ah_hat_flags |= HAT_MERGING_OK; 1308 break; 1309 case DDI_LOADCACHING_OK_ACC: 1310 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1311 break; 1312 case DDI_STORECACHING_OK_ACC: 1313 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1314 break; 1315 default: 1316 return (DDI_FAILURE); 1317 } 1318 1319 rbase = (rootnex_addr_t)rp->regspec_addr & 1320 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1321 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1322 1323 if (rp->regspec_size == 0) 1324 return (DDI_ME_INVAL); 1325 1326 #ifdef __xpv 1327 /* 1328 * If we're dom0, we're using a real device so we need to translate 1329 * the MA to a PA. 1330 */ 1331 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1332 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1333 (rbase & MMU_PAGEOFFSET); 1334 } else { 1335 pbase = rbase; 1336 } 1337 #else 1338 pbase = rbase; 1339 #endif 1340 1341 hp->ah_pfn = mmu_btop(pbase); 1342 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1343 1344 return (DDI_SUCCESS); 1345 } 1346 1347 1348 1349 /* 1350 * ************************ 1351 * interrupt related code 1352 * ************************ 1353 */ 1354 1355 /* 1356 * rootnex_intr_ops() 1357 * bus_intr_op() function for interrupt support 1358 */ 1359 /* ARGSUSED */ 1360 static int 1361 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1362 ddi_intr_handle_impl_t *hdlp, void *result) 1363 { 1364 struct intrspec *ispec; 1365 struct ddi_parent_private_data *pdp; 1366 1367 DDI_INTR_NEXDBG((CE_CONT, 1368 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1369 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1370 1371 /* Process the interrupt operation */ 1372 switch (intr_op) { 1373 case DDI_INTROP_GETCAP: 1374 /* First check with pcplusmp */ 1375 if (psm_intr_ops == NULL) 1376 return (DDI_FAILURE); 1377 1378 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1379 *(int *)result = 0; 1380 return (DDI_FAILURE); 1381 } 1382 break; 1383 case DDI_INTROP_SETCAP: 1384 if (psm_intr_ops == NULL) 1385 return (DDI_FAILURE); 1386 1387 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1388 return (DDI_FAILURE); 1389 break; 1390 case DDI_INTROP_ALLOC: 1391 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1392 return (DDI_FAILURE); 1393 hdlp->ih_pri = ispec->intrspec_pri; 1394 *(int *)result = hdlp->ih_scratch1; 1395 break; 1396 case DDI_INTROP_FREE: 1397 pdp = ddi_get_parent_data(rdip); 1398 /* 1399 * Special case for 'pcic' driver' only. 1400 * If an intrspec was created for it, clean it up here 1401 * See detailed comments on this in the function 1402 * rootnex_get_ispec(). 1403 */ 1404 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1405 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1406 pdp->par_nintr); 1407 /* 1408 * Set it to zero; so that 1409 * DDI framework doesn't free it again 1410 */ 1411 pdp->par_intr = NULL; 1412 pdp->par_nintr = 0; 1413 } 1414 break; 1415 case DDI_INTROP_GETPRI: 1416 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1417 return (DDI_FAILURE); 1418 *(int *)result = ispec->intrspec_pri; 1419 break; 1420 case DDI_INTROP_SETPRI: 1421 /* Validate the interrupt priority passed to us */ 1422 if (*(int *)result > LOCK_LEVEL) 1423 return (DDI_FAILURE); 1424 1425 /* Ensure that PSM is all initialized and ispec is ok */ 1426 if ((psm_intr_ops == NULL) || 1427 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1428 return (DDI_FAILURE); 1429 1430 /* Change the priority */ 1431 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1432 PSM_FAILURE) 1433 return (DDI_FAILURE); 1434 1435 /* update the ispec with the new priority */ 1436 ispec->intrspec_pri = *(int *)result; 1437 break; 1438 case DDI_INTROP_ADDISR: 1439 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1440 return (DDI_FAILURE); 1441 ispec->intrspec_func = hdlp->ih_cb_func; 1442 break; 1443 case DDI_INTROP_REMISR: 1444 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 return (DDI_FAILURE); 1446 ispec->intrspec_func = (uint_t (*)()) 0; 1447 break; 1448 case DDI_INTROP_ENABLE: 1449 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1450 return (DDI_FAILURE); 1451 1452 /* Call psmi to translate irq with the dip */ 1453 if (psm_intr_ops == NULL) 1454 return (DDI_FAILURE); 1455 1456 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1457 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1458 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1459 return (DDI_FAILURE); 1460 1461 /* Add the interrupt handler */ 1462 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1463 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1464 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1465 return (DDI_FAILURE); 1466 break; 1467 case DDI_INTROP_DISABLE: 1468 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1469 return (DDI_FAILURE); 1470 1471 /* Call psm_ops() to translate irq with the dip */ 1472 if (psm_intr_ops == NULL) 1473 return (DDI_FAILURE); 1474 1475 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1476 (void) (*psm_intr_ops)(rdip, hdlp, 1477 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1478 1479 /* Remove the interrupt handler */ 1480 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1481 hdlp->ih_cb_func, hdlp->ih_vector); 1482 break; 1483 case DDI_INTROP_SETMASK: 1484 if (psm_intr_ops == NULL) 1485 return (DDI_FAILURE); 1486 1487 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1488 return (DDI_FAILURE); 1489 break; 1490 case DDI_INTROP_CLRMASK: 1491 if (psm_intr_ops == NULL) 1492 return (DDI_FAILURE); 1493 1494 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1495 return (DDI_FAILURE); 1496 break; 1497 case DDI_INTROP_GETPENDING: 1498 if (psm_intr_ops == NULL) 1499 return (DDI_FAILURE); 1500 1501 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1502 result)) { 1503 *(int *)result = 0; 1504 return (DDI_FAILURE); 1505 } 1506 break; 1507 case DDI_INTROP_NAVAIL: 1508 case DDI_INTROP_NINTRS: 1509 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1510 if (*(int *)result == 0) { 1511 /* 1512 * Special case for 'pcic' driver' only. This driver 1513 * driver is a child of 'isa' and 'rootnex' drivers. 1514 * 1515 * See detailed comments on this in the function 1516 * rootnex_get_ispec(). 1517 * 1518 * Children of 'pcic' send 'NINITR' request all the 1519 * way to rootnex driver. But, the 'pdp->par_nintr' 1520 * field may not initialized. So, we fake it here 1521 * to return 1 (a la what PCMCIA nexus does). 1522 */ 1523 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1524 *(int *)result = 1; 1525 else 1526 return (DDI_FAILURE); 1527 } 1528 break; 1529 case DDI_INTROP_SUPPORTED_TYPES: 1530 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1531 break; 1532 default: 1533 return (DDI_FAILURE); 1534 } 1535 1536 return (DDI_SUCCESS); 1537 } 1538 1539 1540 /* 1541 * rootnex_get_ispec() 1542 * convert an interrupt number to an interrupt specification. 1543 * The interrupt number determines which interrupt spec will be 1544 * returned if more than one exists. 1545 * 1546 * Look into the parent private data area of the 'rdip' to find out 1547 * the interrupt specification. First check to make sure there is 1548 * one that matchs "inumber" and then return a pointer to it. 1549 * 1550 * Return NULL if one could not be found. 1551 * 1552 * NOTE: This is needed for rootnex_intr_ops() 1553 */ 1554 static struct intrspec * 1555 rootnex_get_ispec(dev_info_t *rdip, int inum) 1556 { 1557 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1558 1559 /* 1560 * Special case handling for drivers that provide their own 1561 * intrspec structures instead of relying on the DDI framework. 1562 * 1563 * A broken hardware driver in ON could potentially provide its 1564 * own intrspec structure, instead of relying on the hardware. 1565 * If these drivers are children of 'rootnex' then we need to 1566 * continue to provide backward compatibility to them here. 1567 * 1568 * Following check is a special case for 'pcic' driver which 1569 * was found to have broken hardwre andby provides its own intrspec. 1570 * 1571 * Verbatim comments from this driver are shown here: 1572 * "Don't use the ddi_add_intr since we don't have a 1573 * default intrspec in all cases." 1574 * 1575 * Since an 'ispec' may not be always created for it, 1576 * check for that and create one if so. 1577 * 1578 * NOTE: Currently 'pcic' is the only driver found to do this. 1579 */ 1580 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1581 pdp->par_nintr = 1; 1582 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1583 pdp->par_nintr, KM_SLEEP); 1584 } 1585 1586 /* Validate the interrupt number */ 1587 if (inum >= pdp->par_nintr) 1588 return (NULL); 1589 1590 /* Get the interrupt structure pointer and return that */ 1591 return ((struct intrspec *)&pdp->par_intr[inum]); 1592 } 1593 1594 1595 /* 1596 * ****************** 1597 * dma related code 1598 * ****************** 1599 */ 1600 1601 /*ARGSUSED*/ 1602 static int 1603 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1604 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1605 ddi_dma_handle_t *handlep) 1606 { 1607 uint64_t maxsegmentsize_ll; 1608 uint_t maxsegmentsize; 1609 ddi_dma_impl_t *hp; 1610 rootnex_dma_t *dma; 1611 uint64_t count_max; 1612 uint64_t seg; 1613 int kmflag; 1614 int e; 1615 1616 1617 /* convert our sleep flags */ 1618 if (waitfp == DDI_DMA_SLEEP) { 1619 kmflag = KM_SLEEP; 1620 } else { 1621 kmflag = KM_NOSLEEP; 1622 } 1623 1624 /* 1625 * We try to do only one memory allocation here. We'll do a little 1626 * pointer manipulation later. If the bind ends up taking more than 1627 * our prealloc's space, we'll have to allocate more memory in the 1628 * bind operation. Not great, but much better than before and the 1629 * best we can do with the current bind interfaces. 1630 */ 1631 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1632 if (hp == NULL) { 1633 if (waitfp != DDI_DMA_DONTWAIT) { 1634 ddi_set_callback(waitfp, arg, 1635 &rootnex_state->r_dvma_call_list_id); 1636 } 1637 return (DDI_DMA_NORESOURCES); 1638 } 1639 1640 /* Do our pointer manipulation now, align the structures */ 1641 hp->dmai_private = (void *)(((uintptr_t)hp + 1642 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1643 dma = (rootnex_dma_t *)hp->dmai_private; 1644 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1645 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1646 1647 /* setup the handle */ 1648 rootnex_clean_dmahdl(hp); 1649 hp->dmai_error.err_fep = NULL; 1650 hp->dmai_error.err_cf = NULL; 1651 dma->dp_dip = rdip; 1652 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1653 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1654 hp->dmai_minxfer = attr->dma_attr_minxfer; 1655 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1656 hp->dmai_rdip = rdip; 1657 hp->dmai_attr = *attr; 1658 1659 /* we don't need to worry about the SPL since we do a tryenter */ 1660 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1661 1662 /* 1663 * Figure out our maximum segment size. If the segment size is greater 1664 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1665 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1666 * dma_attr_count_max are size-1 type values. 1667 * 1668 * Maximum segment size is the largest physically contiguous chunk of 1669 * memory that we can return from a bind (i.e. the maximum size of a 1670 * single cookie). 1671 */ 1672 1673 /* handle the rollover cases */ 1674 seg = attr->dma_attr_seg + 1; 1675 if (seg < attr->dma_attr_seg) { 1676 seg = attr->dma_attr_seg; 1677 } 1678 count_max = attr->dma_attr_count_max + 1; 1679 if (count_max < attr->dma_attr_count_max) { 1680 count_max = attr->dma_attr_count_max; 1681 } 1682 1683 /* 1684 * granularity may or may not be a power of two. If it isn't, we can't 1685 * use a simple mask. 1686 */ 1687 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1688 dma->dp_granularity_power_2 = B_FALSE; 1689 } else { 1690 dma->dp_granularity_power_2 = B_TRUE; 1691 } 1692 1693 /* 1694 * maxxfer should be a whole multiple of granularity. If we're going to 1695 * break up a window because we're greater than maxxfer, we might as 1696 * well make sure it's maxxfer is a whole multiple so we don't have to 1697 * worry about triming the window later on for this case. 1698 */ 1699 if (attr->dma_attr_granular > 1) { 1700 if (dma->dp_granularity_power_2) { 1701 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1702 (attr->dma_attr_maxxfer & 1703 (attr->dma_attr_granular - 1)); 1704 } else { 1705 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1706 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1707 } 1708 } else { 1709 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1710 } 1711 1712 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1713 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1714 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1715 maxsegmentsize = 0xFFFFFFFF; 1716 } else { 1717 maxsegmentsize = maxsegmentsize_ll; 1718 } 1719 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1720 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1721 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1722 1723 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1724 if (rootnex_alloc_check_parms) { 1725 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1726 if (e != DDI_SUCCESS) { 1727 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1728 (void) rootnex_dma_freehdl(dip, rdip, 1729 (ddi_dma_handle_t)hp); 1730 return (e); 1731 } 1732 } 1733 1734 *handlep = (ddi_dma_handle_t)hp; 1735 1736 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1737 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1738 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1739 1740 return (DDI_SUCCESS); 1741 } 1742 1743 1744 /* 1745 * rootnex_dma_allochdl() 1746 * called from ddi_dma_alloc_handle(). 1747 */ 1748 static int 1749 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1750 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1751 { 1752 int retval; 1753 #if defined(__amd64) && !defined(__xpv) 1754 uint_t error = ENOTSUP; 1755 1756 retval = iommulib_nex_open(rdip, &error); 1757 1758 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1759 /* No IOMMU */ 1760 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1761 handlep)); 1762 } else if (retval != DDI_SUCCESS) { 1763 return (DDI_FAILURE); 1764 } 1765 1766 ASSERT(IOMMU_USED(rdip)); 1767 1768 /* has an IOMMU */ 1769 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1770 waitfp, arg, handlep); 1771 #else 1772 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1773 handlep); 1774 #endif 1775 if (retval == DDI_SUCCESS) 1776 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1777 return (retval); 1778 } 1779 1780 /*ARGSUSED*/ 1781 static int 1782 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1783 ddi_dma_handle_t handle) 1784 { 1785 ddi_dma_impl_t *hp; 1786 rootnex_dma_t *dma; 1787 1788 1789 hp = (ddi_dma_impl_t *)handle; 1790 dma = (rootnex_dma_t *)hp->dmai_private; 1791 1792 /* unbind should have been called first */ 1793 ASSERT(!dma->dp_inuse); 1794 1795 mutex_destroy(&dma->dp_mutex); 1796 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1797 1798 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1799 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1800 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1801 1802 if (rootnex_state->r_dvma_call_list_id) 1803 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1804 1805 return (DDI_SUCCESS); 1806 } 1807 1808 /* 1809 * rootnex_dma_freehdl() 1810 * called from ddi_dma_free_handle(). 1811 */ 1812 static int 1813 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1814 { 1815 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1816 #if defined(__amd64) && !defined(__xpv) 1817 if (IOMMU_USED(rdip)) { 1818 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1819 } 1820 #endif 1821 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1822 } 1823 1824 /*ARGSUSED*/ 1825 static int 1826 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1827 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1828 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1829 { 1830 rootnex_sglinfo_t *sinfo; 1831 ddi_dma_attr_t *attr; 1832 ddi_dma_impl_t *hp; 1833 rootnex_dma_t *dma; 1834 int kmflag; 1835 int e; 1836 1837 hp = (ddi_dma_impl_t *)handle; 1838 dma = (rootnex_dma_t *)hp->dmai_private; 1839 sinfo = &dma->dp_sglinfo; 1840 attr = &hp->dmai_attr; 1841 1842 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1843 dma->dp_sleep_flags = KM_SLEEP; 1844 } else { 1845 dma->dp_sleep_flags = KM_NOSLEEP; 1846 } 1847 1848 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1849 1850 /* 1851 * This is useful for debugging a driver. Not as useful in a production 1852 * system. The only time this will fail is if you have a driver bug. 1853 */ 1854 if (rootnex_bind_check_inuse) { 1855 /* 1856 * No one else should ever have this lock unless someone else 1857 * is trying to use this handle. So contention on the lock 1858 * is the same as inuse being set. 1859 */ 1860 e = mutex_tryenter(&dma->dp_mutex); 1861 if (e == 0) { 1862 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1863 return (DDI_DMA_INUSE); 1864 } 1865 if (dma->dp_inuse) { 1866 mutex_exit(&dma->dp_mutex); 1867 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1868 return (DDI_DMA_INUSE); 1869 } 1870 dma->dp_inuse = B_TRUE; 1871 mutex_exit(&dma->dp_mutex); 1872 } 1873 1874 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1875 if (rootnex_bind_check_parms) { 1876 e = rootnex_valid_bind_parms(dmareq, attr); 1877 if (e != DDI_SUCCESS) { 1878 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1879 rootnex_clean_dmahdl(hp); 1880 return (e); 1881 } 1882 } 1883 1884 /* save away the original bind info */ 1885 dma->dp_dma = dmareq->dmar_object; 1886 1887 #if defined(__amd64) && !defined(__xpv) 1888 e = immu_map_sgl(hp, dmareq, rootnex_prealloc_cookies, rdip); 1889 switch (e) { 1890 case DDI_DMA_MAPPED: 1891 goto out; 1892 case DDI_DMA_USE_PHYSICAL: 1893 break; 1894 case DDI_DMA_PARTIAL: 1895 ddi_err(DER_PANIC, rdip, "Partial DVMA map"); 1896 e = DDI_DMA_NORESOURCES; 1897 /*FALLTHROUGH*/ 1898 default: 1899 ddi_err(DER_MODE, rdip, "DVMA map failed"); 1900 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1901 rootnex_clean_dmahdl(hp); 1902 return (e); 1903 } 1904 #endif 1905 1906 /* 1907 * Figure out a rough estimate of what maximum number of pages this 1908 * buffer could use (a high estimate of course). 1909 */ 1910 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1911 1912 /* 1913 * We'll use the pre-allocated cookies for any bind that will *always* 1914 * fit (more important to be consistent, we don't want to create 1915 * additional degenerate cases). 1916 */ 1917 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1918 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1919 dma->dp_need_to_free_cookie = B_FALSE; 1920 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1921 uint_t, sinfo->si_max_pages); 1922 1923 /* 1924 * For anything larger than that, we'll go ahead and allocate the 1925 * maximum number of pages we expect to see. Hopefuly, we won't be 1926 * seeing this path in the fast path for high performance devices very 1927 * frequently. 1928 * 1929 * a ddi bind interface that allowed the driver to provide storage to 1930 * the bind interface would speed this case up. 1931 */ 1932 } else { 1933 /* convert the sleep flags */ 1934 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1935 kmflag = KM_SLEEP; 1936 } else { 1937 kmflag = KM_NOSLEEP; 1938 } 1939 1940 /* 1941 * Save away how much memory we allocated. If we're doing a 1942 * nosleep, the alloc could fail... 1943 */ 1944 dma->dp_cookie_size = sinfo->si_max_pages * 1945 sizeof (ddi_dma_cookie_t); 1946 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1947 if (dma->dp_cookies == NULL) { 1948 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1949 rootnex_clean_dmahdl(hp); 1950 return (DDI_DMA_NORESOURCES); 1951 } 1952 dma->dp_need_to_free_cookie = B_TRUE; 1953 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1954 sinfo->si_max_pages); 1955 } 1956 hp->dmai_cookie = dma->dp_cookies; 1957 1958 /* 1959 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1960 * looking at the constraints in the dma structure. It will then put 1961 * some additional state about the sgl in the dma struct (i.e. is 1962 * the sgl clean, or do we need to do some munging; how many pages 1963 * need to be copied, etc.) 1964 */ 1965 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1966 &dma->dp_sglinfo); 1967 1968 out: 1969 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1970 /* if we don't need a copy buffer, we don't need to sync */ 1971 if (sinfo->si_copybuf_req == 0) { 1972 hp->dmai_rflags |= DMP_NOSYNC; 1973 } 1974 1975 /* 1976 * if we don't need the copybuf and we don't need to do a partial, we 1977 * hit the fast path. All the high performance devices should be trying 1978 * to hit this path. To hit this path, a device should be able to reach 1979 * all of memory, shouldn't try to bind more than it can transfer, and 1980 * the buffer shouldn't require more cookies than the driver/device can 1981 * handle [sgllen]). 1982 */ 1983 if ((sinfo->si_copybuf_req == 0) && 1984 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1985 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1986 /* 1987 * If the driver supports FMA, insert the handle in the FMA DMA 1988 * handle cache. 1989 */ 1990 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 1991 hp->dmai_error.err_cf = rootnex_dma_check; 1992 1993 /* 1994 * copy out the first cookie and ccountp, set the cookie 1995 * pointer to the second cookie. The first cookie is passed 1996 * back on the stack. Additional cookies are accessed via 1997 * ddi_dma_nextcookie() 1998 */ 1999 *cookiep = dma->dp_cookies[0]; 2000 *ccountp = sinfo->si_sgl_size; 2001 hp->dmai_cookie++; 2002 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2003 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2004 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, 2005 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2006 uint_t, dma->dp_dma.dmao_size); 2007 2008 2009 return (DDI_DMA_MAPPED); 2010 } 2011 2012 /* 2013 * go to the slow path, we may need to alloc more memory, create 2014 * multiple windows, and munge up a sgl to make the device happy. 2015 */ 2016 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2017 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2018 if (dma->dp_need_to_free_cookie) { 2019 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2020 } 2021 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2022 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2023 return (e); 2024 } 2025 2026 /* 2027 * If the driver supports FMA, insert the handle in the FMA DMA handle 2028 * cache. 2029 */ 2030 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2031 hp->dmai_error.err_cf = rootnex_dma_check; 2032 2033 /* if the first window uses the copy buffer, sync it for the device */ 2034 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2035 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2036 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2037 DDI_DMA_SYNC_FORDEV); 2038 } 2039 2040 /* 2041 * copy out the first cookie and ccountp, set the cookie pointer to the 2042 * second cookie. Make sure the partial flag is set/cleared correctly. 2043 * If we have a partial map (i.e. multiple windows), the number of 2044 * cookies we return is the number of cookies in the first window. 2045 */ 2046 if (e == DDI_DMA_MAPPED) { 2047 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2048 *ccountp = sinfo->si_sgl_size; 2049 hp->dmai_nwin = 1; 2050 } else { 2051 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2052 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2053 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2054 } 2055 *cookiep = dma->dp_cookies[0]; 2056 hp->dmai_cookie++; 2057 2058 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2059 ROOTNEX_DPROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2060 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2061 dma->dp_dma.dmao_size); 2062 return (e); 2063 } 2064 2065 /* 2066 * rootnex_dma_bindhdl() 2067 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2068 */ 2069 static int 2070 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2071 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2072 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2073 { 2074 #if defined(__amd64) && !defined(__xpv) 2075 if (IOMMU_USED(rdip)) { 2076 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2077 cookiep, ccountp)); 2078 } 2079 #endif 2080 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2081 cookiep, ccountp)); 2082 } 2083 2084 2085 2086 /*ARGSUSED*/ 2087 static int 2088 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2089 ddi_dma_handle_t handle) 2090 { 2091 ddi_dma_impl_t *hp; 2092 rootnex_dma_t *dma; 2093 int e; 2094 2095 hp = (ddi_dma_impl_t *)handle; 2096 dma = (rootnex_dma_t *)hp->dmai_private; 2097 2098 /* make sure the buffer wasn't free'd before calling unbind */ 2099 if (rootnex_unbind_verify_buffer) { 2100 e = rootnex_verify_buffer(dma); 2101 if (e != DDI_SUCCESS) { 2102 ASSERT(0); 2103 return (DDI_FAILURE); 2104 } 2105 } 2106 2107 /* sync the current window before unbinding the buffer */ 2108 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2109 (hp->dmai_rflags & DDI_DMA_READ)) { 2110 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2111 DDI_DMA_SYNC_FORCPU); 2112 } 2113 2114 /* 2115 * cleanup and copy buffer or window state. if we didn't use the copy 2116 * buffer or windows, there won't be much to do :-) 2117 */ 2118 rootnex_teardown_copybuf(dma); 2119 rootnex_teardown_windows(dma); 2120 2121 #if defined(__amd64) && !defined(__xpv) 2122 /* 2123 * Clean up the page tables and free the dvma 2124 */ 2125 e = immu_unmap_sgl(hp, rdip); 2126 if (e != DDI_DMA_USE_PHYSICAL && e != DDI_SUCCESS) { 2127 return (e); 2128 } 2129 #endif 2130 2131 /* 2132 * If we had to allocate space to for the worse case sgl (it didn't 2133 * fit into our pre-allocate buffer), free that up now 2134 */ 2135 if (dma->dp_need_to_free_cookie) { 2136 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2137 } 2138 2139 /* 2140 * clean up the handle so it's ready for the next bind (i.e. if the 2141 * handle is reused). 2142 */ 2143 rootnex_clean_dmahdl(hp); 2144 hp->dmai_error.err_cf = NULL; 2145 2146 if (rootnex_state->r_dvma_call_list_id) 2147 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2148 2149 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2150 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2151 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2152 2153 return (DDI_SUCCESS); 2154 } 2155 2156 /* 2157 * rootnex_dma_unbindhdl() 2158 * called from ddi_dma_unbind_handle() 2159 */ 2160 /*ARGSUSED*/ 2161 static int 2162 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2163 ddi_dma_handle_t handle) 2164 { 2165 #if defined(__amd64) && !defined(__xpv) 2166 if (IOMMU_USED(rdip)) { 2167 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2168 } 2169 #endif 2170 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2171 } 2172 2173 #if defined(__amd64) && !defined(__xpv) 2174 2175 static int 2176 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2177 { 2178 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2179 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2180 2181 if (dma->dp_sleep_flags != KM_SLEEP && 2182 dma->dp_sleep_flags != KM_NOSLEEP) 2183 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2184 return (dma->dp_sleep_flags); 2185 } 2186 /*ARGSUSED*/ 2187 static void 2188 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2189 { 2190 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2191 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2192 rootnex_window_t *window; 2193 2194 if (dma->dp_window) { 2195 window = &dma->dp_window[dma->dp_current_win]; 2196 hp->dmai_cookie = window->wd_first_cookie; 2197 } else { 2198 hp->dmai_cookie = dma->dp_cookies; 2199 } 2200 hp->dmai_cookie++; 2201 } 2202 2203 /*ARGSUSED*/ 2204 static int 2205 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2206 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2207 { 2208 int i; 2209 int km_flags; 2210 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2211 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2212 rootnex_window_t *window; 2213 ddi_dma_cookie_t *cp; 2214 ddi_dma_cookie_t *cookie; 2215 2216 ASSERT(*cookiepp == NULL); 2217 ASSERT(*ccountp == 0); 2218 2219 if (dma->dp_window) { 2220 window = &dma->dp_window[dma->dp_current_win]; 2221 cp = window->wd_first_cookie; 2222 *ccountp = window->wd_cookie_cnt; 2223 } else { 2224 cp = dma->dp_cookies; 2225 *ccountp = dma->dp_sglinfo.si_sgl_size; 2226 } 2227 2228 km_flags = rootnex_coredma_get_sleep_flags(handle); 2229 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2230 if (cookie == NULL) { 2231 return (DDI_DMA_NORESOURCES); 2232 } 2233 2234 for (i = 0; i < *ccountp; i++) { 2235 cookie[i].dmac_notused = cp[i].dmac_notused; 2236 cookie[i].dmac_type = cp[i].dmac_type; 2237 cookie[i].dmac_address = cp[i].dmac_address; 2238 cookie[i].dmac_size = cp[i].dmac_size; 2239 } 2240 2241 *cookiepp = cookie; 2242 2243 return (DDI_SUCCESS); 2244 } 2245 2246 /*ARGSUSED*/ 2247 static int 2248 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2249 ddi_dma_cookie_t *cookiep, uint_t ccount) 2250 { 2251 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2252 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2253 rootnex_window_t *window; 2254 ddi_dma_cookie_t *cur_cookiep; 2255 2256 ASSERT(cookiep); 2257 ASSERT(ccount != 0); 2258 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2259 2260 if (dma->dp_window) { 2261 window = &dma->dp_window[dma->dp_current_win]; 2262 dma->dp_saved_cookies = window->wd_first_cookie; 2263 window->wd_first_cookie = cookiep; 2264 ASSERT(ccount == window->wd_cookie_cnt); 2265 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2266 + window->wd_first_cookie; 2267 } else { 2268 dma->dp_saved_cookies = dma->dp_cookies; 2269 dma->dp_cookies = cookiep; 2270 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2271 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2272 + dma->dp_cookies; 2273 } 2274 2275 dma->dp_need_to_switch_cookies = B_TRUE; 2276 hp->dmai_cookie = cur_cookiep; 2277 2278 return (DDI_SUCCESS); 2279 } 2280 2281 /*ARGSUSED*/ 2282 static int 2283 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2284 { 2285 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2286 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2287 rootnex_window_t *window; 2288 ddi_dma_cookie_t *cur_cookiep; 2289 ddi_dma_cookie_t *cookie_array; 2290 uint_t ccount; 2291 2292 /* check if cookies have not been switched */ 2293 if (dma->dp_need_to_switch_cookies == B_FALSE) 2294 return (DDI_SUCCESS); 2295 2296 ASSERT(dma->dp_saved_cookies); 2297 2298 if (dma->dp_window) { 2299 window = &dma->dp_window[dma->dp_current_win]; 2300 cookie_array = window->wd_first_cookie; 2301 window->wd_first_cookie = dma->dp_saved_cookies; 2302 dma->dp_saved_cookies = NULL; 2303 ccount = window->wd_cookie_cnt; 2304 cur_cookiep = (hp->dmai_cookie - cookie_array) 2305 + window->wd_first_cookie; 2306 } else { 2307 cookie_array = dma->dp_cookies; 2308 dma->dp_cookies = dma->dp_saved_cookies; 2309 dma->dp_saved_cookies = NULL; 2310 ccount = dma->dp_sglinfo.si_sgl_size; 2311 cur_cookiep = (hp->dmai_cookie - cookie_array) 2312 + dma->dp_cookies; 2313 } 2314 2315 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2316 2317 hp->dmai_cookie = cur_cookiep; 2318 2319 dma->dp_need_to_switch_cookies = B_FALSE; 2320 2321 return (DDI_SUCCESS); 2322 } 2323 2324 #endif 2325 2326 /* 2327 * rootnex_verify_buffer() 2328 * verify buffer wasn't free'd 2329 */ 2330 static int 2331 rootnex_verify_buffer(rootnex_dma_t *dma) 2332 { 2333 page_t **pplist; 2334 caddr_t vaddr; 2335 uint_t pcnt; 2336 uint_t poff; 2337 page_t *pp; 2338 char b; 2339 int i; 2340 2341 /* Figure out how many pages this buffer occupies */ 2342 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2343 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2344 } else { 2345 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2346 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2347 } 2348 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2349 2350 switch (dma->dp_dma.dmao_type) { 2351 case DMA_OTYP_PAGES: 2352 /* 2353 * for a linked list of pp's walk through them to make sure 2354 * they're locked and not free. 2355 */ 2356 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2357 for (i = 0; i < pcnt; i++) { 2358 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2359 return (DDI_FAILURE); 2360 } 2361 pp = pp->p_next; 2362 } 2363 break; 2364 2365 case DMA_OTYP_VADDR: 2366 case DMA_OTYP_BUFVADDR: 2367 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2368 /* 2369 * for an array of pp's walk through them to make sure they're 2370 * not free. It's possible that they may not be locked. 2371 */ 2372 if (pplist) { 2373 for (i = 0; i < pcnt; i++) { 2374 if (PP_ISFREE(pplist[i])) { 2375 return (DDI_FAILURE); 2376 } 2377 } 2378 2379 /* For a virtual address, try to peek at each page */ 2380 } else { 2381 if (dma->dp_sglinfo.si_asp == &kas) { 2382 for (i = 0; i < pcnt; i++) { 2383 if (ddi_peek8(NULL, vaddr, &b) == 2384 DDI_FAILURE) 2385 return (DDI_FAILURE); 2386 vaddr += MMU_PAGESIZE; 2387 } 2388 } 2389 } 2390 break; 2391 2392 default: 2393 ASSERT(0); 2394 break; 2395 } 2396 2397 return (DDI_SUCCESS); 2398 } 2399 2400 2401 /* 2402 * rootnex_clean_dmahdl() 2403 * Clean the dma handle. This should be called on a handle alloc and an 2404 * unbind handle. Set the handle state to the default settings. 2405 */ 2406 static void 2407 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2408 { 2409 rootnex_dma_t *dma; 2410 2411 2412 dma = (rootnex_dma_t *)hp->dmai_private; 2413 2414 hp->dmai_nwin = 0; 2415 dma->dp_current_cookie = 0; 2416 dma->dp_copybuf_size = 0; 2417 dma->dp_window = NULL; 2418 dma->dp_cbaddr = NULL; 2419 dma->dp_inuse = B_FALSE; 2420 dma->dp_need_to_free_cookie = B_FALSE; 2421 dma->dp_need_to_switch_cookies = B_FALSE; 2422 dma->dp_saved_cookies = NULL; 2423 dma->dp_sleep_flags = KM_PANIC; 2424 dma->dp_need_to_free_window = B_FALSE; 2425 dma->dp_partial_required = B_FALSE; 2426 dma->dp_trim_required = B_FALSE; 2427 dma->dp_sglinfo.si_copybuf_req = 0; 2428 #if !defined(__amd64) 2429 dma->dp_cb_remaping = B_FALSE; 2430 dma->dp_kva = NULL; 2431 #endif 2432 2433 /* FMA related initialization */ 2434 hp->dmai_fault = 0; 2435 hp->dmai_fault_check = NULL; 2436 hp->dmai_fault_notify = NULL; 2437 hp->dmai_error.err_ena = 0; 2438 hp->dmai_error.err_status = DDI_FM_OK; 2439 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2440 hp->dmai_error.err_ontrap = NULL; 2441 } 2442 2443 2444 /* 2445 * rootnex_valid_alloc_parms() 2446 * Called in ddi_dma_alloc_handle path to validate its parameters. 2447 */ 2448 static int 2449 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2450 { 2451 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2452 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2453 (attr->dma_attr_granular > MMU_PAGESIZE) || 2454 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2455 return (DDI_DMA_BADATTR); 2456 } 2457 2458 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2459 return (DDI_DMA_BADATTR); 2460 } 2461 2462 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2463 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2464 attr->dma_attr_sgllen <= 0) { 2465 return (DDI_DMA_BADATTR); 2466 } 2467 2468 /* We should be able to DMA into every byte offset in a page */ 2469 if (maxsegmentsize < MMU_PAGESIZE) { 2470 return (DDI_DMA_BADATTR); 2471 } 2472 2473 /* if we're bouncing on seg, seg must be <= addr_hi */ 2474 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2475 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2476 return (DDI_DMA_BADATTR); 2477 } 2478 return (DDI_SUCCESS); 2479 } 2480 2481 /* 2482 * rootnex_valid_bind_parms() 2483 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2484 */ 2485 /* ARGSUSED */ 2486 static int 2487 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2488 { 2489 #if !defined(__amd64) 2490 /* 2491 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2492 * we can track the offset for the obsoleted interfaces. 2493 */ 2494 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2495 return (DDI_DMA_TOOBIG); 2496 } 2497 #endif 2498 2499 return (DDI_SUCCESS); 2500 } 2501 2502 2503 /* 2504 * rootnex_need_bounce_seg() 2505 * check to see if the buffer lives on both side of the seg. 2506 */ 2507 static boolean_t 2508 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2509 { 2510 ddi_dma_atyp_t buftype; 2511 rootnex_addr_t raddr; 2512 boolean_t lower_addr; 2513 boolean_t upper_addr; 2514 uint64_t offset; 2515 page_t **pplist; 2516 uint64_t paddr; 2517 uint32_t psize; 2518 uint32_t size; 2519 caddr_t vaddr; 2520 uint_t pcnt; 2521 page_t *pp; 2522 2523 2524 /* shortcuts */ 2525 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2526 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2527 buftype = dmar_object->dmao_type; 2528 size = dmar_object->dmao_size; 2529 2530 lower_addr = B_FALSE; 2531 upper_addr = B_FALSE; 2532 pcnt = 0; 2533 2534 /* 2535 * Process the first page to handle the initial offset of the buffer. 2536 * We'll use the base address we get later when we loop through all 2537 * the pages. 2538 */ 2539 if (buftype == DMA_OTYP_PAGES) { 2540 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2541 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2542 MMU_PAGEOFFSET; 2543 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2544 psize = MIN(size, (MMU_PAGESIZE - offset)); 2545 pp = pp->p_next; 2546 sglinfo->si_asp = NULL; 2547 } else if (pplist != NULL) { 2548 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2549 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2550 if (sglinfo->si_asp == NULL) { 2551 sglinfo->si_asp = &kas; 2552 } 2553 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2554 paddr += offset; 2555 psize = MIN(size, (MMU_PAGESIZE - offset)); 2556 pcnt++; 2557 } else { 2558 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2559 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2560 if (sglinfo->si_asp == NULL) { 2561 sglinfo->si_asp = &kas; 2562 } 2563 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2564 paddr += offset; 2565 psize = MIN(size, (MMU_PAGESIZE - offset)); 2566 vaddr += psize; 2567 } 2568 2569 #ifdef __xpv 2570 /* 2571 * If we're dom0, we're using a real device so we need to load 2572 * the cookies with MFNs instead of PFNs. 2573 */ 2574 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2575 #else 2576 raddr = paddr; 2577 #endif 2578 2579 if ((raddr + psize) > sglinfo->si_segmask) { 2580 upper_addr = B_TRUE; 2581 } else { 2582 lower_addr = B_TRUE; 2583 } 2584 size -= psize; 2585 2586 /* 2587 * Walk through the rest of the pages in the buffer. Track to see 2588 * if we have pages on both sides of the segment boundary. 2589 */ 2590 while (size > 0) { 2591 /* partial or full page */ 2592 psize = MIN(size, MMU_PAGESIZE); 2593 2594 if (buftype == DMA_OTYP_PAGES) { 2595 /* get the paddr from the page_t */ 2596 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2597 paddr = pfn_to_pa(pp->p_pagenum); 2598 pp = pp->p_next; 2599 } else if (pplist != NULL) { 2600 /* index into the array of page_t's to get the paddr */ 2601 ASSERT(!PP_ISFREE(pplist[pcnt])); 2602 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2603 pcnt++; 2604 } else { 2605 /* call into the VM to get the paddr */ 2606 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2607 vaddr)); 2608 vaddr += psize; 2609 } 2610 2611 #ifdef __xpv 2612 /* 2613 * If we're dom0, we're using a real device so we need to load 2614 * the cookies with MFNs instead of PFNs. 2615 */ 2616 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2617 #else 2618 raddr = paddr; 2619 #endif 2620 2621 if ((raddr + psize) > sglinfo->si_segmask) { 2622 upper_addr = B_TRUE; 2623 } else { 2624 lower_addr = B_TRUE; 2625 } 2626 /* 2627 * if the buffer lives both above and below the segment 2628 * boundary, or the current page is the page immediately 2629 * after the segment, we will use a copy/bounce buffer for 2630 * all pages > seg. 2631 */ 2632 if ((lower_addr && upper_addr) || 2633 (raddr == (sglinfo->si_segmask + 1))) { 2634 return (B_TRUE); 2635 } 2636 2637 size -= psize; 2638 } 2639 2640 return (B_FALSE); 2641 } 2642 2643 2644 /* 2645 * rootnex_get_sgl() 2646 * Called in bind fastpath to get the sgl. Most of this will be replaced 2647 * with a call to the vm layer when vm2.0 comes around... 2648 */ 2649 static void 2650 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2651 rootnex_sglinfo_t *sglinfo) 2652 { 2653 ddi_dma_atyp_t buftype; 2654 rootnex_addr_t raddr; 2655 uint64_t last_page; 2656 uint64_t offset; 2657 uint64_t addrhi; 2658 uint64_t addrlo; 2659 uint64_t maxseg; 2660 page_t **pplist; 2661 uint64_t paddr; 2662 uint32_t psize; 2663 uint32_t size; 2664 caddr_t vaddr; 2665 uint_t pcnt; 2666 page_t *pp; 2667 uint_t cnt; 2668 2669 2670 /* shortcuts */ 2671 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2672 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2673 maxseg = sglinfo->si_max_cookie_size; 2674 buftype = dmar_object->dmao_type; 2675 addrhi = sglinfo->si_max_addr; 2676 addrlo = sglinfo->si_min_addr; 2677 size = dmar_object->dmao_size; 2678 2679 pcnt = 0; 2680 cnt = 0; 2681 2682 2683 /* 2684 * check to see if we need to use the copy buffer for pages over 2685 * the segment attr. 2686 */ 2687 sglinfo->si_bounce_on_seg = B_FALSE; 2688 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2689 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2690 dmar_object, sglinfo); 2691 } 2692 2693 /* 2694 * if we were passed down a linked list of pages, i.e. pointer to 2695 * page_t, use this to get our physical address and buf offset. 2696 */ 2697 if (buftype == DMA_OTYP_PAGES) { 2698 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2699 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2700 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2701 MMU_PAGEOFFSET; 2702 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2703 psize = MIN(size, (MMU_PAGESIZE - offset)); 2704 pp = pp->p_next; 2705 sglinfo->si_asp = NULL; 2706 2707 /* 2708 * We weren't passed down a linked list of pages, but if we were passed 2709 * down an array of pages, use this to get our physical address and buf 2710 * offset. 2711 */ 2712 } else if (pplist != NULL) { 2713 ASSERT((buftype == DMA_OTYP_VADDR) || 2714 (buftype == DMA_OTYP_BUFVADDR)); 2715 2716 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2717 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2718 if (sglinfo->si_asp == NULL) { 2719 sglinfo->si_asp = &kas; 2720 } 2721 2722 ASSERT(!PP_ISFREE(pplist[pcnt])); 2723 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2724 paddr += offset; 2725 psize = MIN(size, (MMU_PAGESIZE - offset)); 2726 pcnt++; 2727 2728 /* 2729 * All we have is a virtual address, we'll need to call into the VM 2730 * to get the physical address. 2731 */ 2732 } else { 2733 ASSERT((buftype == DMA_OTYP_VADDR) || 2734 (buftype == DMA_OTYP_BUFVADDR)); 2735 2736 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2737 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2738 if (sglinfo->si_asp == NULL) { 2739 sglinfo->si_asp = &kas; 2740 } 2741 2742 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2743 paddr += offset; 2744 psize = MIN(size, (MMU_PAGESIZE - offset)); 2745 vaddr += psize; 2746 } 2747 2748 #ifdef __xpv 2749 /* 2750 * If we're dom0, we're using a real device so we need to load 2751 * the cookies with MFNs instead of PFNs. 2752 */ 2753 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2754 #else 2755 raddr = paddr; 2756 #endif 2757 2758 /* 2759 * Setup the first cookie with the physical address of the page and the 2760 * size of the page (which takes into account the initial offset into 2761 * the page. 2762 */ 2763 sgl[cnt].dmac_laddress = raddr; 2764 sgl[cnt].dmac_size = psize; 2765 sgl[cnt].dmac_type = 0; 2766 2767 /* 2768 * Save away the buffer offset into the page. We'll need this later in 2769 * the copy buffer code to help figure out the page index within the 2770 * buffer and the offset into the current page. 2771 */ 2772 sglinfo->si_buf_offset = offset; 2773 2774 /* 2775 * If we are using the copy buffer for anything over the segment 2776 * boundary, and this page is over the segment boundary. 2777 * OR 2778 * if the DMA engine can't reach the physical address. 2779 */ 2780 if (((sglinfo->si_bounce_on_seg) && 2781 ((raddr + psize) > sglinfo->si_segmask)) || 2782 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2783 /* 2784 * Increase how much copy buffer we use. We always increase by 2785 * pagesize so we don't have to worry about converting offsets. 2786 * Set a flag in the cookies dmac_type to indicate that it uses 2787 * the copy buffer. If this isn't the last cookie, go to the 2788 * next cookie (since we separate each page which uses the copy 2789 * buffer in case the copy buffer is not physically contiguous. 2790 */ 2791 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2792 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2793 if ((cnt + 1) < sglinfo->si_max_pages) { 2794 cnt++; 2795 sgl[cnt].dmac_laddress = 0; 2796 sgl[cnt].dmac_size = 0; 2797 sgl[cnt].dmac_type = 0; 2798 } 2799 } 2800 2801 /* 2802 * save this page's physical address so we can figure out if the next 2803 * page is physically contiguous. Keep decrementing size until we are 2804 * done with the buffer. 2805 */ 2806 last_page = raddr & MMU_PAGEMASK; 2807 size -= psize; 2808 2809 while (size > 0) { 2810 /* Get the size for this page (i.e. partial or full page) */ 2811 psize = MIN(size, MMU_PAGESIZE); 2812 2813 if (buftype == DMA_OTYP_PAGES) { 2814 /* get the paddr from the page_t */ 2815 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2816 paddr = pfn_to_pa(pp->p_pagenum); 2817 pp = pp->p_next; 2818 } else if (pplist != NULL) { 2819 /* index into the array of page_t's to get the paddr */ 2820 ASSERT(!PP_ISFREE(pplist[pcnt])); 2821 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2822 pcnt++; 2823 } else { 2824 /* call into the VM to get the paddr */ 2825 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2826 vaddr)); 2827 vaddr += psize; 2828 } 2829 2830 #ifdef __xpv 2831 /* 2832 * If we're dom0, we're using a real device so we need to load 2833 * the cookies with MFNs instead of PFNs. 2834 */ 2835 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2836 #else 2837 raddr = paddr; 2838 #endif 2839 2840 /* 2841 * If we are using the copy buffer for anything over the 2842 * segment boundary, and this page is over the segment 2843 * boundary. 2844 * OR 2845 * if the DMA engine can't reach the physical address. 2846 */ 2847 if (((sglinfo->si_bounce_on_seg) && 2848 ((raddr + psize) > sglinfo->si_segmask)) || 2849 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2850 2851 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2852 2853 /* 2854 * if there is something in the current cookie, go to 2855 * the next one. We only want one page in a cookie which 2856 * uses the copybuf since the copybuf doesn't have to 2857 * be physically contiguous. 2858 */ 2859 if (sgl[cnt].dmac_size != 0) { 2860 cnt++; 2861 } 2862 sgl[cnt].dmac_laddress = raddr; 2863 sgl[cnt].dmac_size = psize; 2864 #if defined(__amd64) 2865 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2866 #else 2867 /* 2868 * save the buf offset for 32-bit kernel. used in the 2869 * obsoleted interfaces. 2870 */ 2871 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2872 (dmar_object->dmao_size - size); 2873 #endif 2874 /* if this isn't the last cookie, go to the next one */ 2875 if ((cnt + 1) < sglinfo->si_max_pages) { 2876 cnt++; 2877 sgl[cnt].dmac_laddress = 0; 2878 sgl[cnt].dmac_size = 0; 2879 sgl[cnt].dmac_type = 0; 2880 } 2881 2882 /* 2883 * this page didn't need the copy buffer, if it's not physically 2884 * contiguous, or it would put us over a segment boundary, or it 2885 * puts us over the max cookie size, or the current sgl doesn't 2886 * have anything in it. 2887 */ 2888 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2889 !(raddr & sglinfo->si_segmask) || 2890 ((sgl[cnt].dmac_size + psize) > maxseg) || 2891 (sgl[cnt].dmac_size == 0)) { 2892 /* 2893 * if we're not already in a new cookie, go to the next 2894 * cookie. 2895 */ 2896 if (sgl[cnt].dmac_size != 0) { 2897 cnt++; 2898 } 2899 2900 /* save the cookie information */ 2901 sgl[cnt].dmac_laddress = raddr; 2902 sgl[cnt].dmac_size = psize; 2903 #if defined(__amd64) 2904 sgl[cnt].dmac_type = 0; 2905 #else 2906 /* 2907 * save the buf offset for 32-bit kernel. used in the 2908 * obsoleted interfaces. 2909 */ 2910 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2911 #endif 2912 2913 /* 2914 * this page didn't need the copy buffer, it is physically 2915 * contiguous with the last page, and it's <= the max cookie 2916 * size. 2917 */ 2918 } else { 2919 sgl[cnt].dmac_size += psize; 2920 2921 /* 2922 * if this exactly == the maximum cookie size, and 2923 * it isn't the last cookie, go to the next cookie. 2924 */ 2925 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2926 ((cnt + 1) < sglinfo->si_max_pages)) { 2927 cnt++; 2928 sgl[cnt].dmac_laddress = 0; 2929 sgl[cnt].dmac_size = 0; 2930 sgl[cnt].dmac_type = 0; 2931 } 2932 } 2933 2934 /* 2935 * save this page's physical address so we can figure out if the 2936 * next page is physically contiguous. Keep decrementing size 2937 * until we are done with the buffer. 2938 */ 2939 last_page = raddr; 2940 size -= psize; 2941 } 2942 2943 /* we're done, save away how many cookies the sgl has */ 2944 if (sgl[cnt].dmac_size == 0) { 2945 ASSERT(cnt < sglinfo->si_max_pages); 2946 sglinfo->si_sgl_size = cnt; 2947 } else { 2948 sglinfo->si_sgl_size = cnt + 1; 2949 } 2950 } 2951 2952 /* 2953 * rootnex_bind_slowpath() 2954 * Call in the bind path if the calling driver can't use the sgl without 2955 * modifying it. We either need to use the copy buffer and/or we will end up 2956 * with a partial bind. 2957 */ 2958 static int 2959 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2960 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2961 { 2962 rootnex_sglinfo_t *sinfo; 2963 rootnex_window_t *window; 2964 ddi_dma_cookie_t *cookie; 2965 size_t copybuf_used; 2966 size_t dmac_size; 2967 boolean_t partial; 2968 off_t cur_offset; 2969 page_t *cur_pp; 2970 major_t mnum; 2971 int e; 2972 int i; 2973 2974 2975 sinfo = &dma->dp_sglinfo; 2976 copybuf_used = 0; 2977 partial = B_FALSE; 2978 2979 /* 2980 * If we're using the copybuf, set the copybuf state in dma struct. 2981 * Needs to be first since it sets the copy buffer size. 2982 */ 2983 if (sinfo->si_copybuf_req != 0) { 2984 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2985 if (e != DDI_SUCCESS) { 2986 return (e); 2987 } 2988 } else { 2989 dma->dp_copybuf_size = 0; 2990 } 2991 2992 /* 2993 * Figure out if we need to do a partial mapping. If so, figure out 2994 * if we need to trim the buffers when we munge the sgl. 2995 */ 2996 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2997 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2998 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2999 dma->dp_partial_required = B_TRUE; 3000 if (attr->dma_attr_granular != 1) { 3001 dma->dp_trim_required = B_TRUE; 3002 } 3003 } else { 3004 dma->dp_partial_required = B_FALSE; 3005 dma->dp_trim_required = B_FALSE; 3006 } 3007 3008 /* If we need to do a partial bind, make sure the driver supports it */ 3009 if (dma->dp_partial_required && 3010 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3011 3012 mnum = ddi_driver_major(dma->dp_dip); 3013 /* 3014 * patchable which allows us to print one warning per major 3015 * number. 3016 */ 3017 if ((rootnex_bind_warn) && 3018 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3019 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3020 cmn_err(CE_WARN, "!%s: coding error detected, the " 3021 "driver is using ddi_dma_attr(9S) incorrectly. " 3022 "There is a small risk of data corruption in " 3023 "particular with large I/Os. The driver should be " 3024 "replaced with a corrected version for proper " 3025 "system operation. To disable this warning, add " 3026 "'set rootnex:rootnex_bind_warn=0' to " 3027 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3028 } 3029 return (DDI_DMA_TOOBIG); 3030 } 3031 3032 /* 3033 * we might need multiple windows, setup state to handle them. In this 3034 * code path, we will have at least one window. 3035 */ 3036 e = rootnex_setup_windows(hp, dma, attr, kmflag); 3037 if (e != DDI_SUCCESS) { 3038 rootnex_teardown_copybuf(dma); 3039 return (e); 3040 } 3041 3042 window = &dma->dp_window[0]; 3043 cookie = &dma->dp_cookies[0]; 3044 cur_offset = 0; 3045 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3046 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 3047 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3048 } 3049 3050 /* loop though all the cookies we got back from get_sgl() */ 3051 for (i = 0; i < sinfo->si_sgl_size; i++) { 3052 /* 3053 * If we're using the copy buffer, check this cookie and setup 3054 * its associated copy buffer state. If this cookie uses the 3055 * copy buffer, make sure we sync this window during dma_sync. 3056 */ 3057 if (dma->dp_copybuf_size > 0) { 3058 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 3059 cur_offset, ©buf_used, &cur_pp); 3060 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3061 window->wd_dosync = B_TRUE; 3062 } 3063 } 3064 3065 /* 3066 * save away the cookie size, since it could be modified in 3067 * the windowing code. 3068 */ 3069 dmac_size = cookie->dmac_size; 3070 3071 /* if we went over max copybuf size */ 3072 if (dma->dp_copybuf_size && 3073 (copybuf_used > dma->dp_copybuf_size)) { 3074 partial = B_TRUE; 3075 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3076 cookie, cur_offset, ©buf_used); 3077 if (e != DDI_SUCCESS) { 3078 rootnex_teardown_copybuf(dma); 3079 rootnex_teardown_windows(dma); 3080 return (e); 3081 } 3082 3083 /* 3084 * if the coookie uses the copy buffer, make sure the 3085 * new window we just moved to is set to sync. 3086 */ 3087 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3088 window->wd_dosync = B_TRUE; 3089 } 3090 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 3091 dma->dp_dip); 3092 3093 /* if the cookie cnt == max sgllen, move to the next window */ 3094 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 3095 partial = B_TRUE; 3096 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3097 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3098 cookie, attr, cur_offset); 3099 if (e != DDI_SUCCESS) { 3100 rootnex_teardown_copybuf(dma); 3101 rootnex_teardown_windows(dma); 3102 return (e); 3103 } 3104 3105 /* 3106 * if the coookie uses the copy buffer, make sure the 3107 * new window we just moved to is set to sync. 3108 */ 3109 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3110 window->wd_dosync = B_TRUE; 3111 } 3112 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 3113 dma->dp_dip); 3114 3115 /* else if we will be over maxxfer */ 3116 } else if ((window->wd_size + dmac_size) > 3117 dma->dp_maxxfer) { 3118 partial = B_TRUE; 3119 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3120 cookie); 3121 if (e != DDI_SUCCESS) { 3122 rootnex_teardown_copybuf(dma); 3123 rootnex_teardown_windows(dma); 3124 return (e); 3125 } 3126 3127 /* 3128 * if the coookie uses the copy buffer, make sure the 3129 * new window we just moved to is set to sync. 3130 */ 3131 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3132 window->wd_dosync = B_TRUE; 3133 } 3134 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 3135 dma->dp_dip); 3136 3137 /* else this cookie fits in the current window */ 3138 } else { 3139 window->wd_cookie_cnt++; 3140 window->wd_size += dmac_size; 3141 } 3142 3143 /* track our offset into the buffer, go to the next cookie */ 3144 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 3145 ASSERT(cookie->dmac_size <= dmac_size); 3146 cur_offset += dmac_size; 3147 cookie++; 3148 } 3149 3150 /* if we ended up with a zero sized window in the end, clean it up */ 3151 if (window->wd_size == 0) { 3152 hp->dmai_nwin--; 3153 window--; 3154 } 3155 3156 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3157 3158 if (!partial) { 3159 return (DDI_DMA_MAPPED); 3160 } 3161 3162 ASSERT(dma->dp_partial_required); 3163 return (DDI_DMA_PARTIAL_MAP); 3164 } 3165 3166 3167 /* 3168 * rootnex_setup_copybuf() 3169 * Called in bind slowpath. Figures out if we're going to use the copy 3170 * buffer, and if we do, sets up the basic state to handle it. 3171 */ 3172 static int 3173 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3174 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3175 { 3176 rootnex_sglinfo_t *sinfo; 3177 ddi_dma_attr_t lattr; 3178 size_t max_copybuf; 3179 int cansleep; 3180 int e; 3181 #if !defined(__amd64) 3182 int vmflag; 3183 #endif 3184 3185 3186 sinfo = &dma->dp_sglinfo; 3187 3188 /* read this first so it's consistent through the routine */ 3189 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3190 3191 /* We need to call into the rootnex on ddi_dma_sync() */ 3192 hp->dmai_rflags &= ~DMP_NOSYNC; 3193 3194 /* make sure the copybuf size <= the max size */ 3195 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3196 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3197 3198 #if !defined(__amd64) 3199 /* 3200 * if we don't have kva space to copy to/from, allocate the KVA space 3201 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3202 * the 64-bit kernel. 3203 */ 3204 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3205 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3206 3207 /* convert the sleep flags */ 3208 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3209 vmflag = VM_SLEEP; 3210 } else { 3211 vmflag = VM_NOSLEEP; 3212 } 3213 3214 /* allocate Kernel VA space that we can bcopy to/from */ 3215 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3216 vmflag); 3217 if (dma->dp_kva == NULL) { 3218 return (DDI_DMA_NORESOURCES); 3219 } 3220 } 3221 #endif 3222 3223 /* convert the sleep flags */ 3224 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3225 cansleep = 1; 3226 } else { 3227 cansleep = 0; 3228 } 3229 3230 /* 3231 * Allocate the actual copy buffer. This needs to fit within the DMA 3232 * engine limits, so we can't use kmem_alloc... We don't need 3233 * contiguous memory (sgllen) since we will be forcing windows on 3234 * sgllen anyway. 3235 */ 3236 lattr = *attr; 3237 lattr.dma_attr_align = MMU_PAGESIZE; 3238 /* 3239 * this should be < 0 to indicate no limit, but due to a bug in 3240 * the rootnex, we'll set it to the maximum positive int. 3241 */ 3242 lattr.dma_attr_sgllen = 0x7fffffff; 3243 /* 3244 * if we're using the copy buffer because of seg, use that for our 3245 * upper address limit. 3246 */ 3247 if (sinfo->si_bounce_on_seg) { 3248 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3249 } 3250 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3251 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3252 if (e != DDI_SUCCESS) { 3253 #if !defined(__amd64) 3254 if (dma->dp_kva != NULL) { 3255 vmem_free(heap_arena, dma->dp_kva, 3256 dma->dp_copybuf_size); 3257 } 3258 #endif 3259 return (DDI_DMA_NORESOURCES); 3260 } 3261 3262 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3263 size_t, dma->dp_copybuf_size); 3264 3265 return (DDI_SUCCESS); 3266 } 3267 3268 3269 /* 3270 * rootnex_setup_windows() 3271 * Called in bind slowpath to setup the window state. We always have windows 3272 * in the slowpath. Even if the window count = 1. 3273 */ 3274 static int 3275 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3276 ddi_dma_attr_t *attr, int kmflag) 3277 { 3278 rootnex_window_t *windowp; 3279 rootnex_sglinfo_t *sinfo; 3280 size_t copy_state_size; 3281 size_t win_state_size; 3282 size_t state_available; 3283 size_t space_needed; 3284 uint_t copybuf_win; 3285 uint_t maxxfer_win; 3286 size_t space_used; 3287 uint_t sglwin; 3288 3289 3290 sinfo = &dma->dp_sglinfo; 3291 3292 dma->dp_current_win = 0; 3293 hp->dmai_nwin = 0; 3294 3295 /* If we don't need to do a partial, we only have one window */ 3296 if (!dma->dp_partial_required) { 3297 dma->dp_max_win = 1; 3298 3299 /* 3300 * we need multiple windows, need to figure out the worse case number 3301 * of windows. 3302 */ 3303 } else { 3304 /* 3305 * if we need windows because we need more copy buffer that 3306 * we allow, the worse case number of windows we could need 3307 * here would be (copybuf space required / copybuf space that 3308 * we have) plus one for remainder, and plus 2 to handle the 3309 * extra pages on the trim for the first and last pages of the 3310 * buffer (a page is the minimum window size so under the right 3311 * attr settings, you could have a window for each page). 3312 * The last page will only be hit here if the size is not a 3313 * multiple of the granularity (which theoretically shouldn't 3314 * be the case but never has been enforced, so we could have 3315 * broken things without it). 3316 */ 3317 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3318 ASSERT(dma->dp_copybuf_size > 0); 3319 copybuf_win = (sinfo->si_copybuf_req / 3320 dma->dp_copybuf_size) + 1 + 2; 3321 } else { 3322 copybuf_win = 0; 3323 } 3324 3325 /* 3326 * if we need windows because we have more cookies than the H/W 3327 * can handle, the number of windows we would need here would 3328 * be (cookie count / cookies count H/W supports) plus one for 3329 * remainder, and plus 2 to handle the extra pages on the trim 3330 * (see above comment about trim) 3331 */ 3332 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3333 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 3334 + 1) + 2; 3335 } else { 3336 sglwin = 0; 3337 } 3338 3339 /* 3340 * if we need windows because we're binding more memory than the 3341 * H/W can transfer at once, the number of windows we would need 3342 * here would be (xfer count / max xfer H/W supports) plus one 3343 * for remainder, and plus 2 to handle the extra pages on the 3344 * trim (see above comment about trim) 3345 */ 3346 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3347 maxxfer_win = (dma->dp_dma.dmao_size / 3348 dma->dp_maxxfer) + 1 + 2; 3349 } else { 3350 maxxfer_win = 0; 3351 } 3352 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3353 ASSERT(dma->dp_max_win > 0); 3354 } 3355 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3356 3357 /* 3358 * Get space for window and potential copy buffer state. Before we 3359 * go and allocate memory, see if we can get away with using what's 3360 * left in the pre-allocted state or the dynamically allocated sgl. 3361 */ 3362 space_used = (uintptr_t)(sinfo->si_sgl_size * 3363 sizeof (ddi_dma_cookie_t)); 3364 3365 /* if we dynamically allocated space for the cookies */ 3366 if (dma->dp_need_to_free_cookie) { 3367 /* if we have more space in the pre-allocted buffer, use it */ 3368 ASSERT(space_used <= dma->dp_cookie_size); 3369 if ((dma->dp_cookie_size - space_used) <= 3370 rootnex_state->r_prealloc_size) { 3371 state_available = rootnex_state->r_prealloc_size; 3372 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3373 3374 /* 3375 * else, we have more free space in the dynamically allocated 3376 * buffer, i.e. the buffer wasn't worse case fragmented so we 3377 * didn't need a lot of cookies. 3378 */ 3379 } else { 3380 state_available = dma->dp_cookie_size - space_used; 3381 windowp = (rootnex_window_t *) 3382 &dma->dp_cookies[sinfo->si_sgl_size]; 3383 } 3384 3385 /* we used the pre-alloced buffer */ 3386 } else { 3387 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3388 state_available = rootnex_state->r_prealloc_size - space_used; 3389 windowp = (rootnex_window_t *) 3390 &dma->dp_cookies[sinfo->si_sgl_size]; 3391 } 3392 3393 /* 3394 * figure out how much state we need to track the copy buffer. Add an 3395 * addition 8 bytes for pointer alignemnt later. 3396 */ 3397 if (dma->dp_copybuf_size > 0) { 3398 copy_state_size = sinfo->si_max_pages * 3399 sizeof (rootnex_pgmap_t); 3400 } else { 3401 copy_state_size = 0; 3402 } 3403 /* add an additional 8 bytes for pointer alignment */ 3404 space_needed = win_state_size + copy_state_size + 0x8; 3405 3406 /* if we have enough space already, use it */ 3407 if (state_available >= space_needed) { 3408 dma->dp_window = windowp; 3409 dma->dp_need_to_free_window = B_FALSE; 3410 3411 /* not enough space, need to allocate more. */ 3412 } else { 3413 dma->dp_window = kmem_alloc(space_needed, kmflag); 3414 if (dma->dp_window == NULL) { 3415 return (DDI_DMA_NORESOURCES); 3416 } 3417 dma->dp_need_to_free_window = B_TRUE; 3418 dma->dp_window_size = space_needed; 3419 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3420 dma->dp_dip, size_t, space_needed); 3421 } 3422 3423 /* 3424 * we allocate copy buffer state and window state at the same time. 3425 * setup our copy buffer state pointers. Make sure it's aligned. 3426 */ 3427 if (dma->dp_copybuf_size > 0) { 3428 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3429 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3430 3431 #if !defined(__amd64) 3432 /* 3433 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3434 * false/NULL. Should be quicker to bzero vs loop and set. 3435 */ 3436 bzero(dma->dp_pgmap, copy_state_size); 3437 #endif 3438 } else { 3439 dma->dp_pgmap = NULL; 3440 } 3441 3442 return (DDI_SUCCESS); 3443 } 3444 3445 3446 /* 3447 * rootnex_teardown_copybuf() 3448 * cleans up after rootnex_setup_copybuf() 3449 */ 3450 static void 3451 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3452 { 3453 #if !defined(__amd64) 3454 int i; 3455 3456 /* 3457 * if we allocated kernel heap VMEM space, go through all the pages and 3458 * map out any of the ones that we're mapped into the kernel heap VMEM 3459 * arena. Then free the VMEM space. 3460 */ 3461 if (dma->dp_kva != NULL) { 3462 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3463 if (dma->dp_pgmap[i].pm_mapped) { 3464 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3465 MMU_PAGESIZE, HAT_UNLOAD); 3466 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3467 } 3468 } 3469 3470 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3471 } 3472 3473 #endif 3474 3475 /* if we allocated a copy buffer, free it */ 3476 if (dma->dp_cbaddr != NULL) { 3477 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3478 } 3479 } 3480 3481 3482 /* 3483 * rootnex_teardown_windows() 3484 * cleans up after rootnex_setup_windows() 3485 */ 3486 static void 3487 rootnex_teardown_windows(rootnex_dma_t *dma) 3488 { 3489 /* 3490 * if we had to allocate window state on the last bind (because we 3491 * didn't have enough pre-allocated space in the handle), free it. 3492 */ 3493 if (dma->dp_need_to_free_window) { 3494 kmem_free(dma->dp_window, dma->dp_window_size); 3495 } 3496 } 3497 3498 3499 /* 3500 * rootnex_init_win() 3501 * Called in bind slow path during creation of a new window. Initializes 3502 * window state to default values. 3503 */ 3504 /*ARGSUSED*/ 3505 static void 3506 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3507 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3508 { 3509 hp->dmai_nwin++; 3510 window->wd_dosync = B_FALSE; 3511 window->wd_offset = cur_offset; 3512 window->wd_size = 0; 3513 window->wd_first_cookie = cookie; 3514 window->wd_cookie_cnt = 0; 3515 window->wd_trim.tr_trim_first = B_FALSE; 3516 window->wd_trim.tr_trim_last = B_FALSE; 3517 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3518 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3519 #if !defined(__amd64) 3520 window->wd_remap_copybuf = dma->dp_cb_remaping; 3521 #endif 3522 } 3523 3524 3525 /* 3526 * rootnex_setup_cookie() 3527 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3528 * the sgl uses the copy buffer, we need to go through each cookie, figure 3529 * out if it uses the copy buffer, and if it does, save away everything we'll 3530 * need during sync. 3531 */ 3532 static void 3533 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3534 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3535 page_t **cur_pp) 3536 { 3537 boolean_t copybuf_sz_power_2; 3538 rootnex_sglinfo_t *sinfo; 3539 paddr_t paddr; 3540 uint_t pidx; 3541 uint_t pcnt; 3542 off_t poff; 3543 #if defined(__amd64) 3544 pfn_t pfn; 3545 #else 3546 page_t **pplist; 3547 #endif 3548 3549 sinfo = &dma->dp_sglinfo; 3550 3551 /* 3552 * Calculate the page index relative to the start of the buffer. The 3553 * index to the current page for our buffer is the offset into the 3554 * first page of the buffer plus our current offset into the buffer 3555 * itself, shifted of course... 3556 */ 3557 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3558 ASSERT(pidx < sinfo->si_max_pages); 3559 3560 /* if this cookie uses the copy buffer */ 3561 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3562 /* 3563 * NOTE: we know that since this cookie uses the copy buffer, it 3564 * is <= MMU_PAGESIZE. 3565 */ 3566 3567 /* 3568 * get the offset into the page. For the 64-bit kernel, get the 3569 * pfn which we'll use with seg kpm. 3570 */ 3571 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3572 #if defined(__amd64) 3573 /* mfn_to_pfn() is a NOP on i86pc */ 3574 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3575 #endif /* __amd64 */ 3576 3577 /* figure out if the copybuf size is a power of 2 */ 3578 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3579 copybuf_sz_power_2 = B_FALSE; 3580 } else { 3581 copybuf_sz_power_2 = B_TRUE; 3582 } 3583 3584 /* This page uses the copy buffer */ 3585 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3586 3587 /* 3588 * save the copy buffer KVA that we'll use with this page. 3589 * if we still fit within the copybuf, it's a simple add. 3590 * otherwise, we need to wrap over using & or % accordingly. 3591 */ 3592 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3593 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3594 *copybuf_used; 3595 } else { 3596 if (copybuf_sz_power_2) { 3597 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3598 (uintptr_t)dma->dp_cbaddr + 3599 (*copybuf_used & 3600 (dma->dp_copybuf_size - 1))); 3601 } else { 3602 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3603 (uintptr_t)dma->dp_cbaddr + 3604 (*copybuf_used % dma->dp_copybuf_size)); 3605 } 3606 } 3607 3608 /* 3609 * over write the cookie physical address with the address of 3610 * the physical address of the copy buffer page that we will 3611 * use. 3612 */ 3613 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3614 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3615 3616 #ifdef __xpv 3617 /* 3618 * If we're dom0, we're using a real device so we need to load 3619 * the cookies with MAs instead of PAs. 3620 */ 3621 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3622 #else 3623 cookie->dmac_laddress = paddr; 3624 #endif 3625 3626 /* if we have a kernel VA, it's easy, just save that address */ 3627 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3628 (sinfo->si_asp == &kas)) { 3629 /* 3630 * save away the page aligned virtual address of the 3631 * driver buffer. Offsets are handled in the sync code. 3632 */ 3633 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3634 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3635 & MMU_PAGEMASK); 3636 #if !defined(__amd64) 3637 /* 3638 * we didn't need to, and will never need to map this 3639 * page. 3640 */ 3641 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3642 #endif 3643 3644 /* we don't have a kernel VA. We need one for the bcopy. */ 3645 } else { 3646 #if defined(__amd64) 3647 /* 3648 * for the 64-bit kernel, it's easy. We use seg kpm to 3649 * get a Kernel VA for the corresponding pfn. 3650 */ 3651 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3652 #else 3653 /* 3654 * for the 32-bit kernel, this is a pain. First we'll 3655 * save away the page_t or user VA for this page. This 3656 * is needed in rootnex_dma_win() when we switch to a 3657 * new window which requires us to re-map the copy 3658 * buffer. 3659 */ 3660 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3661 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3662 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3663 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3664 } else if (pplist != NULL) { 3665 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3666 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3667 } else { 3668 dma->dp_pgmap[pidx].pm_pp = NULL; 3669 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3670 (((uintptr_t) 3671 dmar_object->dmao_obj.virt_obj.v_addr + 3672 cur_offset) & MMU_PAGEMASK); 3673 } 3674 3675 /* 3676 * save away the page aligned virtual address which was 3677 * allocated from the kernel heap arena (taking into 3678 * account if we need more copy buffer than we alloced 3679 * and use multiple windows to handle this, i.e. &,%). 3680 * NOTE: there isn't and physical memory backing up this 3681 * virtual address space currently. 3682 */ 3683 if ((*copybuf_used + MMU_PAGESIZE) <= 3684 dma->dp_copybuf_size) { 3685 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3686 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3687 MMU_PAGEMASK); 3688 } else { 3689 if (copybuf_sz_power_2) { 3690 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3691 (((uintptr_t)dma->dp_kva + 3692 (*copybuf_used & 3693 (dma->dp_copybuf_size - 1))) & 3694 MMU_PAGEMASK); 3695 } else { 3696 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3697 (((uintptr_t)dma->dp_kva + 3698 (*copybuf_used % 3699 dma->dp_copybuf_size)) & 3700 MMU_PAGEMASK); 3701 } 3702 } 3703 3704 /* 3705 * if we haven't used up the available copy buffer yet, 3706 * map the kva to the physical page. 3707 */ 3708 if (!dma->dp_cb_remaping && ((*copybuf_used + 3709 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3710 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3711 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3712 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3713 dma->dp_pgmap[pidx].pm_kaddr); 3714 } else { 3715 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3716 sinfo->si_asp, 3717 dma->dp_pgmap[pidx].pm_kaddr); 3718 } 3719 3720 /* 3721 * we've used up the available copy buffer, this page 3722 * will have to be mapped during rootnex_dma_win() when 3723 * we switch to a new window which requires a re-map 3724 * the copy buffer. (32-bit kernel only) 3725 */ 3726 } else { 3727 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3728 } 3729 #endif 3730 /* go to the next page_t */ 3731 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3732 *cur_pp = (*cur_pp)->p_next; 3733 } 3734 } 3735 3736 /* add to the copy buffer count */ 3737 *copybuf_used += MMU_PAGESIZE; 3738 3739 /* 3740 * This cookie doesn't use the copy buffer. Walk through the pages this 3741 * cookie occupies to reflect this. 3742 */ 3743 } else { 3744 /* 3745 * figure out how many pages the cookie occupies. We need to 3746 * use the original page offset of the buffer and the cookies 3747 * offset in the buffer to do this. 3748 */ 3749 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3750 pcnt = mmu_btopr(cookie->dmac_size + poff); 3751 3752 while (pcnt > 0) { 3753 #if !defined(__amd64) 3754 /* 3755 * the 32-bit kernel doesn't have seg kpm, so we need 3756 * to map in the driver buffer (if it didn't come down 3757 * with a kernel VA) on the fly. Since this page doesn't 3758 * use the copy buffer, it's not, or will it ever, have 3759 * to be mapped in. 3760 */ 3761 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3762 #endif 3763 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3764 3765 /* 3766 * we need to update pidx and cur_pp or we'll loose 3767 * track of where we are. 3768 */ 3769 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3770 *cur_pp = (*cur_pp)->p_next; 3771 } 3772 pidx++; 3773 pcnt--; 3774 } 3775 } 3776 } 3777 3778 3779 /* 3780 * rootnex_sgllen_window_boundary() 3781 * Called in the bind slow path when the next cookie causes us to exceed (in 3782 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3783 * length supported by the DMA H/W. 3784 */ 3785 static int 3786 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3787 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3788 off_t cur_offset) 3789 { 3790 off_t new_offset; 3791 size_t trim_sz; 3792 off_t coffset; 3793 3794 3795 /* 3796 * if we know we'll never have to trim, it's pretty easy. Just move to 3797 * the next window and init it. We're done. 3798 */ 3799 if (!dma->dp_trim_required) { 3800 (*windowp)++; 3801 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3802 (*windowp)->wd_cookie_cnt++; 3803 (*windowp)->wd_size = cookie->dmac_size; 3804 return (DDI_SUCCESS); 3805 } 3806 3807 /* figure out how much we need to trim from the window */ 3808 ASSERT(attr->dma_attr_granular != 0); 3809 if (dma->dp_granularity_power_2) { 3810 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3811 } else { 3812 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3813 } 3814 3815 /* The window's a whole multiple of granularity. We're done */ 3816 if (trim_sz == 0) { 3817 (*windowp)++; 3818 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3819 (*windowp)->wd_cookie_cnt++; 3820 (*windowp)->wd_size = cookie->dmac_size; 3821 return (DDI_SUCCESS); 3822 } 3823 3824 /* 3825 * The window's not a whole multiple of granularity, since we know this 3826 * is due to the sgllen, we need to go back to the last cookie and trim 3827 * that one, add the left over part of the old cookie into the new 3828 * window, and then add in the new cookie into the new window. 3829 */ 3830 3831 /* 3832 * make sure the driver isn't making us do something bad... Trimming and 3833 * sgllen == 1 don't go together. 3834 */ 3835 if (attr->dma_attr_sgllen == 1) { 3836 return (DDI_DMA_NOMAPPING); 3837 } 3838 3839 /* 3840 * first, setup the current window to account for the trim. Need to go 3841 * back to the last cookie for this. 3842 */ 3843 cookie--; 3844 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3845 (*windowp)->wd_trim.tr_last_cookie = cookie; 3846 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3847 ASSERT(cookie->dmac_size > trim_sz); 3848 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3849 (*windowp)->wd_size -= trim_sz; 3850 3851 /* save the buffer offsets for the next window */ 3852 coffset = cookie->dmac_size - trim_sz; 3853 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3854 3855 /* 3856 * set this now in case this is the first window. all other cases are 3857 * set in dma_win() 3858 */ 3859 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3860 3861 /* 3862 * initialize the next window using what's left over in the previous 3863 * cookie. 3864 */ 3865 (*windowp)++; 3866 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3867 (*windowp)->wd_cookie_cnt++; 3868 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3869 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3870 (*windowp)->wd_trim.tr_first_size = trim_sz; 3871 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3872 (*windowp)->wd_dosync = B_TRUE; 3873 } 3874 3875 /* 3876 * now go back to the current cookie and add it to the new window. set 3877 * the new window size to the what was left over from the previous 3878 * cookie and what's in the current cookie. 3879 */ 3880 cookie++; 3881 (*windowp)->wd_cookie_cnt++; 3882 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3883 3884 /* 3885 * trim plus the next cookie could put us over maxxfer (a cookie can be 3886 * a max size of maxxfer). Handle that case. 3887 */ 3888 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3889 /* 3890 * maxxfer is already a whole multiple of granularity, and this 3891 * trim will be <= the previous trim (since a cookie can't be 3892 * larger than maxxfer). Make things simple here. 3893 */ 3894 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3895 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3896 (*windowp)->wd_trim.tr_last_cookie = cookie; 3897 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3898 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3899 (*windowp)->wd_size -= trim_sz; 3900 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3901 3902 /* save the buffer offsets for the next window */ 3903 coffset = cookie->dmac_size - trim_sz; 3904 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3905 3906 /* setup the next window */ 3907 (*windowp)++; 3908 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3909 (*windowp)->wd_cookie_cnt++; 3910 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3911 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3912 coffset; 3913 (*windowp)->wd_trim.tr_first_size = trim_sz; 3914 } 3915 3916 return (DDI_SUCCESS); 3917 } 3918 3919 3920 /* 3921 * rootnex_copybuf_window_boundary() 3922 * Called in bind slowpath when we get to a window boundary because we used 3923 * up all the copy buffer that we have. 3924 */ 3925 static int 3926 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3927 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3928 size_t *copybuf_used) 3929 { 3930 rootnex_sglinfo_t *sinfo; 3931 off_t new_offset; 3932 size_t trim_sz; 3933 paddr_t paddr; 3934 off_t coffset; 3935 uint_t pidx; 3936 off_t poff; 3937 3938 3939 sinfo = &dma->dp_sglinfo; 3940 3941 /* 3942 * the copy buffer should be a whole multiple of page size. We know that 3943 * this cookie is <= MMU_PAGESIZE. 3944 */ 3945 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3946 3947 /* 3948 * from now on, all new windows in this bind need to be re-mapped during 3949 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3950 * space... 3951 */ 3952 #if !defined(__amd64) 3953 dma->dp_cb_remaping = B_TRUE; 3954 #endif 3955 3956 /* reset copybuf used */ 3957 *copybuf_used = 0; 3958 3959 /* 3960 * if we don't have to trim (since granularity is set to 1), go to the 3961 * next window and add the current cookie to it. We know the current 3962 * cookie uses the copy buffer since we're in this code path. 3963 */ 3964 if (!dma->dp_trim_required) { 3965 (*windowp)++; 3966 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3967 3968 /* Add this cookie to the new window */ 3969 (*windowp)->wd_cookie_cnt++; 3970 (*windowp)->wd_size += cookie->dmac_size; 3971 *copybuf_used += MMU_PAGESIZE; 3972 return (DDI_SUCCESS); 3973 } 3974 3975 /* 3976 * *** may need to trim, figure it out. 3977 */ 3978 3979 /* figure out how much we need to trim from the window */ 3980 if (dma->dp_granularity_power_2) { 3981 trim_sz = (*windowp)->wd_size & 3982 (hp->dmai_attr.dma_attr_granular - 1); 3983 } else { 3984 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3985 } 3986 3987 /* 3988 * if the window's a whole multiple of granularity, go to the next 3989 * window, init it, then add in the current cookie. We know the current 3990 * cookie uses the copy buffer since we're in this code path. 3991 */ 3992 if (trim_sz == 0) { 3993 (*windowp)++; 3994 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3995 3996 /* Add this cookie to the new window */ 3997 (*windowp)->wd_cookie_cnt++; 3998 (*windowp)->wd_size += cookie->dmac_size; 3999 *copybuf_used += MMU_PAGESIZE; 4000 return (DDI_SUCCESS); 4001 } 4002 4003 /* 4004 * *** We figured it out, we definitly need to trim 4005 */ 4006 4007 /* 4008 * make sure the driver isn't making us do something bad... 4009 * Trimming and sgllen == 1 don't go together. 4010 */ 4011 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4012 return (DDI_DMA_NOMAPPING); 4013 } 4014 4015 /* 4016 * first, setup the current window to account for the trim. Need to go 4017 * back to the last cookie for this. Some of the last cookie will be in 4018 * the current window, and some of the last cookie will be in the new 4019 * window. All of the current cookie will be in the new window. 4020 */ 4021 cookie--; 4022 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4023 (*windowp)->wd_trim.tr_last_cookie = cookie; 4024 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4025 ASSERT(cookie->dmac_size > trim_sz); 4026 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4027 (*windowp)->wd_size -= trim_sz; 4028 4029 /* 4030 * we're trimming the last cookie (not the current cookie). So that 4031 * last cookie may have or may not have been using the copy buffer ( 4032 * we know the cookie passed in uses the copy buffer since we're in 4033 * this code path). 4034 * 4035 * If the last cookie doesn't use the copy buffer, nothing special to 4036 * do. However, if it does uses the copy buffer, it will be both the 4037 * last page in the current window and the first page in the next 4038 * window. Since we are reusing the copy buffer (and KVA space on the 4039 * 32-bit kernel), this page will use the end of the copy buffer in the 4040 * current window, and the start of the copy buffer in the next window. 4041 * Track that info... The cookie physical address was already set to 4042 * the copy buffer physical address in setup_cookie.. 4043 */ 4044 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4045 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4046 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4047 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4048 (*windowp)->wd_trim.tr_last_pidx = pidx; 4049 (*windowp)->wd_trim.tr_last_cbaddr = 4050 dma->dp_pgmap[pidx].pm_cbaddr; 4051 #if !defined(__amd64) 4052 (*windowp)->wd_trim.tr_last_kaddr = 4053 dma->dp_pgmap[pidx].pm_kaddr; 4054 #endif 4055 } 4056 4057 /* save the buffer offsets for the next window */ 4058 coffset = cookie->dmac_size - trim_sz; 4059 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4060 4061 /* 4062 * set this now in case this is the first window. all other cases are 4063 * set in dma_win() 4064 */ 4065 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4066 4067 /* 4068 * initialize the next window using what's left over in the previous 4069 * cookie. 4070 */ 4071 (*windowp)++; 4072 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4073 (*windowp)->wd_cookie_cnt++; 4074 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4075 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4076 (*windowp)->wd_trim.tr_first_size = trim_sz; 4077 4078 /* 4079 * again, we're tracking if the last cookie uses the copy buffer. 4080 * read the comment above for more info on why we need to track 4081 * additional state. 4082 * 4083 * For the first cookie in the new window, we need reset the physical 4084 * address to DMA into to the start of the copy buffer plus any 4085 * initial page offset which may be present. 4086 */ 4087 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4088 (*windowp)->wd_dosync = B_TRUE; 4089 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4090 (*windowp)->wd_trim.tr_first_pidx = pidx; 4091 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4092 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4093 4094 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4095 poff; 4096 #ifdef __xpv 4097 /* 4098 * If we're dom0, we're using a real device so we need to load 4099 * the cookies with MAs instead of PAs. 4100 */ 4101 (*windowp)->wd_trim.tr_first_paddr = 4102 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 4103 #else 4104 (*windowp)->wd_trim.tr_first_paddr = paddr; 4105 #endif 4106 4107 #if !defined(__amd64) 4108 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4109 #endif 4110 /* account for the cookie copybuf usage in the new window */ 4111 *copybuf_used += MMU_PAGESIZE; 4112 4113 /* 4114 * every piece of code has to have a hack, and here is this 4115 * ones :-) 4116 * 4117 * There is a complex interaction between setup_cookie and the 4118 * copybuf window boundary. The complexity had to be in either 4119 * the maxxfer window, or the copybuf window, and I chose the 4120 * copybuf code. 4121 * 4122 * So in this code path, we have taken the last cookie, 4123 * virtually broken it in half due to the trim, and it happens 4124 * to use the copybuf which further complicates life. At the 4125 * same time, we have already setup the current cookie, which 4126 * is now wrong. More background info: the current cookie uses 4127 * the copybuf, so it is only a page long max. So we need to 4128 * fix the current cookies copy buffer address, physical 4129 * address, and kva for the 32-bit kernel. We due this by 4130 * bumping them by page size (of course, we can't due this on 4131 * the physical address since the copy buffer may not be 4132 * physically contiguous). 4133 */ 4134 cookie++; 4135 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4136 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4137 4138 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4139 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4140 #ifdef __xpv 4141 /* 4142 * If we're dom0, we're using a real device so we need to load 4143 * the cookies with MAs instead of PAs. 4144 */ 4145 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 4146 #else 4147 cookie->dmac_laddress = paddr; 4148 #endif 4149 4150 #if !defined(__amd64) 4151 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4152 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4153 #endif 4154 } else { 4155 /* go back to the current cookie */ 4156 cookie++; 4157 } 4158 4159 /* 4160 * add the current cookie to the new window. set the new window size to 4161 * the what was left over from the previous cookie and what's in the 4162 * current cookie. 4163 */ 4164 (*windowp)->wd_cookie_cnt++; 4165 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4166 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4167 4168 /* 4169 * we know that the cookie passed in always uses the copy buffer. We 4170 * wouldn't be here if it didn't. 4171 */ 4172 *copybuf_used += MMU_PAGESIZE; 4173 4174 return (DDI_SUCCESS); 4175 } 4176 4177 4178 /* 4179 * rootnex_maxxfer_window_boundary() 4180 * Called in bind slowpath when we get to a window boundary because we will 4181 * go over maxxfer. 4182 */ 4183 static int 4184 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4185 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4186 { 4187 size_t dmac_size; 4188 off_t new_offset; 4189 size_t trim_sz; 4190 off_t coffset; 4191 4192 4193 /* 4194 * calculate how much we have to trim off of the current cookie to equal 4195 * maxxfer. We don't have to account for granularity here since our 4196 * maxxfer already takes that into account. 4197 */ 4198 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4199 ASSERT(trim_sz <= cookie->dmac_size); 4200 ASSERT(trim_sz <= dma->dp_maxxfer); 4201 4202 /* save cookie size since we need it later and we might change it */ 4203 dmac_size = cookie->dmac_size; 4204 4205 /* 4206 * if we're not trimming the entire cookie, setup the current window to 4207 * account for the trim. 4208 */ 4209 if (trim_sz < cookie->dmac_size) { 4210 (*windowp)->wd_cookie_cnt++; 4211 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4212 (*windowp)->wd_trim.tr_last_cookie = cookie; 4213 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4214 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4215 (*windowp)->wd_size = dma->dp_maxxfer; 4216 4217 /* 4218 * set the adjusted cookie size now in case this is the first 4219 * window. All other windows are taken care of in get win 4220 */ 4221 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4222 } 4223 4224 /* 4225 * coffset is the current offset within the cookie, new_offset is the 4226 * current offset with the entire buffer. 4227 */ 4228 coffset = dmac_size - trim_sz; 4229 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4230 4231 /* initialize the next window */ 4232 (*windowp)++; 4233 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4234 (*windowp)->wd_cookie_cnt++; 4235 (*windowp)->wd_size = trim_sz; 4236 if (trim_sz < dmac_size) { 4237 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4238 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4239 coffset; 4240 (*windowp)->wd_trim.tr_first_size = trim_sz; 4241 } 4242 4243 return (DDI_SUCCESS); 4244 } 4245 4246 4247 /*ARGSUSED*/ 4248 static int 4249 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4250 off_t off, size_t len, uint_t cache_flags) 4251 { 4252 rootnex_sglinfo_t *sinfo; 4253 rootnex_pgmap_t *cbpage; 4254 rootnex_window_t *win; 4255 ddi_dma_impl_t *hp; 4256 rootnex_dma_t *dma; 4257 caddr_t fromaddr; 4258 caddr_t toaddr; 4259 uint_t psize; 4260 off_t offset; 4261 uint_t pidx; 4262 size_t size; 4263 off_t poff; 4264 int e; 4265 4266 4267 hp = (ddi_dma_impl_t *)handle; 4268 dma = (rootnex_dma_t *)hp->dmai_private; 4269 sinfo = &dma->dp_sglinfo; 4270 4271 /* 4272 * if we don't have any windows, we don't need to sync. A copybuf 4273 * will cause us to have at least one window. 4274 */ 4275 if (dma->dp_window == NULL) { 4276 return (DDI_SUCCESS); 4277 } 4278 4279 /* This window may not need to be sync'd */ 4280 win = &dma->dp_window[dma->dp_current_win]; 4281 if (!win->wd_dosync) { 4282 return (DDI_SUCCESS); 4283 } 4284 4285 /* handle off and len special cases */ 4286 if ((off == 0) || (rootnex_sync_ignore_params)) { 4287 offset = win->wd_offset; 4288 } else { 4289 offset = off; 4290 } 4291 if ((len == 0) || (rootnex_sync_ignore_params)) { 4292 size = win->wd_size; 4293 } else { 4294 size = len; 4295 } 4296 4297 /* check the sync args to make sure they make a little sense */ 4298 if (rootnex_sync_check_parms) { 4299 e = rootnex_valid_sync_parms(hp, win, offset, size, 4300 cache_flags); 4301 if (e != DDI_SUCCESS) { 4302 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4303 return (DDI_FAILURE); 4304 } 4305 } 4306 4307 /* 4308 * special case the first page to handle the offset into the page. The 4309 * offset to the current page for our buffer is the offset into the 4310 * first page of the buffer plus our current offset into the buffer 4311 * itself, masked of course. 4312 */ 4313 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4314 psize = MIN((MMU_PAGESIZE - poff), size); 4315 4316 /* go through all the pages that we want to sync */ 4317 while (size > 0) { 4318 /* 4319 * Calculate the page index relative to the start of the buffer. 4320 * The index to the current page for our buffer is the offset 4321 * into the first page of the buffer plus our current offset 4322 * into the buffer itself, shifted of course... 4323 */ 4324 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4325 ASSERT(pidx < sinfo->si_max_pages); 4326 4327 /* 4328 * if this page uses the copy buffer, we need to sync it, 4329 * otherwise, go on to the next page. 4330 */ 4331 cbpage = &dma->dp_pgmap[pidx]; 4332 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4333 (cbpage->pm_uses_copybuf == B_FALSE)); 4334 if (cbpage->pm_uses_copybuf) { 4335 /* cbaddr and kaddr should be page aligned */ 4336 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4337 MMU_PAGEOFFSET) == 0); 4338 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4339 MMU_PAGEOFFSET) == 0); 4340 4341 /* 4342 * if we're copying for the device, we are going to 4343 * copy from the drivers buffer and to the rootnex 4344 * allocated copy buffer. 4345 */ 4346 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4347 fromaddr = cbpage->pm_kaddr + poff; 4348 toaddr = cbpage->pm_cbaddr + poff; 4349 DTRACE_PROBE2(rootnex__sync__dev, 4350 dev_info_t *, dma->dp_dip, size_t, psize); 4351 4352 /* 4353 * if we're copying for the cpu/kernel, we are going to 4354 * copy from the rootnex allocated copy buffer to the 4355 * drivers buffer. 4356 */ 4357 } else { 4358 fromaddr = cbpage->pm_cbaddr + poff; 4359 toaddr = cbpage->pm_kaddr + poff; 4360 DTRACE_PROBE2(rootnex__sync__cpu, 4361 dev_info_t *, dma->dp_dip, size_t, psize); 4362 } 4363 4364 bcopy(fromaddr, toaddr, psize); 4365 } 4366 4367 /* 4368 * decrement size until we're done, update our offset into the 4369 * buffer, and get the next page size. 4370 */ 4371 size -= psize; 4372 offset += psize; 4373 psize = MIN(MMU_PAGESIZE, size); 4374 4375 /* page offset is zero for the rest of this loop */ 4376 poff = 0; 4377 } 4378 4379 return (DDI_SUCCESS); 4380 } 4381 4382 /* 4383 * rootnex_dma_sync() 4384 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4385 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4386 * is set, ddi_dma_sync() returns immediately passing back success. 4387 */ 4388 /*ARGSUSED*/ 4389 static int 4390 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4391 off_t off, size_t len, uint_t cache_flags) 4392 { 4393 #if defined(__amd64) && !defined(__xpv) 4394 if (IOMMU_USED(rdip)) { 4395 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4396 cache_flags)); 4397 } 4398 #endif 4399 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4400 cache_flags)); 4401 } 4402 4403 /* 4404 * rootnex_valid_sync_parms() 4405 * checks the parameters passed to sync to verify they are correct. 4406 */ 4407 static int 4408 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4409 off_t offset, size_t size, uint_t cache_flags) 4410 { 4411 off_t woffset; 4412 4413 4414 /* 4415 * the first part of the test to make sure the offset passed in is 4416 * within the window. 4417 */ 4418 if (offset < win->wd_offset) { 4419 return (DDI_FAILURE); 4420 } 4421 4422 /* 4423 * second and last part of the test to make sure the offset and length 4424 * passed in is within the window. 4425 */ 4426 woffset = offset - win->wd_offset; 4427 if ((woffset + size) > win->wd_size) { 4428 return (DDI_FAILURE); 4429 } 4430 4431 /* 4432 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4433 * be set too. 4434 */ 4435 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4436 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4437 return (DDI_SUCCESS); 4438 } 4439 4440 /* 4441 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4442 * should be set. Also DDI_DMA_READ should be set in the flags. 4443 */ 4444 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4445 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4446 (hp->dmai_rflags & DDI_DMA_READ)) { 4447 return (DDI_SUCCESS); 4448 } 4449 4450 return (DDI_FAILURE); 4451 } 4452 4453 4454 /*ARGSUSED*/ 4455 static int 4456 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4457 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4458 uint_t *ccountp) 4459 { 4460 rootnex_window_t *window; 4461 rootnex_trim_t *trim; 4462 ddi_dma_impl_t *hp; 4463 rootnex_dma_t *dma; 4464 #if !defined(__amd64) 4465 rootnex_sglinfo_t *sinfo; 4466 rootnex_pgmap_t *pmap; 4467 uint_t pidx; 4468 uint_t pcnt; 4469 off_t poff; 4470 int i; 4471 #endif 4472 4473 4474 hp = (ddi_dma_impl_t *)handle; 4475 dma = (rootnex_dma_t *)hp->dmai_private; 4476 #if !defined(__amd64) 4477 sinfo = &dma->dp_sglinfo; 4478 #endif 4479 4480 /* If we try and get a window which doesn't exist, return failure */ 4481 if (win >= hp->dmai_nwin) { 4482 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4483 return (DDI_FAILURE); 4484 } 4485 4486 /* 4487 * if we don't have any windows, and they're asking for the first 4488 * window, setup the cookie pointer to the first cookie in the bind. 4489 * setup our return values, then increment the cookie since we return 4490 * the first cookie on the stack. 4491 */ 4492 if (dma->dp_window == NULL) { 4493 if (win != 0) { 4494 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4495 return (DDI_FAILURE); 4496 } 4497 hp->dmai_cookie = dma->dp_cookies; 4498 *offp = 0; 4499 *lenp = dma->dp_dma.dmao_size; 4500 *ccountp = dma->dp_sglinfo.si_sgl_size; 4501 *cookiep = hp->dmai_cookie[0]; 4502 hp->dmai_cookie++; 4503 return (DDI_SUCCESS); 4504 } 4505 4506 /* sync the old window before moving on to the new one */ 4507 window = &dma->dp_window[dma->dp_current_win]; 4508 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4509 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4510 DDI_DMA_SYNC_FORCPU); 4511 } 4512 4513 #if !defined(__amd64) 4514 /* 4515 * before we move to the next window, if we need to re-map, unmap all 4516 * the pages in this window. 4517 */ 4518 if (dma->dp_cb_remaping) { 4519 /* 4520 * If we switch to this window again, we'll need to map in 4521 * on the fly next time. 4522 */ 4523 window->wd_remap_copybuf = B_TRUE; 4524 4525 /* 4526 * calculate the page index into the buffer where this window 4527 * starts, and the number of pages this window takes up. 4528 */ 4529 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4530 MMU_PAGESHIFT; 4531 poff = (sinfo->si_buf_offset + window->wd_offset) & 4532 MMU_PAGEOFFSET; 4533 pcnt = mmu_btopr(window->wd_size + poff); 4534 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4535 4536 /* unmap pages which are currently mapped in this window */ 4537 for (i = 0; i < pcnt; i++) { 4538 if (dma->dp_pgmap[pidx].pm_mapped) { 4539 hat_unload(kas.a_hat, 4540 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4541 HAT_UNLOAD); 4542 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4543 } 4544 pidx++; 4545 } 4546 } 4547 #endif 4548 4549 /* 4550 * Move to the new window. 4551 * NOTE: current_win must be set for sync to work right 4552 */ 4553 dma->dp_current_win = win; 4554 window = &dma->dp_window[win]; 4555 4556 /* if needed, adjust the first and/or last cookies for trim */ 4557 trim = &window->wd_trim; 4558 if (trim->tr_trim_first) { 4559 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4560 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4561 #if !defined(__amd64) 4562 window->wd_first_cookie->dmac_type = 4563 (window->wd_first_cookie->dmac_type & 4564 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4565 #endif 4566 if (trim->tr_first_copybuf_win) { 4567 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4568 trim->tr_first_cbaddr; 4569 #if !defined(__amd64) 4570 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4571 trim->tr_first_kaddr; 4572 #endif 4573 } 4574 } 4575 if (trim->tr_trim_last) { 4576 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4577 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4578 if (trim->tr_last_copybuf_win) { 4579 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4580 trim->tr_last_cbaddr; 4581 #if !defined(__amd64) 4582 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4583 trim->tr_last_kaddr; 4584 #endif 4585 } 4586 } 4587 4588 /* 4589 * setup the cookie pointer to the first cookie in the window. setup 4590 * our return values, then increment the cookie since we return the 4591 * first cookie on the stack. 4592 */ 4593 hp->dmai_cookie = window->wd_first_cookie; 4594 *offp = window->wd_offset; 4595 *lenp = window->wd_size; 4596 *ccountp = window->wd_cookie_cnt; 4597 *cookiep = hp->dmai_cookie[0]; 4598 hp->dmai_cookie++; 4599 4600 #if !defined(__amd64) 4601 /* re-map copybuf if required for this window */ 4602 if (dma->dp_cb_remaping) { 4603 /* 4604 * calculate the page index into the buffer where this 4605 * window starts. 4606 */ 4607 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4608 MMU_PAGESHIFT; 4609 ASSERT(pidx < sinfo->si_max_pages); 4610 4611 /* 4612 * the first page can get unmapped if it's shared with the 4613 * previous window. Even if the rest of this window is already 4614 * mapped in, we need to still check this one. 4615 */ 4616 pmap = &dma->dp_pgmap[pidx]; 4617 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4618 if (pmap->pm_pp != NULL) { 4619 pmap->pm_mapped = B_TRUE; 4620 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4621 } else if (pmap->pm_vaddr != NULL) { 4622 pmap->pm_mapped = B_TRUE; 4623 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4624 pmap->pm_kaddr); 4625 } 4626 } 4627 pidx++; 4628 4629 /* map in the rest of the pages if required */ 4630 if (window->wd_remap_copybuf) { 4631 window->wd_remap_copybuf = B_FALSE; 4632 4633 /* figure out many pages this window takes up */ 4634 poff = (sinfo->si_buf_offset + window->wd_offset) & 4635 MMU_PAGEOFFSET; 4636 pcnt = mmu_btopr(window->wd_size + poff); 4637 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4638 4639 /* map pages which require it */ 4640 for (i = 1; i < pcnt; i++) { 4641 pmap = &dma->dp_pgmap[pidx]; 4642 if (pmap->pm_uses_copybuf) { 4643 ASSERT(pmap->pm_mapped == B_FALSE); 4644 if (pmap->pm_pp != NULL) { 4645 pmap->pm_mapped = B_TRUE; 4646 i86_pp_map(pmap->pm_pp, 4647 pmap->pm_kaddr); 4648 } else if (pmap->pm_vaddr != NULL) { 4649 pmap->pm_mapped = B_TRUE; 4650 i86_va_map(pmap->pm_vaddr, 4651 sinfo->si_asp, 4652 pmap->pm_kaddr); 4653 } 4654 } 4655 pidx++; 4656 } 4657 } 4658 } 4659 #endif 4660 4661 /* if the new window uses the copy buffer, sync it for the device */ 4662 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4663 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4664 DDI_DMA_SYNC_FORDEV); 4665 } 4666 4667 return (DDI_SUCCESS); 4668 } 4669 4670 /* 4671 * rootnex_dma_win() 4672 * called from ddi_dma_getwin() 4673 */ 4674 /*ARGSUSED*/ 4675 static int 4676 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4677 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4678 uint_t *ccountp) 4679 { 4680 #if defined(__amd64) && !defined(__xpv) 4681 if (IOMMU_USED(rdip)) { 4682 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4683 cookiep, ccountp)); 4684 } 4685 #endif 4686 4687 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4688 cookiep, ccountp)); 4689 } 4690 4691 /* 4692 * ************************ 4693 * obsoleted dma routines 4694 * ************************ 4695 */ 4696 4697 /* 4698 * rootnex_dma_map() 4699 * called from ddi_dma_setup() 4700 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 4701 */ 4702 /* ARGSUSED */ 4703 static int 4704 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4705 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4706 { 4707 #if defined(__amd64) 4708 /* 4709 * this interface is not supported in 64-bit x86 kernel. See comment in 4710 * rootnex_dma_mctl() 4711 */ 4712 return (DDI_DMA_NORESOURCES); 4713 4714 #else /* 32-bit x86 kernel */ 4715 ddi_dma_handle_t *lhandlep; 4716 ddi_dma_handle_t lhandle; 4717 ddi_dma_cookie_t cookie; 4718 ddi_dma_attr_t dma_attr; 4719 ddi_dma_lim_t *dma_lim; 4720 uint_t ccnt; 4721 int e; 4722 4723 4724 /* 4725 * if the driver is just testing to see if it's possible to do the bind, 4726 * we'll use local state. Otherwise, use the handle pointer passed in. 4727 */ 4728 if (handlep == NULL) { 4729 lhandlep = &lhandle; 4730 } else { 4731 lhandlep = handlep; 4732 } 4733 4734 /* convert the limit structure to a dma_attr one */ 4735 dma_lim = dmareq->dmar_limits; 4736 dma_attr.dma_attr_version = DMA_ATTR_V0; 4737 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4738 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4739 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4740 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4741 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4742 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4743 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4744 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4745 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4746 dma_attr.dma_attr_align = MMU_PAGESIZE; 4747 dma_attr.dma_attr_flags = 0; 4748 4749 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4750 dmareq->dmar_arg, lhandlep); 4751 if (e != DDI_SUCCESS) { 4752 return (e); 4753 } 4754 4755 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4756 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4757 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4758 return (e); 4759 } 4760 4761 /* 4762 * if the driver is just testing to see if it's possible to do the bind, 4763 * free up the local state and return the result. 4764 */ 4765 if (handlep == NULL) { 4766 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4767 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4768 if (e == DDI_DMA_MAPPED) { 4769 return (DDI_DMA_MAPOK); 4770 } else { 4771 return (DDI_DMA_NOMAPPING); 4772 } 4773 } 4774 4775 return (e); 4776 #endif /* defined(__amd64) */ 4777 } 4778 4779 /* 4780 * rootnex_dma_mctl() 4781 * 4782 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 4783 */ 4784 /* ARGSUSED */ 4785 static int 4786 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4787 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4788 uint_t cache_flags) 4789 { 4790 #if defined(__amd64) 4791 /* 4792 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4793 * common implementation in genunix, so they no longer have x86 4794 * specific functionality which called into dma_ctl. 4795 * 4796 * The rest of the obsoleted interfaces were never supported in the 4797 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4798 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4799 * implementation issues. 4800 * 4801 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4802 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4803 * reflect that now too... 4804 * 4805 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4806 * not going to put this functionality into the 64-bit x86 kernel now. 4807 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4808 * that in a future release. 4809 */ 4810 return (DDI_FAILURE); 4811 4812 #else /* 32-bit x86 kernel */ 4813 ddi_dma_cookie_t lcookie; 4814 ddi_dma_cookie_t *cookie; 4815 rootnex_window_t *window; 4816 ddi_dma_impl_t *hp; 4817 rootnex_dma_t *dma; 4818 uint_t nwin; 4819 uint_t ccnt; 4820 size_t len; 4821 off_t off; 4822 int e; 4823 4824 4825 /* 4826 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4827 * hacky since were optimizing for the current interfaces and so we can 4828 * cleanup the mess in genunix. Hopefully we will remove the this 4829 * obsoleted routines someday soon. 4830 */ 4831 4832 switch (request) { 4833 4834 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4835 hp = (ddi_dma_impl_t *)handle; 4836 cookie = (ddi_dma_cookie_t *)objpp; 4837 4838 /* 4839 * convert segment to cookie. We don't distinguish between the 4840 * two :-) 4841 */ 4842 *cookie = *hp->dmai_cookie; 4843 *lenp = cookie->dmac_size; 4844 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4845 return (DDI_SUCCESS); 4846 4847 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4848 hp = (ddi_dma_impl_t *)handle; 4849 dma = (rootnex_dma_t *)hp->dmai_private; 4850 4851 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4852 return (DDI_DMA_STALE); 4853 } 4854 4855 /* handle the case where we don't have any windows */ 4856 if (dma->dp_window == NULL) { 4857 /* 4858 * if seg == NULL, and we don't have any windows, 4859 * return the first cookie in the sgl. 4860 */ 4861 if (*lenp == NULL) { 4862 dma->dp_current_cookie = 0; 4863 hp->dmai_cookie = dma->dp_cookies; 4864 *objpp = (caddr_t)handle; 4865 return (DDI_SUCCESS); 4866 4867 /* if we have more cookies, go to the next cookie */ 4868 } else { 4869 if ((dma->dp_current_cookie + 1) >= 4870 dma->dp_sglinfo.si_sgl_size) { 4871 return (DDI_DMA_DONE); 4872 } 4873 dma->dp_current_cookie++; 4874 hp->dmai_cookie++; 4875 return (DDI_SUCCESS); 4876 } 4877 } 4878 4879 /* We have one or more windows */ 4880 window = &dma->dp_window[dma->dp_current_win]; 4881 4882 /* 4883 * if seg == NULL, return the first cookie in the current 4884 * window 4885 */ 4886 if (*lenp == NULL) { 4887 dma->dp_current_cookie = 0; 4888 hp->dmai_cookie = window->wd_first_cookie; 4889 4890 /* 4891 * go to the next cookie in the window then see if we done with 4892 * this window. 4893 */ 4894 } else { 4895 if ((dma->dp_current_cookie + 1) >= 4896 window->wd_cookie_cnt) { 4897 return (DDI_DMA_DONE); 4898 } 4899 dma->dp_current_cookie++; 4900 hp->dmai_cookie++; 4901 } 4902 *objpp = (caddr_t)handle; 4903 return (DDI_SUCCESS); 4904 4905 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4906 hp = (ddi_dma_impl_t *)handle; 4907 dma = (rootnex_dma_t *)hp->dmai_private; 4908 4909 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4910 return (DDI_DMA_STALE); 4911 } 4912 4913 /* if win == NULL, return the first window in the bind */ 4914 if (*offp == NULL) { 4915 nwin = 0; 4916 4917 /* 4918 * else, go to the next window then see if we're done with all 4919 * the windows. 4920 */ 4921 } else { 4922 nwin = dma->dp_current_win + 1; 4923 if (nwin >= hp->dmai_nwin) { 4924 return (DDI_DMA_DONE); 4925 } 4926 } 4927 4928 /* switch to the next window */ 4929 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4930 &lcookie, &ccnt); 4931 ASSERT(e == DDI_SUCCESS); 4932 if (e != DDI_SUCCESS) { 4933 return (DDI_DMA_STALE); 4934 } 4935 4936 /* reset the cookie back to the first cookie in the window */ 4937 if (dma->dp_window != NULL) { 4938 window = &dma->dp_window[dma->dp_current_win]; 4939 hp->dmai_cookie = window->wd_first_cookie; 4940 } else { 4941 hp->dmai_cookie = dma->dp_cookies; 4942 } 4943 4944 *objpp = (caddr_t)handle; 4945 return (DDI_SUCCESS); 4946 4947 case DDI_DMA_FREE: /* ddi_dma_free() */ 4948 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4949 (void) rootnex_dma_freehdl(dip, rdip, handle); 4950 if (rootnex_state->r_dvma_call_list_id) { 4951 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4952 } 4953 return (DDI_SUCCESS); 4954 4955 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4956 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4957 /* should never get here, handled in genunix */ 4958 ASSERT(0); 4959 return (DDI_FAILURE); 4960 4961 case DDI_DMA_KVADDR: 4962 case DDI_DMA_GETERR: 4963 case DDI_DMA_COFF: 4964 return (DDI_FAILURE); 4965 } 4966 4967 return (DDI_FAILURE); 4968 #endif /* defined(__amd64) */ 4969 } 4970 4971 /* 4972 * ********* 4973 * FMA Code 4974 * ********* 4975 */ 4976 4977 /* 4978 * rootnex_fm_init() 4979 * FMA init busop 4980 */ 4981 /* ARGSUSED */ 4982 static int 4983 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4984 ddi_iblock_cookie_t *ibc) 4985 { 4986 *ibc = rootnex_state->r_err_ibc; 4987 4988 return (ddi_system_fmcap); 4989 } 4990 4991 /* 4992 * rootnex_dma_check() 4993 * Function called after a dma fault occurred to find out whether the 4994 * fault address is associated with a driver that is able to handle faults 4995 * and recover from faults. 4996 */ 4997 /* ARGSUSED */ 4998 static int 4999 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5000 const void *not_used) 5001 { 5002 rootnex_window_t *window; 5003 uint64_t start_addr; 5004 uint64_t fault_addr; 5005 ddi_dma_impl_t *hp; 5006 rootnex_dma_t *dma; 5007 uint64_t end_addr; 5008 size_t csize; 5009 int i; 5010 int j; 5011 5012 5013 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5014 hp = (ddi_dma_impl_t *)handle; 5015 ASSERT(hp); 5016 5017 dma = (rootnex_dma_t *)hp->dmai_private; 5018 5019 /* Get the address that we need to search for */ 5020 fault_addr = *(uint64_t *)addr; 5021 5022 /* 5023 * if we don't have any windows, we can just walk through all the 5024 * cookies. 5025 */ 5026 if (dma->dp_window == NULL) { 5027 /* for each cookie */ 5028 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5029 /* 5030 * if the faulted address is within the physical address 5031 * range of the cookie, return DDI_FM_NONFATAL. 5032 */ 5033 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5034 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5035 dma->dp_cookies[i].dmac_size))) { 5036 return (DDI_FM_NONFATAL); 5037 } 5038 } 5039 5040 /* fault_addr not within this DMA handle */ 5041 return (DDI_FM_UNKNOWN); 5042 } 5043 5044 /* we have mutiple windows, walk through each window */ 5045 for (i = 0; i < hp->dmai_nwin; i++) { 5046 window = &dma->dp_window[i]; 5047 5048 /* Go through all the cookies in the window */ 5049 for (j = 0; j < window->wd_cookie_cnt; j++) { 5050 5051 start_addr = window->wd_first_cookie[j].dmac_laddress; 5052 csize = window->wd_first_cookie[j].dmac_size; 5053 5054 /* 5055 * if we are trimming the first cookie in the window, 5056 * and this is the first cookie, adjust the start 5057 * address and size of the cookie to account for the 5058 * trim. 5059 */ 5060 if (window->wd_trim.tr_trim_first && (j == 0)) { 5061 start_addr = window->wd_trim.tr_first_paddr; 5062 csize = window->wd_trim.tr_first_size; 5063 } 5064 5065 /* 5066 * if we are trimming the last cookie in the window, 5067 * and this is the last cookie, adjust the start 5068 * address and size of the cookie to account for the 5069 * trim. 5070 */ 5071 if (window->wd_trim.tr_trim_last && 5072 (j == (window->wd_cookie_cnt - 1))) { 5073 start_addr = window->wd_trim.tr_last_paddr; 5074 csize = window->wd_trim.tr_last_size; 5075 } 5076 5077 end_addr = start_addr + csize; 5078 5079 /* 5080 * if the faulted address is within the physical 5081 * address of the cookie, return DDI_FM_NONFATAL. 5082 */ 5083 if ((fault_addr >= start_addr) && 5084 (fault_addr <= end_addr)) { 5085 return (DDI_FM_NONFATAL); 5086 } 5087 } 5088 } 5089 5090 /* fault_addr not within this DMA handle */ 5091 return (DDI_FM_UNKNOWN); 5092 } 5093 5094 /*ARGSUSED*/ 5095 static int 5096 rootnex_quiesce(dev_info_t *dip) 5097 { 5098 #if defined(__amd64) && !defined(__xpv) 5099 return (immu_quiesce()); 5100 #else 5101 return (DDI_SUCCESS); 5102 #endif 5103 } 5104 5105 #if defined(__xpv) 5106 void 5107 immu_init(void) 5108 { 5109 ; 5110 } 5111 5112 void 5113 immu_startup(void) 5114 { 5115 ; 5116 } 5117 /*ARGSUSED*/ 5118 void 5119 immu_physmem_update(uint64_t addr, uint64_t size) 5120 { 5121 ; 5122 } 5123 #endif 5124