1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * x86 root nexus driver 27 */ 28 29 #include <sys/sysmacros.h> 30 #include <sys/conf.h> 31 #include <sys/autoconf.h> 32 #include <sys/sysmacros.h> 33 #include <sys/debug.h> 34 #include <sys/psw.h> 35 #include <sys/ddidmareq.h> 36 #include <sys/promif.h> 37 #include <sys/devops.h> 38 #include <sys/kmem.h> 39 #include <sys/cmn_err.h> 40 #include <vm/seg.h> 41 #include <vm/seg_kmem.h> 42 #include <vm/seg_dev.h> 43 #include <sys/vmem.h> 44 #include <sys/mman.h> 45 #include <vm/hat.h> 46 #include <vm/as.h> 47 #include <vm/page.h> 48 #include <sys/avintr.h> 49 #include <sys/errno.h> 50 #include <sys/modctl.h> 51 #include <sys/ddi_impldefs.h> 52 #include <sys/sunddi.h> 53 #include <sys/sunndi.h> 54 #include <sys/mach_intr.h> 55 #include <sys/psm.h> 56 #include <sys/ontrap.h> 57 #include <sys/atomic.h> 58 #include <sys/sdt.h> 59 #include <sys/rootnex.h> 60 #include <vm/hat_i86.h> 61 #include <sys/ddifm.h> 62 #include <sys/ddi_isa.h> 63 #include <sys/apic.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #endif 71 72 #if defined(__amd64) && !defined(__xpv) 73 #include <sys/immu.h> 74 #endif 75 76 77 /* 78 * enable/disable extra checking of function parameters. Useful for debugging 79 * drivers. 80 */ 81 #ifdef DEBUG 82 int rootnex_alloc_check_parms = 1; 83 int rootnex_bind_check_parms = 1; 84 int rootnex_bind_check_inuse = 1; 85 int rootnex_unbind_verify_buffer = 0; 86 int rootnex_sync_check_parms = 1; 87 #else 88 int rootnex_alloc_check_parms = 0; 89 int rootnex_bind_check_parms = 0; 90 int rootnex_bind_check_inuse = 0; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 0; 93 #endif 94 95 boolean_t rootnex_dmar_not_setup; 96 97 /* Master Abort and Target Abort panic flag */ 98 int rootnex_fm_ma_ta_panic_flag = 0; 99 100 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 101 int rootnex_bind_fail = 1; 102 int rootnex_bind_warn = 1; 103 uint8_t *rootnex_warn_list; 104 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 105 #define ROOTNEX_BIND_WARNING (0x1 << 0) 106 107 /* 108 * revert back to old broken behavior of always sync'ing entire copy buffer. 109 * This is useful if be have a buggy driver which doesn't correctly pass in 110 * the offset and size into ddi_dma_sync(). 111 */ 112 int rootnex_sync_ignore_params = 0; 113 114 /* 115 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 116 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 117 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 118 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 119 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 120 * (< 8K). We will still need to allocate the copy buffer during bind though 121 * (if we need one). These can only be modified in /etc/system before rootnex 122 * attach. 123 */ 124 #if defined(__amd64) 125 int rootnex_prealloc_cookies = 65; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #else 129 int rootnex_prealloc_cookies = 33; 130 int rootnex_prealloc_windows = 4; 131 int rootnex_prealloc_copybuf = 2; 132 #endif 133 134 /* driver global state */ 135 static rootnex_state_t *rootnex_state; 136 137 #ifdef DEBUG 138 /* shortcut to rootnex counters */ 139 static uint64_t *rootnex_cnt; 140 #endif 141 142 /* 143 * XXX - does x86 even need these or are they left over from the SPARC days? 144 */ 145 /* statically defined integer/boolean properties for the root node */ 146 static rootnex_intprop_t rootnex_intprp[] = { 147 { "PAGESIZE", PAGESIZE }, 148 { "MMU_PAGESIZE", MMU_PAGESIZE }, 149 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 150 { DDI_RELATIVE_ADDRESSING, 1 }, 151 }; 152 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 153 154 /* 155 * If we're dom0, we're using a real device so we need to load 156 * the cookies with MFNs instead of PFNs. 157 */ 158 #ifdef __xpv 159 typedef maddr_t rootnex_addr_t; 160 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 161 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 162 #else 163 typedef paddr_t rootnex_addr_t; 164 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 165 #endif 166 167 #if !defined(__xpv) 168 char _depends_on[] = "misc/iommulib misc/acpica"; 169 #endif 170 171 static struct cb_ops rootnex_cb_ops = { 172 nodev, /* open */ 173 nodev, /* close */ 174 nodev, /* strategy */ 175 nodev, /* print */ 176 nodev, /* dump */ 177 nodev, /* read */ 178 nodev, /* write */ 179 nodev, /* ioctl */ 180 nodev, /* devmap */ 181 nodev, /* mmap */ 182 nodev, /* segmap */ 183 nochpoll, /* chpoll */ 184 ddi_prop_op, /* cb_prop_op */ 185 NULL, /* struct streamtab */ 186 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 187 CB_REV, /* Rev */ 188 nodev, /* cb_aread */ 189 nodev /* cb_awrite */ 190 }; 191 192 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 193 off_t offset, off_t len, caddr_t *vaddrp); 194 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 195 struct hat *hat, struct seg *seg, caddr_t addr, 196 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 197 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 198 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 201 ddi_dma_handle_t *handlep); 202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle); 204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle); 209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 213 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 218 ddi_ctl_enum_t ctlop, void *arg, void *result); 219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 220 ddi_iblock_cookie_t *ibc); 221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 224 void *); 225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 226 227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 229 ddi_dma_handle_t *handlep); 230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 231 ddi_dma_handle_t handle); 232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 234 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 236 ddi_dma_handle_t handle); 237 #if defined(__amd64) && !defined(__xpv) 238 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 239 ddi_dma_handle_t handle); 240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 243 ddi_dma_cookie_t *cookiep, uint_t ccount); 244 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 245 ddi_dma_handle_t handle); 246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 247 #endif 248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 252 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 253 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 255 ddi_dma_handle_t handle, void *v); 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 257 ddi_dma_handle_t handle); 258 259 260 static struct bus_ops rootnex_bus_ops = { 261 BUSO_REV, 262 rootnex_map, 263 NULL, 264 NULL, 265 NULL, 266 rootnex_map_fault, 267 rootnex_dma_map, 268 rootnex_dma_allochdl, 269 rootnex_dma_freehdl, 270 rootnex_dma_bindhdl, 271 rootnex_dma_unbindhdl, 272 rootnex_dma_sync, 273 rootnex_dma_win, 274 rootnex_dma_mctl, 275 rootnex_ctlops, 276 ddi_bus_prop_op, 277 i_ddi_rootnex_get_eventcookie, 278 i_ddi_rootnex_add_eventcall, 279 i_ddi_rootnex_remove_eventcall, 280 i_ddi_rootnex_post_event, 281 0, /* bus_intr_ctl */ 282 0, /* bus_config */ 283 0, /* bus_unconfig */ 284 rootnex_fm_init, /* bus_fm_init */ 285 NULL, /* bus_fm_fini */ 286 NULL, /* bus_fm_access_enter */ 287 NULL, /* bus_fm_access_exit */ 288 NULL, /* bus_powr */ 289 rootnex_intr_ops /* bus_intr_op */ 290 }; 291 292 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 293 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 294 static int rootnex_quiesce(dev_info_t *dip); 295 296 static struct dev_ops rootnex_ops = { 297 DEVO_REV, 298 0, 299 ddi_no_info, 300 nulldev, 301 nulldev, 302 rootnex_attach, 303 rootnex_detach, 304 nulldev, 305 &rootnex_cb_ops, 306 &rootnex_bus_ops, 307 NULL, 308 rootnex_quiesce, /* quiesce */ 309 }; 310 311 static struct modldrv rootnex_modldrv = { 312 &mod_driverops, 313 "i86pc root nexus", 314 &rootnex_ops 315 }; 316 317 static struct modlinkage rootnex_modlinkage = { 318 MODREV_1, 319 (void *)&rootnex_modldrv, 320 NULL 321 }; 322 323 #if defined(__amd64) && !defined(__xpv) 324 static iommulib_nexops_t iommulib_nexops = { 325 IOMMU_NEXOPS_VERSION, 326 "Rootnex IOMMU ops Vers 1.1", 327 NULL, 328 rootnex_coredma_allochdl, 329 rootnex_coredma_freehdl, 330 rootnex_coredma_bindhdl, 331 rootnex_coredma_unbindhdl, 332 rootnex_coredma_reset_cookies, 333 rootnex_coredma_get_cookies, 334 rootnex_coredma_set_cookies, 335 rootnex_coredma_clear_cookies, 336 rootnex_coredma_get_sleep_flags, 337 rootnex_coredma_sync, 338 rootnex_coredma_win, 339 rootnex_dma_map, 340 rootnex_dma_mctl, 341 rootnex_coredma_hdl_setprivate, 342 rootnex_coredma_hdl_getprivate 343 }; 344 #endif 345 346 /* 347 * extern hacks 348 */ 349 extern struct seg_ops segdev_ops; 350 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 351 #ifdef DDI_MAP_DEBUG 352 extern int ddi_map_debug_flag; 353 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 354 #endif 355 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 358 psm_intr_op_t, int *); 359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 361 362 /* 363 * Use device arena to use for device control register mappings. 364 * Various kernel memory walkers (debugger, dtrace) need to know 365 * to avoid this address range to prevent undesired device activity. 366 */ 367 extern void *device_arena_alloc(size_t size, int vm_flag); 368 extern void device_arena_free(void * vaddr, size_t size); 369 370 371 /* 372 * Internal functions 373 */ 374 static int rootnex_dma_init(); 375 static void rootnex_add_props(dev_info_t *); 376 static int rootnex_ctl_reportdev(dev_info_t *dip); 377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 380 static int rootnex_map_handle(ddi_map_req_t *mp); 381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 384 ddi_dma_attr_t *attr); 385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 386 rootnex_sglinfo_t *sglinfo); 387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 388 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 390 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 392 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 395 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 396 static void rootnex_teardown_windows(rootnex_dma_t *dma); 397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 398 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 400 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 401 size_t *copybuf_used, page_t **cur_pp); 402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 403 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 404 ddi_dma_attr_t *attr, off_t cur_offset); 405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 406 rootnex_dma_t *dma, rootnex_window_t **windowp, 407 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 409 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 411 off_t offset, size_t size, uint_t cache_flags); 412 static int rootnex_verify_buffer(rootnex_dma_t *dma); 413 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 414 const void *comp_addr, const void *not_used); 415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 416 rootnex_sglinfo_t *sglinfo); 417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 418 419 /* 420 * _init() 421 * 422 */ 423 int 424 _init(void) 425 { 426 427 rootnex_state = NULL; 428 return (mod_install(&rootnex_modlinkage)); 429 } 430 431 432 /* 433 * _info() 434 * 435 */ 436 int 437 _info(struct modinfo *modinfop) 438 { 439 return (mod_info(&rootnex_modlinkage, modinfop)); 440 } 441 442 443 /* 444 * _fini() 445 * 446 */ 447 int 448 _fini(void) 449 { 450 return (EBUSY); 451 } 452 453 454 /* 455 * rootnex_attach() 456 * 457 */ 458 static int 459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 460 { 461 int fmcap; 462 int e; 463 464 switch (cmd) { 465 case DDI_ATTACH: 466 break; 467 case DDI_RESUME: 468 #if defined(__amd64) && !defined(__xpv) 469 return (immu_unquiesce()); 470 #else 471 return (DDI_SUCCESS); 472 #endif 473 default: 474 return (DDI_FAILURE); 475 } 476 477 /* 478 * We should only have one instance of rootnex. Save it away since we 479 * don't have an easy way to get it back later. 480 */ 481 ASSERT(rootnex_state == NULL); 482 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 483 484 rootnex_state->r_dip = dip; 485 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 486 rootnex_state->r_reserved_msg_printed = B_FALSE; 487 #ifdef DEBUG 488 rootnex_cnt = &rootnex_state->r_counters[0]; 489 #endif 490 491 /* 492 * Set minimum fm capability level for i86pc platforms and then 493 * initialize error handling. Since we're the rootnex, we don't 494 * care what's returned in the fmcap field. 495 */ 496 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 497 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 498 fmcap = ddi_system_fmcap; 499 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 500 501 /* initialize DMA related state */ 502 e = rootnex_dma_init(); 503 if (e != DDI_SUCCESS) { 504 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 505 return (DDI_FAILURE); 506 } 507 508 /* Add static root node properties */ 509 rootnex_add_props(dip); 510 511 /* since we can't call ddi_report_dev() */ 512 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 513 514 /* Initialize rootnex event handle */ 515 i_ddi_rootnex_init_events(dip); 516 517 #if defined(__amd64) && !defined(__xpv) 518 e = iommulib_nexus_register(dip, &iommulib_nexops, 519 &rootnex_state->r_iommulib_handle); 520 521 ASSERT(e == DDI_SUCCESS); 522 #endif 523 524 return (DDI_SUCCESS); 525 } 526 527 528 /* 529 * rootnex_detach() 530 * 531 */ 532 /*ARGSUSED*/ 533 static int 534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 535 { 536 switch (cmd) { 537 case DDI_SUSPEND: 538 #if defined(__amd64) && !defined(__xpv) 539 return (immu_quiesce()); 540 #else 541 return (DDI_SUCCESS); 542 #endif 543 default: 544 return (DDI_FAILURE); 545 } 546 /*NOTREACHED*/ 547 548 } 549 550 551 /* 552 * rootnex_dma_init() 553 * 554 */ 555 /*ARGSUSED*/ 556 static int 557 rootnex_dma_init() 558 { 559 size_t bufsize; 560 561 562 /* 563 * size of our cookie/window/copybuf state needed in dma bind that we 564 * pre-alloc in dma_alloc_handle 565 */ 566 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 567 rootnex_state->r_prealloc_size = 568 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 569 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 570 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 571 572 /* 573 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 574 * allocate 16 extra bytes for struct pointer alignment 575 * (p->dmai_private & dma->dp_prealloc_buffer) 576 */ 577 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 578 rootnex_state->r_prealloc_size + 0x10; 579 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 580 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 581 if (rootnex_state->r_dmahdl_cache == NULL) { 582 return (DDI_FAILURE); 583 } 584 585 /* 586 * allocate array to track which major numbers we have printed warnings 587 * for. 588 */ 589 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 590 KM_SLEEP); 591 592 return (DDI_SUCCESS); 593 } 594 595 596 /* 597 * rootnex_add_props() 598 * 599 */ 600 static void 601 rootnex_add_props(dev_info_t *dip) 602 { 603 rootnex_intprop_t *rpp; 604 int i; 605 606 /* Add static integer/boolean properties to the root node */ 607 rpp = rootnex_intprp; 608 for (i = 0; i < NROOT_INTPROPS; i++) { 609 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 610 rpp[i].prop_name, rpp[i].prop_value); 611 } 612 } 613 614 615 616 /* 617 * ************************* 618 * ctlops related routines 619 * ************************* 620 */ 621 622 /* 623 * rootnex_ctlops() 624 * 625 */ 626 /*ARGSUSED*/ 627 static int 628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 629 void *arg, void *result) 630 { 631 int n, *ptr; 632 struct ddi_parent_private_data *pdp; 633 634 switch (ctlop) { 635 case DDI_CTLOPS_DMAPMAPC: 636 /* 637 * Return 'partial' to indicate that dma mapping 638 * has to be done in the main MMU. 639 */ 640 return (DDI_DMA_PARTIAL); 641 642 case DDI_CTLOPS_BTOP: 643 /* 644 * Convert byte count input to physical page units. 645 * (byte counts that are not a page-size multiple 646 * are rounded down) 647 */ 648 *(ulong_t *)result = btop(*(ulong_t *)arg); 649 return (DDI_SUCCESS); 650 651 case DDI_CTLOPS_PTOB: 652 /* 653 * Convert size in physical pages to bytes 654 */ 655 *(ulong_t *)result = ptob(*(ulong_t *)arg); 656 return (DDI_SUCCESS); 657 658 case DDI_CTLOPS_BTOPR: 659 /* 660 * Convert byte count input to physical page units 661 * (byte counts that are not a page-size multiple 662 * are rounded up) 663 */ 664 *(ulong_t *)result = btopr(*(ulong_t *)arg); 665 return (DDI_SUCCESS); 666 667 case DDI_CTLOPS_INITCHILD: 668 return (impl_ddi_sunbus_initchild(arg)); 669 670 case DDI_CTLOPS_UNINITCHILD: 671 impl_ddi_sunbus_removechild(arg); 672 return (DDI_SUCCESS); 673 674 case DDI_CTLOPS_REPORTDEV: 675 return (rootnex_ctl_reportdev(rdip)); 676 677 case DDI_CTLOPS_IOMIN: 678 /* 679 * Nothing to do here but reflect back.. 680 */ 681 return (DDI_SUCCESS); 682 683 case DDI_CTLOPS_REGSIZE: 684 case DDI_CTLOPS_NREGS: 685 break; 686 687 case DDI_CTLOPS_SIDDEV: 688 if (ndi_dev_is_prom_node(rdip)) 689 return (DDI_SUCCESS); 690 if (ndi_dev_is_persistent_node(rdip)) 691 return (DDI_SUCCESS); 692 return (DDI_FAILURE); 693 694 case DDI_CTLOPS_POWER: 695 return ((*pm_platform_power)((power_req_t *)arg)); 696 697 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 698 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 699 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 700 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 701 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 702 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 703 if (!rootnex_state->r_reserved_msg_printed) { 704 rootnex_state->r_reserved_msg_printed = B_TRUE; 705 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 706 "1 or more reserved/obsolete operations."); 707 } 708 return (DDI_FAILURE); 709 710 default: 711 return (DDI_FAILURE); 712 } 713 /* 714 * The rest are for "hardware" properties 715 */ 716 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 717 return (DDI_FAILURE); 718 719 if (ctlop == DDI_CTLOPS_NREGS) { 720 ptr = (int *)result; 721 *ptr = pdp->par_nreg; 722 } else { 723 off_t *size = (off_t *)result; 724 725 ptr = (int *)arg; 726 n = *ptr; 727 if (n >= pdp->par_nreg) { 728 return (DDI_FAILURE); 729 } 730 *size = (off_t)pdp->par_reg[n].regspec_size; 731 } 732 return (DDI_SUCCESS); 733 } 734 735 736 /* 737 * rootnex_ctl_reportdev() 738 * 739 */ 740 static int 741 rootnex_ctl_reportdev(dev_info_t *dev) 742 { 743 int i, n, len, f_len = 0; 744 char *buf; 745 746 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 747 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 748 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 749 len = strlen(buf); 750 751 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 752 753 struct regspec *rp = sparc_pd_getreg(dev, i); 754 755 if (i == 0) 756 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 757 ": "); 758 else 759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 760 " and "); 761 len = strlen(buf); 762 763 switch (rp->regspec_bustype) { 764 765 case BTEISA: 766 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 767 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 768 break; 769 770 case BTISA: 771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 772 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 773 break; 774 775 default: 776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 777 "space %x offset %x", 778 rp->regspec_bustype, rp->regspec_addr); 779 break; 780 } 781 len = strlen(buf); 782 } 783 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 784 int pri; 785 786 if (i != 0) { 787 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 788 ","); 789 len = strlen(buf); 790 } 791 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 793 " sparc ipl %d", pri); 794 len = strlen(buf); 795 } 796 #ifdef DEBUG 797 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 798 cmn_err(CE_NOTE, "next message is truncated: " 799 "printed length 1024, real length %d", f_len); 800 } 801 #endif /* DEBUG */ 802 cmn_err(CE_CONT, "?%s\n", buf); 803 kmem_free(buf, REPORTDEV_BUFSIZE); 804 return (DDI_SUCCESS); 805 } 806 807 808 /* 809 * ****************** 810 * map related code 811 * ****************** 812 */ 813 814 /* 815 * rootnex_map() 816 * 817 */ 818 static int 819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 820 off_t len, caddr_t *vaddrp) 821 { 822 struct regspec *rp, tmp_reg; 823 ddi_map_req_t mr = *mp; /* Get private copy of request */ 824 int error; 825 826 mp = &mr; 827 828 switch (mp->map_op) { 829 case DDI_MO_MAP_LOCKED: 830 case DDI_MO_UNMAP: 831 case DDI_MO_MAP_HANDLE: 832 break; 833 default: 834 #ifdef DDI_MAP_DEBUG 835 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 836 mp->map_op); 837 #endif /* DDI_MAP_DEBUG */ 838 return (DDI_ME_UNIMPLEMENTED); 839 } 840 841 if (mp->map_flags & DDI_MF_USER_MAPPING) { 842 #ifdef DDI_MAP_DEBUG 843 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 844 #endif /* DDI_MAP_DEBUG */ 845 return (DDI_ME_UNIMPLEMENTED); 846 } 847 848 /* 849 * First, if given an rnumber, convert it to a regspec... 850 * (Presumably, this is on behalf of a child of the root node?) 851 */ 852 853 if (mp->map_type == DDI_MT_RNUMBER) { 854 855 int rnumber = mp->map_obj.rnumber; 856 #ifdef DDI_MAP_DEBUG 857 static char *out_of_range = 858 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 859 #endif /* DDI_MAP_DEBUG */ 860 861 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 862 if (rp == NULL) { 863 #ifdef DDI_MAP_DEBUG 864 cmn_err(CE_WARN, out_of_range, rnumber, 865 ddi_get_name(rdip)); 866 #endif /* DDI_MAP_DEBUG */ 867 return (DDI_ME_RNUMBER_RANGE); 868 } 869 870 /* 871 * Convert the given ddi_map_req_t from rnumber to regspec... 872 */ 873 874 mp->map_type = DDI_MT_REGSPEC; 875 mp->map_obj.rp = rp; 876 } 877 878 /* 879 * Adjust offset and length correspnding to called values... 880 * XXX: A non-zero length means override the one in the regspec 881 * XXX: (regardless of what's in the parent's range?) 882 */ 883 884 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 885 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 886 887 #ifdef DDI_MAP_DEBUG 888 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 889 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 890 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 891 len, mp->map_handlep); 892 #endif /* DDI_MAP_DEBUG */ 893 894 /* 895 * I/O or memory mapping: 896 * 897 * <bustype=0, addr=x, len=x>: memory 898 * <bustype=1, addr=x, len=x>: i/o 899 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 900 */ 901 902 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 903 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 904 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 905 ddi_get_name(rdip), rp->regspec_bustype, 906 rp->regspec_addr, rp->regspec_size); 907 return (DDI_ME_INVAL); 908 } 909 910 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 911 /* 912 * compatibility i/o mapping 913 */ 914 rp->regspec_bustype += (uint_t)offset; 915 } else { 916 /* 917 * Normal memory or i/o mapping 918 */ 919 rp->regspec_addr += (uint_t)offset; 920 } 921 922 if (len != 0) 923 rp->regspec_size = (uint_t)len; 924 925 #ifdef DDI_MAP_DEBUG 926 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 927 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 928 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 929 offset, len, mp->map_handlep); 930 #endif /* DDI_MAP_DEBUG */ 931 932 /* 933 * Apply any parent ranges at this level, if applicable. 934 * (This is where nexus specific regspec translation takes place. 935 * Use of this function is implicit agreement that translation is 936 * provided via ddi_apply_range.) 937 */ 938 939 #ifdef DDI_MAP_DEBUG 940 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 941 ddi_get_name(dip), ddi_get_name(rdip)); 942 #endif /* DDI_MAP_DEBUG */ 943 944 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 945 return (error); 946 947 switch (mp->map_op) { 948 case DDI_MO_MAP_LOCKED: 949 950 /* 951 * Set up the locked down kernel mapping to the regspec... 952 */ 953 954 return (rootnex_map_regspec(mp, vaddrp)); 955 956 case DDI_MO_UNMAP: 957 958 /* 959 * Release mapping... 960 */ 961 962 return (rootnex_unmap_regspec(mp, vaddrp)); 963 964 case DDI_MO_MAP_HANDLE: 965 966 return (rootnex_map_handle(mp)); 967 968 default: 969 return (DDI_ME_UNIMPLEMENTED); 970 } 971 } 972 973 974 /* 975 * rootnex_map_fault() 976 * 977 * fault in mappings for requestors 978 */ 979 /*ARGSUSED*/ 980 static int 981 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 982 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 983 uint_t lock) 984 { 985 986 #ifdef DDI_MAP_DEBUG 987 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 988 ddi_map_debug(" Seg <%s>\n", 989 seg->s_ops == &segdev_ops ? "segdev" : 990 seg == &kvseg ? "segkmem" : "NONE!"); 991 #endif /* DDI_MAP_DEBUG */ 992 993 /* 994 * This is all terribly broken, but it is a start 995 * 996 * XXX Note that this test means that segdev_ops 997 * must be exported from seg_dev.c. 998 * XXX What about devices with their own segment drivers? 999 */ 1000 if (seg->s_ops == &segdev_ops) { 1001 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1002 1003 if (hat == NULL) { 1004 /* 1005 * This is one plausible interpretation of 1006 * a null hat i.e. use the first hat on the 1007 * address space hat list which by convention is 1008 * the hat of the system MMU. At alternative 1009 * would be to panic .. this might well be better .. 1010 */ 1011 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1012 hat = seg->s_as->a_hat; 1013 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1014 } 1015 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1016 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1017 } else if (seg == &kvseg && dp == NULL) { 1018 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1019 HAT_LOAD_LOCK); 1020 } else 1021 return (DDI_FAILURE); 1022 return (DDI_SUCCESS); 1023 } 1024 1025 1026 /* 1027 * rootnex_map_regspec() 1028 * we don't support mapping of I/O cards above 4Gb 1029 */ 1030 static int 1031 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1032 { 1033 rootnex_addr_t rbase; 1034 void *cvaddr; 1035 uint_t npages, pgoffset; 1036 struct regspec *rp; 1037 ddi_acc_hdl_t *hp; 1038 ddi_acc_impl_t *ap; 1039 uint_t hat_acc_flags; 1040 paddr_t pbase; 1041 1042 rp = mp->map_obj.rp; 1043 hp = mp->map_handlep; 1044 1045 #ifdef DDI_MAP_DEBUG 1046 ddi_map_debug( 1047 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1048 rp->regspec_bustype, rp->regspec_addr, 1049 rp->regspec_size, mp->map_handlep); 1050 #endif /* DDI_MAP_DEBUG */ 1051 1052 /* 1053 * I/O or memory mapping 1054 * 1055 * <bustype=0, addr=x, len=x>: memory 1056 * <bustype=1, addr=x, len=x>: i/o 1057 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1058 */ 1059 1060 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1061 cmn_err(CE_WARN, "rootnex: invalid register spec" 1062 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1063 rp->regspec_addr, rp->regspec_size); 1064 return (DDI_FAILURE); 1065 } 1066 1067 if (rp->regspec_bustype != 0) { 1068 /* 1069 * I/O space - needs a handle. 1070 */ 1071 if (hp == NULL) { 1072 return (DDI_FAILURE); 1073 } 1074 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1075 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1076 impl_acc_hdl_init(hp); 1077 1078 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1079 #ifdef DDI_MAP_DEBUG 1080 ddi_map_debug("rootnex_map_regspec: mmap() " 1081 "to I/O space is not supported.\n"); 1082 #endif /* DDI_MAP_DEBUG */ 1083 return (DDI_ME_INVAL); 1084 } else { 1085 /* 1086 * 1275-compliant vs. compatibility i/o mapping 1087 */ 1088 *vaddrp = 1089 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1090 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1091 ((caddr_t)(uintptr_t)rp->regspec_addr); 1092 #ifdef __xpv 1093 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1094 hp->ah_pfn = xen_assign_pfn( 1095 mmu_btop((ulong_t)rp->regspec_addr & 1096 MMU_PAGEMASK)); 1097 } else { 1098 hp->ah_pfn = mmu_btop( 1099 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1100 } 1101 #else 1102 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1103 MMU_PAGEMASK); 1104 #endif 1105 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1106 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1107 } 1108 1109 #ifdef DDI_MAP_DEBUG 1110 ddi_map_debug( 1111 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1112 rp->regspec_size, *vaddrp); 1113 #endif /* DDI_MAP_DEBUG */ 1114 return (DDI_SUCCESS); 1115 } 1116 1117 /* 1118 * Memory space 1119 */ 1120 1121 if (hp != NULL) { 1122 /* 1123 * hat layer ignores 1124 * hp->ah_acc.devacc_attr_endian_flags. 1125 */ 1126 switch (hp->ah_acc.devacc_attr_dataorder) { 1127 case DDI_STRICTORDER_ACC: 1128 hat_acc_flags = HAT_STRICTORDER; 1129 break; 1130 case DDI_UNORDERED_OK_ACC: 1131 hat_acc_flags = HAT_UNORDERED_OK; 1132 break; 1133 case DDI_MERGING_OK_ACC: 1134 hat_acc_flags = HAT_MERGING_OK; 1135 break; 1136 case DDI_LOADCACHING_OK_ACC: 1137 hat_acc_flags = HAT_LOADCACHING_OK; 1138 break; 1139 case DDI_STORECACHING_OK_ACC: 1140 hat_acc_flags = HAT_STORECACHING_OK; 1141 break; 1142 } 1143 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1144 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1145 impl_acc_hdl_init(hp); 1146 hp->ah_hat_flags = hat_acc_flags; 1147 } else { 1148 hat_acc_flags = HAT_STRICTORDER; 1149 } 1150 1151 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1152 #ifdef __xpv 1153 /* 1154 * If we're dom0, we're using a real device so we need to translate 1155 * the MA to a PA. 1156 */ 1157 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1158 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1159 } else { 1160 pbase = rbase; 1161 } 1162 #else 1163 pbase = rbase; 1164 #endif 1165 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1166 1167 if (rp->regspec_size == 0) { 1168 #ifdef DDI_MAP_DEBUG 1169 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1170 #endif /* DDI_MAP_DEBUG */ 1171 return (DDI_ME_INVAL); 1172 } 1173 1174 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1175 /* extra cast to make gcc happy */ 1176 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1177 } else { 1178 npages = mmu_btopr(rp->regspec_size + pgoffset); 1179 1180 #ifdef DDI_MAP_DEBUG 1181 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1182 "physical %llx", npages, pbase); 1183 #endif /* DDI_MAP_DEBUG */ 1184 1185 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1186 if (cvaddr == NULL) 1187 return (DDI_ME_NORESOURCES); 1188 1189 /* 1190 * Now map in the pages we've allocated... 1191 */ 1192 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1193 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1194 HAT_LOAD_LOCK); 1195 *vaddrp = (caddr_t)cvaddr + pgoffset; 1196 1197 /* save away pfn and npages for FMA */ 1198 hp = mp->map_handlep; 1199 if (hp) { 1200 hp->ah_pfn = mmu_btop(pbase); 1201 hp->ah_pnum = npages; 1202 } 1203 } 1204 1205 #ifdef DDI_MAP_DEBUG 1206 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1207 #endif /* DDI_MAP_DEBUG */ 1208 return (DDI_SUCCESS); 1209 } 1210 1211 1212 /* 1213 * rootnex_unmap_regspec() 1214 * 1215 */ 1216 static int 1217 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1218 { 1219 caddr_t addr = (caddr_t)*vaddrp; 1220 uint_t npages, pgoffset; 1221 struct regspec *rp; 1222 1223 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1224 return (0); 1225 1226 rp = mp->map_obj.rp; 1227 1228 if (rp->regspec_size == 0) { 1229 #ifdef DDI_MAP_DEBUG 1230 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1231 #endif /* DDI_MAP_DEBUG */ 1232 return (DDI_ME_INVAL); 1233 } 1234 1235 /* 1236 * I/O or memory mapping: 1237 * 1238 * <bustype=0, addr=x, len=x>: memory 1239 * <bustype=1, addr=x, len=x>: i/o 1240 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1241 */ 1242 if (rp->regspec_bustype != 0) { 1243 /* 1244 * This is I/O space, which requires no particular 1245 * processing on unmap since it isn't mapped in the 1246 * first place. 1247 */ 1248 return (DDI_SUCCESS); 1249 } 1250 1251 /* 1252 * Memory space 1253 */ 1254 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1255 npages = mmu_btopr(rp->regspec_size + pgoffset); 1256 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1257 device_arena_free(addr - pgoffset, ptob(npages)); 1258 1259 /* 1260 * Destroy the pointer - the mapping has logically gone 1261 */ 1262 *vaddrp = NULL; 1263 1264 return (DDI_SUCCESS); 1265 } 1266 1267 1268 /* 1269 * rootnex_map_handle() 1270 * 1271 */ 1272 static int 1273 rootnex_map_handle(ddi_map_req_t *mp) 1274 { 1275 rootnex_addr_t rbase; 1276 ddi_acc_hdl_t *hp; 1277 uint_t pgoffset; 1278 struct regspec *rp; 1279 paddr_t pbase; 1280 1281 rp = mp->map_obj.rp; 1282 1283 #ifdef DDI_MAP_DEBUG 1284 ddi_map_debug( 1285 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1286 rp->regspec_bustype, rp->regspec_addr, 1287 rp->regspec_size, mp->map_handlep); 1288 #endif /* DDI_MAP_DEBUG */ 1289 1290 /* 1291 * I/O or memory mapping: 1292 * 1293 * <bustype=0, addr=x, len=x>: memory 1294 * <bustype=1, addr=x, len=x>: i/o 1295 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1296 */ 1297 if (rp->regspec_bustype != 0) { 1298 /* 1299 * This refers to I/O space, and we don't support "mapping" 1300 * I/O space to a user. 1301 */ 1302 return (DDI_FAILURE); 1303 } 1304 1305 /* 1306 * Set up the hat_flags for the mapping. 1307 */ 1308 hp = mp->map_handlep; 1309 1310 switch (hp->ah_acc.devacc_attr_endian_flags) { 1311 case DDI_NEVERSWAP_ACC: 1312 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1313 break; 1314 case DDI_STRUCTURE_LE_ACC: 1315 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1316 break; 1317 case DDI_STRUCTURE_BE_ACC: 1318 return (DDI_FAILURE); 1319 default: 1320 return (DDI_REGS_ACC_CONFLICT); 1321 } 1322 1323 switch (hp->ah_acc.devacc_attr_dataorder) { 1324 case DDI_STRICTORDER_ACC: 1325 break; 1326 case DDI_UNORDERED_OK_ACC: 1327 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1328 break; 1329 case DDI_MERGING_OK_ACC: 1330 hp->ah_hat_flags |= HAT_MERGING_OK; 1331 break; 1332 case DDI_LOADCACHING_OK_ACC: 1333 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1334 break; 1335 case DDI_STORECACHING_OK_ACC: 1336 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1337 break; 1338 default: 1339 return (DDI_FAILURE); 1340 } 1341 1342 rbase = (rootnex_addr_t)rp->regspec_addr & 1343 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1344 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1345 1346 if (rp->regspec_size == 0) 1347 return (DDI_ME_INVAL); 1348 1349 #ifdef __xpv 1350 /* 1351 * If we're dom0, we're using a real device so we need to translate 1352 * the MA to a PA. 1353 */ 1354 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1355 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1356 (rbase & MMU_PAGEOFFSET); 1357 } else { 1358 pbase = rbase; 1359 } 1360 #else 1361 pbase = rbase; 1362 #endif 1363 1364 hp->ah_pfn = mmu_btop(pbase); 1365 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1366 1367 return (DDI_SUCCESS); 1368 } 1369 1370 1371 1372 /* 1373 * ************************ 1374 * interrupt related code 1375 * ************************ 1376 */ 1377 1378 /* 1379 * rootnex_intr_ops() 1380 * bus_intr_op() function for interrupt support 1381 */ 1382 /* ARGSUSED */ 1383 static int 1384 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1385 ddi_intr_handle_impl_t *hdlp, void *result) 1386 { 1387 struct intrspec *ispec; 1388 1389 DDI_INTR_NEXDBG((CE_CONT, 1390 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1391 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1392 1393 /* Process the interrupt operation */ 1394 switch (intr_op) { 1395 case DDI_INTROP_GETCAP: 1396 /* First check with pcplusmp */ 1397 if (psm_intr_ops == NULL) 1398 return (DDI_FAILURE); 1399 1400 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1401 *(int *)result = 0; 1402 return (DDI_FAILURE); 1403 } 1404 break; 1405 case DDI_INTROP_SETCAP: 1406 if (psm_intr_ops == NULL) 1407 return (DDI_FAILURE); 1408 1409 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1410 return (DDI_FAILURE); 1411 break; 1412 case DDI_INTROP_ALLOC: 1413 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1414 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1415 case DDI_INTROP_FREE: 1416 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1417 return (rootnex_free_intr_fixed(rdip, hdlp)); 1418 case DDI_INTROP_GETPRI: 1419 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1420 return (DDI_FAILURE); 1421 *(int *)result = ispec->intrspec_pri; 1422 break; 1423 case DDI_INTROP_SETPRI: 1424 /* Validate the interrupt priority passed to us */ 1425 if (*(int *)result > LOCK_LEVEL) 1426 return (DDI_FAILURE); 1427 1428 /* Ensure that PSM is all initialized and ispec is ok */ 1429 if ((psm_intr_ops == NULL) || 1430 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1431 return (DDI_FAILURE); 1432 1433 /* Change the priority */ 1434 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1435 PSM_FAILURE) 1436 return (DDI_FAILURE); 1437 1438 /* update the ispec with the new priority */ 1439 ispec->intrspec_pri = *(int *)result; 1440 break; 1441 case DDI_INTROP_ADDISR: 1442 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1443 return (DDI_FAILURE); 1444 ispec->intrspec_func = hdlp->ih_cb_func; 1445 break; 1446 case DDI_INTROP_REMISR: 1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1448 return (DDI_FAILURE); 1449 ispec->intrspec_func = (uint_t (*)()) 0; 1450 break; 1451 case DDI_INTROP_ENABLE: 1452 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1453 return (DDI_FAILURE); 1454 1455 /* Call psmi to translate irq with the dip */ 1456 if (psm_intr_ops == NULL) 1457 return (DDI_FAILURE); 1458 1459 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1460 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1461 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1462 return (DDI_FAILURE); 1463 1464 /* Add the interrupt handler */ 1465 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1466 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1467 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1468 return (DDI_FAILURE); 1469 break; 1470 case DDI_INTROP_DISABLE: 1471 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1472 return (DDI_FAILURE); 1473 1474 /* Call psm_ops() to translate irq with the dip */ 1475 if (psm_intr_ops == NULL) 1476 return (DDI_FAILURE); 1477 1478 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1479 (void) (*psm_intr_ops)(rdip, hdlp, 1480 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1481 1482 /* Remove the interrupt handler */ 1483 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1484 hdlp->ih_cb_func, hdlp->ih_vector); 1485 break; 1486 case DDI_INTROP_SETMASK: 1487 if (psm_intr_ops == NULL) 1488 return (DDI_FAILURE); 1489 1490 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1491 return (DDI_FAILURE); 1492 break; 1493 case DDI_INTROP_CLRMASK: 1494 if (psm_intr_ops == NULL) 1495 return (DDI_FAILURE); 1496 1497 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1498 return (DDI_FAILURE); 1499 break; 1500 case DDI_INTROP_GETPENDING: 1501 if (psm_intr_ops == NULL) 1502 return (DDI_FAILURE); 1503 1504 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1505 result)) { 1506 *(int *)result = 0; 1507 return (DDI_FAILURE); 1508 } 1509 break; 1510 case DDI_INTROP_NAVAIL: 1511 case DDI_INTROP_NINTRS: 1512 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1513 if (*(int *)result == 0) { 1514 /* 1515 * Special case for 'pcic' driver' only. This driver 1516 * driver is a child of 'isa' and 'rootnex' drivers. 1517 * 1518 * See detailed comments on this in the function 1519 * rootnex_get_ispec(). 1520 * 1521 * Children of 'pcic' send 'NINITR' request all the 1522 * way to rootnex driver. But, the 'pdp->par_nintr' 1523 * field may not initialized. So, we fake it here 1524 * to return 1 (a la what PCMCIA nexus does). 1525 */ 1526 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1527 *(int *)result = 1; 1528 else 1529 return (DDI_FAILURE); 1530 } 1531 break; 1532 case DDI_INTROP_SUPPORTED_TYPES: 1533 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1534 break; 1535 default: 1536 return (DDI_FAILURE); 1537 } 1538 1539 return (DDI_SUCCESS); 1540 } 1541 1542 1543 /* 1544 * rootnex_get_ispec() 1545 * convert an interrupt number to an interrupt specification. 1546 * The interrupt number determines which interrupt spec will be 1547 * returned if more than one exists. 1548 * 1549 * Look into the parent private data area of the 'rdip' to find out 1550 * the interrupt specification. First check to make sure there is 1551 * one that matchs "inumber" and then return a pointer to it. 1552 * 1553 * Return NULL if one could not be found. 1554 * 1555 * NOTE: This is needed for rootnex_intr_ops() 1556 */ 1557 static struct intrspec * 1558 rootnex_get_ispec(dev_info_t *rdip, int inum) 1559 { 1560 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1561 1562 /* 1563 * Special case handling for drivers that provide their own 1564 * intrspec structures instead of relying on the DDI framework. 1565 * 1566 * A broken hardware driver in ON could potentially provide its 1567 * own intrspec structure, instead of relying on the hardware. 1568 * If these drivers are children of 'rootnex' then we need to 1569 * continue to provide backward compatibility to them here. 1570 * 1571 * Following check is a special case for 'pcic' driver which 1572 * was found to have broken hardwre andby provides its own intrspec. 1573 * 1574 * Verbatim comments from this driver are shown here: 1575 * "Don't use the ddi_add_intr since we don't have a 1576 * default intrspec in all cases." 1577 * 1578 * Since an 'ispec' may not be always created for it, 1579 * check for that and create one if so. 1580 * 1581 * NOTE: Currently 'pcic' is the only driver found to do this. 1582 */ 1583 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1584 pdp->par_nintr = 1; 1585 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1586 pdp->par_nintr, KM_SLEEP); 1587 } 1588 1589 /* Validate the interrupt number */ 1590 if (inum >= pdp->par_nintr) 1591 return (NULL); 1592 1593 /* Get the interrupt structure pointer and return that */ 1594 return ((struct intrspec *)&pdp->par_intr[inum]); 1595 } 1596 1597 /* 1598 * Allocate interrupt vector for FIXED (legacy) type. 1599 */ 1600 static int 1601 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1602 void *result) 1603 { 1604 struct intrspec *ispec; 1605 ddi_intr_handle_impl_t info_hdl; 1606 int ret; 1607 int free_phdl = 0; 1608 apic_get_type_t type_info; 1609 1610 if (psm_intr_ops == NULL) 1611 return (DDI_FAILURE); 1612 1613 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1614 return (DDI_FAILURE); 1615 1616 /* 1617 * If the PSM module is "APIX" then pass the request for it 1618 * to allocate the vector now. 1619 */ 1620 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1621 info_hdl.ih_private = &type_info; 1622 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1623 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1624 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1625 free_phdl = 1; 1626 i_ddi_alloc_intr_phdl(hdlp); 1627 } 1628 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1629 ret = (*psm_intr_ops)(rdip, hdlp, 1630 PSM_INTR_OP_ALLOC_VECTORS, result); 1631 if (free_phdl) { /* free up the phdl structure */ 1632 free_phdl = 0; 1633 i_ddi_free_intr_phdl(hdlp); 1634 hdlp->ih_private = NULL; 1635 } 1636 } else { 1637 /* 1638 * No APIX module; fall back to the old scheme where the 1639 * interrupt vector is allocated during ddi_enable_intr() call. 1640 */ 1641 hdlp->ih_pri = ispec->intrspec_pri; 1642 *(int *)result = hdlp->ih_scratch1; 1643 ret = DDI_SUCCESS; 1644 } 1645 1646 return (ret); 1647 } 1648 1649 /* 1650 * Free up interrupt vector for FIXED (legacy) type. 1651 */ 1652 static int 1653 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1654 { 1655 struct intrspec *ispec; 1656 struct ddi_parent_private_data *pdp; 1657 ddi_intr_handle_impl_t info_hdl; 1658 int ret; 1659 apic_get_type_t type_info; 1660 1661 if (psm_intr_ops == NULL) 1662 return (DDI_FAILURE); 1663 1664 /* 1665 * If the PSM module is "APIX" then pass the request for it 1666 * to free up the vector now. 1667 */ 1668 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1669 info_hdl.ih_private = &type_info; 1670 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1671 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1672 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1673 return (DDI_FAILURE); 1674 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1675 ret = (*psm_intr_ops)(rdip, hdlp, 1676 PSM_INTR_OP_FREE_VECTORS, NULL); 1677 } else { 1678 /* 1679 * No APIX module; fall back to the old scheme where 1680 * the interrupt vector was already freed during 1681 * ddi_disable_intr() call. 1682 */ 1683 ret = DDI_SUCCESS; 1684 } 1685 1686 pdp = ddi_get_parent_data(rdip); 1687 1688 /* 1689 * Special case for 'pcic' driver' only. 1690 * If an intrspec was created for it, clean it up here 1691 * See detailed comments on this in the function 1692 * rootnex_get_ispec(). 1693 */ 1694 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1695 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1696 pdp->par_nintr); 1697 /* 1698 * Set it to zero; so that 1699 * DDI framework doesn't free it again 1700 */ 1701 pdp->par_intr = NULL; 1702 pdp->par_nintr = 0; 1703 } 1704 1705 return (ret); 1706 } 1707 1708 1709 /* 1710 * ****************** 1711 * dma related code 1712 * ****************** 1713 */ 1714 1715 /*ARGSUSED*/ 1716 static int 1717 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1718 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1719 ddi_dma_handle_t *handlep) 1720 { 1721 uint64_t maxsegmentsize_ll; 1722 uint_t maxsegmentsize; 1723 ddi_dma_impl_t *hp; 1724 rootnex_dma_t *dma; 1725 uint64_t count_max; 1726 uint64_t seg; 1727 int kmflag; 1728 int e; 1729 1730 1731 /* convert our sleep flags */ 1732 if (waitfp == DDI_DMA_SLEEP) { 1733 kmflag = KM_SLEEP; 1734 } else { 1735 kmflag = KM_NOSLEEP; 1736 } 1737 1738 /* 1739 * We try to do only one memory allocation here. We'll do a little 1740 * pointer manipulation later. If the bind ends up taking more than 1741 * our prealloc's space, we'll have to allocate more memory in the 1742 * bind operation. Not great, but much better than before and the 1743 * best we can do with the current bind interfaces. 1744 */ 1745 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1746 if (hp == NULL) 1747 return (DDI_DMA_NORESOURCES); 1748 1749 /* Do our pointer manipulation now, align the structures */ 1750 hp->dmai_private = (void *)(((uintptr_t)hp + 1751 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1752 dma = (rootnex_dma_t *)hp->dmai_private; 1753 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1754 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1755 1756 /* setup the handle */ 1757 rootnex_clean_dmahdl(hp); 1758 hp->dmai_error.err_fep = NULL; 1759 hp->dmai_error.err_cf = NULL; 1760 dma->dp_dip = rdip; 1761 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1762 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1763 1764 /* 1765 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1766 * is being used. Set the upper limit to the seg value. 1767 * There will be enough DVMA space to always get addresses 1768 * that will match the constraints. 1769 */ 1770 if (IOMMU_USED(rdip) && 1771 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1772 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1773 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1774 } else 1775 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1776 1777 hp->dmai_minxfer = attr->dma_attr_minxfer; 1778 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1779 hp->dmai_rdip = rdip; 1780 hp->dmai_attr = *attr; 1781 1782 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1783 dma->dp_sglinfo.si_cancross = B_FALSE; 1784 else 1785 dma->dp_sglinfo.si_cancross = B_TRUE; 1786 1787 /* we don't need to worry about the SPL since we do a tryenter */ 1788 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1789 1790 /* 1791 * Figure out our maximum segment size. If the segment size is greater 1792 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1793 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1794 * dma_attr_count_max are size-1 type values. 1795 * 1796 * Maximum segment size is the largest physically contiguous chunk of 1797 * memory that we can return from a bind (i.e. the maximum size of a 1798 * single cookie). 1799 */ 1800 1801 /* handle the rollover cases */ 1802 seg = attr->dma_attr_seg + 1; 1803 if (seg < attr->dma_attr_seg) { 1804 seg = attr->dma_attr_seg; 1805 } 1806 count_max = attr->dma_attr_count_max + 1; 1807 if (count_max < attr->dma_attr_count_max) { 1808 count_max = attr->dma_attr_count_max; 1809 } 1810 1811 /* 1812 * granularity may or may not be a power of two. If it isn't, we can't 1813 * use a simple mask. 1814 */ 1815 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1816 dma->dp_granularity_power_2 = B_FALSE; 1817 } else { 1818 dma->dp_granularity_power_2 = B_TRUE; 1819 } 1820 1821 /* 1822 * maxxfer should be a whole multiple of granularity. If we're going to 1823 * break up a window because we're greater than maxxfer, we might as 1824 * well make sure it's maxxfer is a whole multiple so we don't have to 1825 * worry about triming the window later on for this case. 1826 */ 1827 if (attr->dma_attr_granular > 1) { 1828 if (dma->dp_granularity_power_2) { 1829 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1830 (attr->dma_attr_maxxfer & 1831 (attr->dma_attr_granular - 1)); 1832 } else { 1833 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1834 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1835 } 1836 } else { 1837 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1838 } 1839 1840 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1841 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1842 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1843 maxsegmentsize = 0xFFFFFFFF; 1844 } else { 1845 maxsegmentsize = maxsegmentsize_ll; 1846 } 1847 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1848 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1849 1850 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1851 if (rootnex_alloc_check_parms) { 1852 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1853 if (e != DDI_SUCCESS) { 1854 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1855 (void) rootnex_dma_freehdl(dip, rdip, 1856 (ddi_dma_handle_t)hp); 1857 return (e); 1858 } 1859 } 1860 1861 *handlep = (ddi_dma_handle_t)hp; 1862 1863 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1864 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1865 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1866 1867 return (DDI_SUCCESS); 1868 } 1869 1870 1871 /* 1872 * rootnex_dma_allochdl() 1873 * called from ddi_dma_alloc_handle(). 1874 */ 1875 static int 1876 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1877 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1878 { 1879 int retval = DDI_SUCCESS; 1880 #if defined(__amd64) && !defined(__xpv) 1881 1882 if (IOMMU_UNITIALIZED(rdip)) { 1883 retval = iommulib_nex_open(dip, rdip); 1884 1885 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1886 return (retval); 1887 } 1888 1889 if (IOMMU_UNUSED(rdip)) { 1890 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1891 handlep); 1892 } else { 1893 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1894 waitfp, arg, handlep); 1895 } 1896 #else 1897 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1898 handlep); 1899 #endif 1900 switch (retval) { 1901 case DDI_DMA_NORESOURCES: 1902 if (waitfp != DDI_DMA_DONTWAIT) { 1903 ddi_set_callback(waitfp, arg, 1904 &rootnex_state->r_dvma_call_list_id); 1905 } 1906 break; 1907 case DDI_SUCCESS: 1908 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1909 break; 1910 default: 1911 break; 1912 } 1913 return (retval); 1914 } 1915 1916 /*ARGSUSED*/ 1917 static int 1918 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1919 ddi_dma_handle_t handle) 1920 { 1921 ddi_dma_impl_t *hp; 1922 rootnex_dma_t *dma; 1923 1924 1925 hp = (ddi_dma_impl_t *)handle; 1926 dma = (rootnex_dma_t *)hp->dmai_private; 1927 1928 /* unbind should have been called first */ 1929 ASSERT(!dma->dp_inuse); 1930 1931 mutex_destroy(&dma->dp_mutex); 1932 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1933 1934 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1935 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1936 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1937 1938 return (DDI_SUCCESS); 1939 } 1940 1941 /* 1942 * rootnex_dma_freehdl() 1943 * called from ddi_dma_free_handle(). 1944 */ 1945 static int 1946 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1947 { 1948 int ret; 1949 1950 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1951 #if defined(__amd64) && !defined(__xpv) 1952 if (IOMMU_USED(rdip)) 1953 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1954 else 1955 #endif 1956 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1957 1958 if (rootnex_state->r_dvma_call_list_id) 1959 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1960 1961 return (ret); 1962 } 1963 1964 /*ARGSUSED*/ 1965 static int 1966 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1967 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1968 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1969 { 1970 rootnex_sglinfo_t *sinfo; 1971 ddi_dma_obj_t *dmao; 1972 struct dvmaseg *dvs; 1973 ddi_dma_cookie_t *cookie; 1974 ddi_dma_attr_t *attr; 1975 ddi_dma_impl_t *hp; 1976 rootnex_dma_t *dma; 1977 int kmflag; 1978 int e; 1979 uint_t ncookies; 1980 1981 hp = (ddi_dma_impl_t *)handle; 1982 dma = (rootnex_dma_t *)hp->dmai_private; 1983 sinfo = &dma->dp_sglinfo; 1984 attr = &hp->dmai_attr; 1985 1986 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1987 dma->dp_sleep_flags = KM_SLEEP; 1988 } else { 1989 dma->dp_sleep_flags = KM_NOSLEEP; 1990 } 1991 1992 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1993 1994 /* 1995 * This is useful for debugging a driver. Not as useful in a production 1996 * system. The only time this will fail is if you have a driver bug. 1997 */ 1998 if (rootnex_bind_check_inuse) { 1999 /* 2000 * No one else should ever have this lock unless someone else 2001 * is trying to use this handle. So contention on the lock 2002 * is the same as inuse being set. 2003 */ 2004 e = mutex_tryenter(&dma->dp_mutex); 2005 if (e == 0) { 2006 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2007 return (DDI_DMA_INUSE); 2008 } 2009 if (dma->dp_inuse) { 2010 mutex_exit(&dma->dp_mutex); 2011 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2012 return (DDI_DMA_INUSE); 2013 } 2014 dma->dp_inuse = B_TRUE; 2015 mutex_exit(&dma->dp_mutex); 2016 } 2017 2018 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2019 if (rootnex_bind_check_parms) { 2020 e = rootnex_valid_bind_parms(dmareq, attr); 2021 if (e != DDI_SUCCESS) { 2022 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2023 rootnex_clean_dmahdl(hp); 2024 return (e); 2025 } 2026 } 2027 2028 /* save away the original bind info */ 2029 dma->dp_dma = dmareq->dmar_object; 2030 2031 #if defined(__amd64) && !defined(__xpv) 2032 if (IOMMU_USED(rdip)) { 2033 dmao = &dma->dp_dvma; 2034 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2035 switch (e) { 2036 case DDI_SUCCESS: 2037 if (sinfo->si_cancross || 2038 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2039 dmao->dmao_size > sinfo->si_max_cookie_size) { 2040 dma->dp_dvma_used = B_TRUE; 2041 break; 2042 } 2043 sinfo->si_sgl_size = 1; 2044 hp->dmai_rflags |= DMP_NOSYNC; 2045 2046 dma->dp_dvma_used = B_TRUE; 2047 dma->dp_need_to_free_cookie = B_FALSE; 2048 2049 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2050 cookie = hp->dmai_cookie = dma->dp_cookies = 2051 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2052 cookie->dmac_laddress = dvs->dvs_start + 2053 dmao->dmao_obj.dvma_obj.dv_off; 2054 cookie->dmac_size = dvs->dvs_len; 2055 cookie->dmac_type = 0; 2056 2057 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2058 rdip); 2059 goto fast; 2060 case DDI_ENOTSUP: 2061 break; 2062 default: 2063 rootnex_clean_dmahdl(hp); 2064 return (e); 2065 } 2066 } 2067 2068 #endif 2069 2070 /* 2071 * Figure out a rough estimate of what maximum number of pages 2072 * this buffer could use (a high estimate of course). 2073 */ 2074 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2075 2076 if (dma->dp_dvma_used) { 2077 /* 2078 * The number of physical pages is the worst case. 2079 * 2080 * For DVMA, the worst case is the length divided 2081 * by the maximum cookie length, plus 1. Add to that 2082 * the number of segment boundaries potentially crossed, and 2083 * the additional number of DVMA segments that was returned. 2084 * 2085 * In the normal case, for modern devices, si_cancross will 2086 * be false, and dv_nseg will be 1, and the fast path will 2087 * have been taken above. 2088 */ 2089 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2090 + 1; 2091 if (sinfo->si_cancross) 2092 ncookies += 2093 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2094 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2095 2096 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2097 } else { 2098 dmao = &dma->dp_dma; 2099 } 2100 2101 /* 2102 * We'll use the pre-allocated cookies for any bind that will *always* 2103 * fit (more important to be consistent, we don't want to create 2104 * additional degenerate cases). 2105 */ 2106 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2107 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2108 dma->dp_need_to_free_cookie = B_FALSE; 2109 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2110 uint_t, sinfo->si_max_pages); 2111 2112 /* 2113 * For anything larger than that, we'll go ahead and allocate the 2114 * maximum number of pages we expect to see. Hopefuly, we won't be 2115 * seeing this path in the fast path for high performance devices very 2116 * frequently. 2117 * 2118 * a ddi bind interface that allowed the driver to provide storage to 2119 * the bind interface would speed this case up. 2120 */ 2121 } else { 2122 /* convert the sleep flags */ 2123 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2124 kmflag = KM_SLEEP; 2125 } else { 2126 kmflag = KM_NOSLEEP; 2127 } 2128 2129 /* 2130 * Save away how much memory we allocated. If we're doing a 2131 * nosleep, the alloc could fail... 2132 */ 2133 dma->dp_cookie_size = sinfo->si_max_pages * 2134 sizeof (ddi_dma_cookie_t); 2135 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2136 if (dma->dp_cookies == NULL) { 2137 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2138 rootnex_clean_dmahdl(hp); 2139 return (DDI_DMA_NORESOURCES); 2140 } 2141 dma->dp_need_to_free_cookie = B_TRUE; 2142 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2143 uint_t, sinfo->si_max_pages); 2144 } 2145 hp->dmai_cookie = dma->dp_cookies; 2146 2147 /* 2148 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2149 * looking at the constraints in the dma structure. It will then put 2150 * some additional state about the sgl in the dma struct (i.e. is 2151 * the sgl clean, or do we need to do some munging; how many pages 2152 * need to be copied, etc.) 2153 */ 2154 if (dma->dp_dvma_used) 2155 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2156 else 2157 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2158 2159 out: 2160 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2161 /* if we don't need a copy buffer, we don't need to sync */ 2162 if (sinfo->si_copybuf_req == 0) { 2163 hp->dmai_rflags |= DMP_NOSYNC; 2164 } 2165 2166 /* 2167 * if we don't need the copybuf and we don't need to do a partial, we 2168 * hit the fast path. All the high performance devices should be trying 2169 * to hit this path. To hit this path, a device should be able to reach 2170 * all of memory, shouldn't try to bind more than it can transfer, and 2171 * the buffer shouldn't require more cookies than the driver/device can 2172 * handle [sgllen]). 2173 */ 2174 if ((sinfo->si_copybuf_req == 0) && 2175 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 2176 (dmao->dmao_size < dma->dp_maxxfer)) { 2177 fast: 2178 /* 2179 * If the driver supports FMA, insert the handle in the FMA DMA 2180 * handle cache. 2181 */ 2182 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2183 hp->dmai_error.err_cf = rootnex_dma_check; 2184 2185 /* 2186 * copy out the first cookie and ccountp, set the cookie 2187 * pointer to the second cookie. The first cookie is passed 2188 * back on the stack. Additional cookies are accessed via 2189 * ddi_dma_nextcookie() 2190 */ 2191 *cookiep = dma->dp_cookies[0]; 2192 *ccountp = sinfo->si_sgl_size; 2193 hp->dmai_cookie++; 2194 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2195 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2196 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2197 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2198 uint_t, dmao->dmao_size, uint_t, *ccountp); 2199 2200 2201 return (DDI_DMA_MAPPED); 2202 } 2203 2204 /* 2205 * go to the slow path, we may need to alloc more memory, create 2206 * multiple windows, and munge up a sgl to make the device happy. 2207 */ 2208 2209 /* 2210 * With the IOMMU mapobject method used, we should never hit 2211 * the slow path. If we do, something is seriously wrong. 2212 * Clean up and return an error. 2213 */ 2214 2215 if (dma->dp_dvma_used) { 2216 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2217 &dma->dp_dvma); 2218 e = DDI_DMA_NOMAPPING; 2219 } else { 2220 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2221 kmflag); 2222 } 2223 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2224 if (dma->dp_need_to_free_cookie) { 2225 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2226 } 2227 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2228 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2229 return (e); 2230 } 2231 2232 /* 2233 * If the driver supports FMA, insert the handle in the FMA DMA handle 2234 * cache. 2235 */ 2236 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2237 hp->dmai_error.err_cf = rootnex_dma_check; 2238 2239 /* if the first window uses the copy buffer, sync it for the device */ 2240 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2241 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2242 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2243 DDI_DMA_SYNC_FORDEV); 2244 } 2245 2246 /* 2247 * copy out the first cookie and ccountp, set the cookie pointer to the 2248 * second cookie. Make sure the partial flag is set/cleared correctly. 2249 * If we have a partial map (i.e. multiple windows), the number of 2250 * cookies we return is the number of cookies in the first window. 2251 */ 2252 if (e == DDI_DMA_MAPPED) { 2253 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2254 *ccountp = sinfo->si_sgl_size; 2255 hp->dmai_nwin = 1; 2256 } else { 2257 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2258 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2259 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2260 } 2261 *cookiep = dma->dp_cookies[0]; 2262 hp->dmai_cookie++; 2263 2264 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2265 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2266 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2267 dmao->dmao_size, uint_t, *ccountp); 2268 return (e); 2269 } 2270 2271 /* 2272 * rootnex_dma_bindhdl() 2273 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2274 */ 2275 static int 2276 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2277 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2278 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2279 { 2280 int ret; 2281 #if defined(__amd64) && !defined(__xpv) 2282 if (IOMMU_USED(rdip)) 2283 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2284 cookiep, ccountp); 2285 else 2286 #endif 2287 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2288 cookiep, ccountp); 2289 2290 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2291 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2292 &rootnex_state->r_dvma_call_list_id); 2293 } 2294 2295 return (ret); 2296 } 2297 2298 2299 2300 /*ARGSUSED*/ 2301 static int 2302 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2303 ddi_dma_handle_t handle) 2304 { 2305 ddi_dma_impl_t *hp; 2306 rootnex_dma_t *dma; 2307 int e; 2308 2309 hp = (ddi_dma_impl_t *)handle; 2310 dma = (rootnex_dma_t *)hp->dmai_private; 2311 2312 /* make sure the buffer wasn't free'd before calling unbind */ 2313 if (rootnex_unbind_verify_buffer) { 2314 e = rootnex_verify_buffer(dma); 2315 if (e != DDI_SUCCESS) { 2316 ASSERT(0); 2317 return (DDI_FAILURE); 2318 } 2319 } 2320 2321 /* sync the current window before unbinding the buffer */ 2322 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2323 (hp->dmai_rflags & DDI_DMA_READ)) { 2324 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2325 DDI_DMA_SYNC_FORCPU); 2326 } 2327 2328 /* 2329 * cleanup and copy buffer or window state. if we didn't use the copy 2330 * buffer or windows, there won't be much to do :-) 2331 */ 2332 rootnex_teardown_copybuf(dma); 2333 rootnex_teardown_windows(dma); 2334 2335 if (IOMMU_USED(rdip)) 2336 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2337 &dma->dp_dvma); 2338 2339 /* 2340 * If we had to allocate space to for the worse case sgl (it didn't 2341 * fit into our pre-allocate buffer), free that up now 2342 */ 2343 if (dma->dp_need_to_free_cookie) { 2344 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2345 } 2346 2347 /* 2348 * clean up the handle so it's ready for the next bind (i.e. if the 2349 * handle is reused). 2350 */ 2351 rootnex_clean_dmahdl(hp); 2352 hp->dmai_error.err_cf = NULL; 2353 2354 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2355 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2356 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2357 2358 return (DDI_SUCCESS); 2359 } 2360 2361 /* 2362 * rootnex_dma_unbindhdl() 2363 * called from ddi_dma_unbind_handle() 2364 */ 2365 /*ARGSUSED*/ 2366 static int 2367 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2368 ddi_dma_handle_t handle) 2369 { 2370 int ret; 2371 2372 #if defined(__amd64) && !defined(__xpv) 2373 if (IOMMU_USED(rdip)) 2374 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2375 else 2376 #endif 2377 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2378 2379 if (rootnex_state->r_dvma_call_list_id) 2380 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2381 2382 return (ret); 2383 } 2384 2385 #if defined(__amd64) && !defined(__xpv) 2386 2387 static int 2388 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2389 { 2390 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2391 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2392 2393 if (dma->dp_sleep_flags != KM_SLEEP && 2394 dma->dp_sleep_flags != KM_NOSLEEP) 2395 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2396 return (dma->dp_sleep_flags); 2397 } 2398 /*ARGSUSED*/ 2399 static void 2400 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2401 { 2402 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2403 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2404 rootnex_window_t *window; 2405 2406 if (dma->dp_window) { 2407 window = &dma->dp_window[dma->dp_current_win]; 2408 hp->dmai_cookie = window->wd_first_cookie; 2409 } else { 2410 hp->dmai_cookie = dma->dp_cookies; 2411 } 2412 hp->dmai_cookie++; 2413 } 2414 2415 /*ARGSUSED*/ 2416 static int 2417 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2418 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2419 { 2420 int i; 2421 int km_flags; 2422 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2423 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2424 rootnex_window_t *window; 2425 ddi_dma_cookie_t *cp; 2426 ddi_dma_cookie_t *cookie; 2427 2428 ASSERT(*cookiepp == NULL); 2429 ASSERT(*ccountp == 0); 2430 2431 if (dma->dp_window) { 2432 window = &dma->dp_window[dma->dp_current_win]; 2433 cp = window->wd_first_cookie; 2434 *ccountp = window->wd_cookie_cnt; 2435 } else { 2436 cp = dma->dp_cookies; 2437 *ccountp = dma->dp_sglinfo.si_sgl_size; 2438 } 2439 2440 km_flags = rootnex_coredma_get_sleep_flags(handle); 2441 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2442 if (cookie == NULL) { 2443 return (DDI_DMA_NORESOURCES); 2444 } 2445 2446 for (i = 0; i < *ccountp; i++) { 2447 cookie[i].dmac_notused = cp[i].dmac_notused; 2448 cookie[i].dmac_type = cp[i].dmac_type; 2449 cookie[i].dmac_address = cp[i].dmac_address; 2450 cookie[i].dmac_size = cp[i].dmac_size; 2451 } 2452 2453 *cookiepp = cookie; 2454 2455 return (DDI_SUCCESS); 2456 } 2457 2458 /*ARGSUSED*/ 2459 static int 2460 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2461 ddi_dma_cookie_t *cookiep, uint_t ccount) 2462 { 2463 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2464 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2465 rootnex_window_t *window; 2466 ddi_dma_cookie_t *cur_cookiep; 2467 2468 ASSERT(cookiep); 2469 ASSERT(ccount != 0); 2470 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2471 2472 if (dma->dp_window) { 2473 window = &dma->dp_window[dma->dp_current_win]; 2474 dma->dp_saved_cookies = window->wd_first_cookie; 2475 window->wd_first_cookie = cookiep; 2476 ASSERT(ccount == window->wd_cookie_cnt); 2477 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2478 + window->wd_first_cookie; 2479 } else { 2480 dma->dp_saved_cookies = dma->dp_cookies; 2481 dma->dp_cookies = cookiep; 2482 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2483 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2484 + dma->dp_cookies; 2485 } 2486 2487 dma->dp_need_to_switch_cookies = B_TRUE; 2488 hp->dmai_cookie = cur_cookiep; 2489 2490 return (DDI_SUCCESS); 2491 } 2492 2493 /*ARGSUSED*/ 2494 static int 2495 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2496 { 2497 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2498 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2499 rootnex_window_t *window; 2500 ddi_dma_cookie_t *cur_cookiep; 2501 ddi_dma_cookie_t *cookie_array; 2502 uint_t ccount; 2503 2504 /* check if cookies have not been switched */ 2505 if (dma->dp_need_to_switch_cookies == B_FALSE) 2506 return (DDI_SUCCESS); 2507 2508 ASSERT(dma->dp_saved_cookies); 2509 2510 if (dma->dp_window) { 2511 window = &dma->dp_window[dma->dp_current_win]; 2512 cookie_array = window->wd_first_cookie; 2513 window->wd_first_cookie = dma->dp_saved_cookies; 2514 dma->dp_saved_cookies = NULL; 2515 ccount = window->wd_cookie_cnt; 2516 cur_cookiep = (hp->dmai_cookie - cookie_array) 2517 + window->wd_first_cookie; 2518 } else { 2519 cookie_array = dma->dp_cookies; 2520 dma->dp_cookies = dma->dp_saved_cookies; 2521 dma->dp_saved_cookies = NULL; 2522 ccount = dma->dp_sglinfo.si_sgl_size; 2523 cur_cookiep = (hp->dmai_cookie - cookie_array) 2524 + dma->dp_cookies; 2525 } 2526 2527 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2528 2529 hp->dmai_cookie = cur_cookiep; 2530 2531 dma->dp_need_to_switch_cookies = B_FALSE; 2532 2533 return (DDI_SUCCESS); 2534 } 2535 2536 #endif 2537 2538 static struct as * 2539 rootnex_get_as(ddi_dma_obj_t *dmao) 2540 { 2541 struct as *asp; 2542 2543 switch (dmao->dmao_type) { 2544 case DMA_OTYP_VADDR: 2545 case DMA_OTYP_BUFVADDR: 2546 asp = dmao->dmao_obj.virt_obj.v_as; 2547 if (asp == NULL) 2548 asp = &kas; 2549 break; 2550 default: 2551 asp = NULL; 2552 break; 2553 } 2554 return (asp); 2555 } 2556 2557 /* 2558 * rootnex_verify_buffer() 2559 * verify buffer wasn't free'd 2560 */ 2561 static int 2562 rootnex_verify_buffer(rootnex_dma_t *dma) 2563 { 2564 page_t **pplist; 2565 caddr_t vaddr; 2566 uint_t pcnt; 2567 uint_t poff; 2568 page_t *pp; 2569 char b; 2570 int i; 2571 2572 /* Figure out how many pages this buffer occupies */ 2573 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2574 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2575 } else { 2576 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2577 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2578 } 2579 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2580 2581 switch (dma->dp_dma.dmao_type) { 2582 case DMA_OTYP_PAGES: 2583 /* 2584 * for a linked list of pp's walk through them to make sure 2585 * they're locked and not free. 2586 */ 2587 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2588 for (i = 0; i < pcnt; i++) { 2589 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2590 return (DDI_FAILURE); 2591 } 2592 pp = pp->p_next; 2593 } 2594 break; 2595 2596 case DMA_OTYP_VADDR: 2597 case DMA_OTYP_BUFVADDR: 2598 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2599 /* 2600 * for an array of pp's walk through them to make sure they're 2601 * not free. It's possible that they may not be locked. 2602 */ 2603 if (pplist) { 2604 for (i = 0; i < pcnt; i++) { 2605 if (PP_ISFREE(pplist[i])) { 2606 return (DDI_FAILURE); 2607 } 2608 } 2609 2610 /* For a virtual address, try to peek at each page */ 2611 } else { 2612 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2613 for (i = 0; i < pcnt; i++) { 2614 if (ddi_peek8(NULL, vaddr, &b) == 2615 DDI_FAILURE) 2616 return (DDI_FAILURE); 2617 vaddr += MMU_PAGESIZE; 2618 } 2619 } 2620 } 2621 break; 2622 2623 default: 2624 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2625 break; 2626 } 2627 2628 return (DDI_SUCCESS); 2629 } 2630 2631 2632 /* 2633 * rootnex_clean_dmahdl() 2634 * Clean the dma handle. This should be called on a handle alloc and an 2635 * unbind handle. Set the handle state to the default settings. 2636 */ 2637 static void 2638 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2639 { 2640 rootnex_dma_t *dma; 2641 2642 2643 dma = (rootnex_dma_t *)hp->dmai_private; 2644 2645 hp->dmai_nwin = 0; 2646 dma->dp_current_cookie = 0; 2647 dma->dp_copybuf_size = 0; 2648 dma->dp_window = NULL; 2649 dma->dp_cbaddr = NULL; 2650 dma->dp_inuse = B_FALSE; 2651 dma->dp_dvma_used = B_FALSE; 2652 dma->dp_need_to_free_cookie = B_FALSE; 2653 dma->dp_need_to_switch_cookies = B_FALSE; 2654 dma->dp_saved_cookies = NULL; 2655 dma->dp_sleep_flags = KM_PANIC; 2656 dma->dp_need_to_free_window = B_FALSE; 2657 dma->dp_partial_required = B_FALSE; 2658 dma->dp_trim_required = B_FALSE; 2659 dma->dp_sglinfo.si_copybuf_req = 0; 2660 #if !defined(__amd64) 2661 dma->dp_cb_remaping = B_FALSE; 2662 dma->dp_kva = NULL; 2663 #endif 2664 2665 /* FMA related initialization */ 2666 hp->dmai_fault = 0; 2667 hp->dmai_fault_check = NULL; 2668 hp->dmai_fault_notify = NULL; 2669 hp->dmai_error.err_ena = 0; 2670 hp->dmai_error.err_status = DDI_FM_OK; 2671 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2672 hp->dmai_error.err_ontrap = NULL; 2673 } 2674 2675 2676 /* 2677 * rootnex_valid_alloc_parms() 2678 * Called in ddi_dma_alloc_handle path to validate its parameters. 2679 */ 2680 static int 2681 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2682 { 2683 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2684 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2685 (attr->dma_attr_granular > MMU_PAGESIZE) || 2686 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2687 return (DDI_DMA_BADATTR); 2688 } 2689 2690 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2691 return (DDI_DMA_BADATTR); 2692 } 2693 2694 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2695 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2696 attr->dma_attr_sgllen <= 0) { 2697 return (DDI_DMA_BADATTR); 2698 } 2699 2700 /* We should be able to DMA into every byte offset in a page */ 2701 if (maxsegmentsize < MMU_PAGESIZE) { 2702 return (DDI_DMA_BADATTR); 2703 } 2704 2705 /* if we're bouncing on seg, seg must be <= addr_hi */ 2706 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2707 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2708 return (DDI_DMA_BADATTR); 2709 } 2710 return (DDI_SUCCESS); 2711 } 2712 2713 /* 2714 * rootnex_valid_bind_parms() 2715 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2716 */ 2717 /* ARGSUSED */ 2718 static int 2719 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2720 { 2721 #if !defined(__amd64) 2722 /* 2723 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2724 * we can track the offset for the obsoleted interfaces. 2725 */ 2726 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2727 return (DDI_DMA_TOOBIG); 2728 } 2729 #endif 2730 2731 return (DDI_SUCCESS); 2732 } 2733 2734 2735 /* 2736 * rootnex_need_bounce_seg() 2737 * check to see if the buffer lives on both side of the seg. 2738 */ 2739 static boolean_t 2740 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2741 { 2742 ddi_dma_atyp_t buftype; 2743 rootnex_addr_t raddr; 2744 boolean_t lower_addr; 2745 boolean_t upper_addr; 2746 uint64_t offset; 2747 page_t **pplist; 2748 uint64_t paddr; 2749 uint32_t psize; 2750 uint32_t size; 2751 caddr_t vaddr; 2752 uint_t pcnt; 2753 page_t *pp; 2754 2755 2756 /* shortcuts */ 2757 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2758 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2759 buftype = dmar_object->dmao_type; 2760 size = dmar_object->dmao_size; 2761 2762 lower_addr = B_FALSE; 2763 upper_addr = B_FALSE; 2764 pcnt = 0; 2765 2766 /* 2767 * Process the first page to handle the initial offset of the buffer. 2768 * We'll use the base address we get later when we loop through all 2769 * the pages. 2770 */ 2771 if (buftype == DMA_OTYP_PAGES) { 2772 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2773 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2774 MMU_PAGEOFFSET; 2775 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2776 psize = MIN(size, (MMU_PAGESIZE - offset)); 2777 pp = pp->p_next; 2778 sglinfo->si_asp = NULL; 2779 } else if (pplist != NULL) { 2780 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2781 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2782 if (sglinfo->si_asp == NULL) { 2783 sglinfo->si_asp = &kas; 2784 } 2785 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2786 paddr += offset; 2787 psize = MIN(size, (MMU_PAGESIZE - offset)); 2788 pcnt++; 2789 } else { 2790 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2791 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2792 if (sglinfo->si_asp == NULL) { 2793 sglinfo->si_asp = &kas; 2794 } 2795 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2796 paddr += offset; 2797 psize = MIN(size, (MMU_PAGESIZE - offset)); 2798 vaddr += psize; 2799 } 2800 2801 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2802 2803 if ((raddr + psize) > sglinfo->si_segmask) { 2804 upper_addr = B_TRUE; 2805 } else { 2806 lower_addr = B_TRUE; 2807 } 2808 size -= psize; 2809 2810 /* 2811 * Walk through the rest of the pages in the buffer. Track to see 2812 * if we have pages on both sides of the segment boundary. 2813 */ 2814 while (size > 0) { 2815 /* partial or full page */ 2816 psize = MIN(size, MMU_PAGESIZE); 2817 2818 if (buftype == DMA_OTYP_PAGES) { 2819 /* get the paddr from the page_t */ 2820 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2821 paddr = pfn_to_pa(pp->p_pagenum); 2822 pp = pp->p_next; 2823 } else if (pplist != NULL) { 2824 /* index into the array of page_t's to get the paddr */ 2825 ASSERT(!PP_ISFREE(pplist[pcnt])); 2826 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2827 pcnt++; 2828 } else { 2829 /* call into the VM to get the paddr */ 2830 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2831 vaddr)); 2832 vaddr += psize; 2833 } 2834 2835 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2836 2837 if ((raddr + psize) > sglinfo->si_segmask) { 2838 upper_addr = B_TRUE; 2839 } else { 2840 lower_addr = B_TRUE; 2841 } 2842 /* 2843 * if the buffer lives both above and below the segment 2844 * boundary, or the current page is the page immediately 2845 * after the segment, we will use a copy/bounce buffer for 2846 * all pages > seg. 2847 */ 2848 if ((lower_addr && upper_addr) || 2849 (raddr == (sglinfo->si_segmask + 1))) { 2850 return (B_TRUE); 2851 } 2852 2853 size -= psize; 2854 } 2855 2856 return (B_FALSE); 2857 } 2858 2859 /* 2860 * rootnex_get_sgl() 2861 * Called in bind fastpath to get the sgl. Most of this will be replaced 2862 * with a call to the vm layer when vm2.0 comes around... 2863 */ 2864 static void 2865 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2866 rootnex_sglinfo_t *sglinfo) 2867 { 2868 ddi_dma_atyp_t buftype; 2869 rootnex_addr_t raddr; 2870 uint64_t last_page; 2871 uint64_t offset; 2872 uint64_t addrhi; 2873 uint64_t addrlo; 2874 uint64_t maxseg; 2875 page_t **pplist; 2876 uint64_t paddr; 2877 uint32_t psize; 2878 uint32_t size; 2879 caddr_t vaddr; 2880 uint_t pcnt; 2881 page_t *pp; 2882 uint_t cnt; 2883 2884 2885 /* shortcuts */ 2886 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2887 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2888 maxseg = sglinfo->si_max_cookie_size; 2889 buftype = dmar_object->dmao_type; 2890 addrhi = sglinfo->si_max_addr; 2891 addrlo = sglinfo->si_min_addr; 2892 size = dmar_object->dmao_size; 2893 2894 pcnt = 0; 2895 cnt = 0; 2896 2897 2898 /* 2899 * check to see if we need to use the copy buffer for pages over 2900 * the segment attr. 2901 */ 2902 sglinfo->si_bounce_on_seg = B_FALSE; 2903 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2904 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2905 dmar_object, sglinfo); 2906 } 2907 2908 /* 2909 * if we were passed down a linked list of pages, i.e. pointer to 2910 * page_t, use this to get our physical address and buf offset. 2911 */ 2912 if (buftype == DMA_OTYP_PAGES) { 2913 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2914 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2915 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2916 MMU_PAGEOFFSET; 2917 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2918 psize = MIN(size, (MMU_PAGESIZE - offset)); 2919 pp = pp->p_next; 2920 sglinfo->si_asp = NULL; 2921 2922 /* 2923 * We weren't passed down a linked list of pages, but if we were passed 2924 * down an array of pages, use this to get our physical address and buf 2925 * offset. 2926 */ 2927 } else if (pplist != NULL) { 2928 ASSERT((buftype == DMA_OTYP_VADDR) || 2929 (buftype == DMA_OTYP_BUFVADDR)); 2930 2931 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2932 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2933 if (sglinfo->si_asp == NULL) { 2934 sglinfo->si_asp = &kas; 2935 } 2936 2937 ASSERT(!PP_ISFREE(pplist[pcnt])); 2938 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2939 paddr += offset; 2940 psize = MIN(size, (MMU_PAGESIZE - offset)); 2941 pcnt++; 2942 2943 /* 2944 * All we have is a virtual address, we'll need to call into the VM 2945 * to get the physical address. 2946 */ 2947 } else { 2948 ASSERT((buftype == DMA_OTYP_VADDR) || 2949 (buftype == DMA_OTYP_BUFVADDR)); 2950 2951 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2952 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2953 if (sglinfo->si_asp == NULL) { 2954 sglinfo->si_asp = &kas; 2955 } 2956 2957 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2958 paddr += offset; 2959 psize = MIN(size, (MMU_PAGESIZE - offset)); 2960 vaddr += psize; 2961 } 2962 2963 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2964 2965 /* 2966 * Setup the first cookie with the physical address of the page and the 2967 * size of the page (which takes into account the initial offset into 2968 * the page. 2969 */ 2970 sgl[cnt].dmac_laddress = raddr; 2971 sgl[cnt].dmac_size = psize; 2972 sgl[cnt].dmac_type = 0; 2973 2974 /* 2975 * Save away the buffer offset into the page. We'll need this later in 2976 * the copy buffer code to help figure out the page index within the 2977 * buffer and the offset into the current page. 2978 */ 2979 sglinfo->si_buf_offset = offset; 2980 2981 /* 2982 * If we are using the copy buffer for anything over the segment 2983 * boundary, and this page is over the segment boundary. 2984 * OR 2985 * if the DMA engine can't reach the physical address. 2986 */ 2987 if (((sglinfo->si_bounce_on_seg) && 2988 ((raddr + psize) > sglinfo->si_segmask)) || 2989 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2990 /* 2991 * Increase how much copy buffer we use. We always increase by 2992 * pagesize so we don't have to worry about converting offsets. 2993 * Set a flag in the cookies dmac_type to indicate that it uses 2994 * the copy buffer. If this isn't the last cookie, go to the 2995 * next cookie (since we separate each page which uses the copy 2996 * buffer in case the copy buffer is not physically contiguous. 2997 */ 2998 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2999 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3000 if ((cnt + 1) < sglinfo->si_max_pages) { 3001 cnt++; 3002 sgl[cnt].dmac_laddress = 0; 3003 sgl[cnt].dmac_size = 0; 3004 sgl[cnt].dmac_type = 0; 3005 } 3006 } 3007 3008 /* 3009 * save this page's physical address so we can figure out if the next 3010 * page is physically contiguous. Keep decrementing size until we are 3011 * done with the buffer. 3012 */ 3013 last_page = raddr & MMU_PAGEMASK; 3014 size -= psize; 3015 3016 while (size > 0) { 3017 /* Get the size for this page (i.e. partial or full page) */ 3018 psize = MIN(size, MMU_PAGESIZE); 3019 3020 if (buftype == DMA_OTYP_PAGES) { 3021 /* get the paddr from the page_t */ 3022 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3023 paddr = pfn_to_pa(pp->p_pagenum); 3024 pp = pp->p_next; 3025 } else if (pplist != NULL) { 3026 /* index into the array of page_t's to get the paddr */ 3027 ASSERT(!PP_ISFREE(pplist[pcnt])); 3028 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3029 pcnt++; 3030 } else { 3031 /* call into the VM to get the paddr */ 3032 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3033 vaddr)); 3034 vaddr += psize; 3035 } 3036 3037 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3038 3039 /* 3040 * If we are using the copy buffer for anything over the 3041 * segment boundary, and this page is over the segment 3042 * boundary. 3043 * OR 3044 * if the DMA engine can't reach the physical address. 3045 */ 3046 if (((sglinfo->si_bounce_on_seg) && 3047 ((raddr + psize) > sglinfo->si_segmask)) || 3048 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3049 3050 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3051 3052 /* 3053 * if there is something in the current cookie, go to 3054 * the next one. We only want one page in a cookie which 3055 * uses the copybuf since the copybuf doesn't have to 3056 * be physically contiguous. 3057 */ 3058 if (sgl[cnt].dmac_size != 0) { 3059 cnt++; 3060 } 3061 sgl[cnt].dmac_laddress = raddr; 3062 sgl[cnt].dmac_size = psize; 3063 #if defined(__amd64) 3064 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3065 #else 3066 /* 3067 * save the buf offset for 32-bit kernel. used in the 3068 * obsoleted interfaces. 3069 */ 3070 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3071 (dmar_object->dmao_size - size); 3072 #endif 3073 /* if this isn't the last cookie, go to the next one */ 3074 if ((cnt + 1) < sglinfo->si_max_pages) { 3075 cnt++; 3076 sgl[cnt].dmac_laddress = 0; 3077 sgl[cnt].dmac_size = 0; 3078 sgl[cnt].dmac_type = 0; 3079 } 3080 3081 /* 3082 * this page didn't need the copy buffer, if it's not physically 3083 * contiguous, or it would put us over a segment boundary, or it 3084 * puts us over the max cookie size, or the current sgl doesn't 3085 * have anything in it. 3086 */ 3087 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3088 !(raddr & sglinfo->si_segmask) || 3089 ((sgl[cnt].dmac_size + psize) > maxseg) || 3090 (sgl[cnt].dmac_size == 0)) { 3091 /* 3092 * if we're not already in a new cookie, go to the next 3093 * cookie. 3094 */ 3095 if (sgl[cnt].dmac_size != 0) { 3096 cnt++; 3097 } 3098 3099 /* save the cookie information */ 3100 sgl[cnt].dmac_laddress = raddr; 3101 sgl[cnt].dmac_size = psize; 3102 #if defined(__amd64) 3103 sgl[cnt].dmac_type = 0; 3104 #else 3105 /* 3106 * save the buf offset for 32-bit kernel. used in the 3107 * obsoleted interfaces. 3108 */ 3109 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3110 #endif 3111 3112 /* 3113 * this page didn't need the copy buffer, it is physically 3114 * contiguous with the last page, and it's <= the max cookie 3115 * size. 3116 */ 3117 } else { 3118 sgl[cnt].dmac_size += psize; 3119 3120 /* 3121 * if this exactly == the maximum cookie size, and 3122 * it isn't the last cookie, go to the next cookie. 3123 */ 3124 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3125 ((cnt + 1) < sglinfo->si_max_pages)) { 3126 cnt++; 3127 sgl[cnt].dmac_laddress = 0; 3128 sgl[cnt].dmac_size = 0; 3129 sgl[cnt].dmac_type = 0; 3130 } 3131 } 3132 3133 /* 3134 * save this page's physical address so we can figure out if the 3135 * next page is physically contiguous. Keep decrementing size 3136 * until we are done with the buffer. 3137 */ 3138 last_page = raddr; 3139 size -= psize; 3140 } 3141 3142 /* we're done, save away how many cookies the sgl has */ 3143 if (sgl[cnt].dmac_size == 0) { 3144 ASSERT(cnt < sglinfo->si_max_pages); 3145 sglinfo->si_sgl_size = cnt; 3146 } else { 3147 sglinfo->si_sgl_size = cnt + 1; 3148 } 3149 } 3150 3151 static void 3152 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3153 rootnex_sglinfo_t *sglinfo) 3154 { 3155 uint64_t offset; 3156 uint64_t maxseg; 3157 uint64_t dvaddr; 3158 struct dvmaseg *dvs; 3159 uint64_t paddr; 3160 uint32_t psize, ssize; 3161 uint32_t size; 3162 uint_t cnt; 3163 int physcontig; 3164 3165 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3166 3167 /* shortcuts */ 3168 maxseg = sglinfo->si_max_cookie_size; 3169 size = dmar_object->dmao_size; 3170 3171 cnt = 0; 3172 sglinfo->si_bounce_on_seg = B_FALSE; 3173 3174 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3175 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3176 ssize = dvs->dvs_len; 3177 paddr = dvs->dvs_start; 3178 paddr += offset; 3179 psize = MIN(ssize, (maxseg - offset)); 3180 dvaddr = paddr + psize; 3181 ssize -= psize; 3182 3183 sgl[cnt].dmac_laddress = paddr; 3184 sgl[cnt].dmac_size = psize; 3185 sgl[cnt].dmac_type = 0; 3186 3187 size -= psize; 3188 while (size > 0) { 3189 if (ssize == 0) { 3190 dvs++; 3191 ssize = dvs->dvs_len; 3192 dvaddr = dvs->dvs_start; 3193 physcontig = 0; 3194 } else 3195 physcontig = 1; 3196 3197 paddr = dvaddr; 3198 psize = MIN(ssize, maxseg); 3199 dvaddr += psize; 3200 ssize -= psize; 3201 3202 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3203 ((sgl[cnt].dmac_size + psize) > maxseg) || 3204 (sgl[cnt].dmac_size == 0)) { 3205 /* 3206 * if we're not already in a new cookie, go to the next 3207 * cookie. 3208 */ 3209 if (sgl[cnt].dmac_size != 0) { 3210 cnt++; 3211 } 3212 3213 /* save the cookie information */ 3214 sgl[cnt].dmac_laddress = paddr; 3215 sgl[cnt].dmac_size = psize; 3216 sgl[cnt].dmac_type = 0; 3217 } else { 3218 sgl[cnt].dmac_size += psize; 3219 3220 /* 3221 * if this exactly == the maximum cookie size, and 3222 * it isn't the last cookie, go to the next cookie. 3223 */ 3224 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3225 ((cnt + 1) < sglinfo->si_max_pages)) { 3226 cnt++; 3227 sgl[cnt].dmac_laddress = 0; 3228 sgl[cnt].dmac_size = 0; 3229 sgl[cnt].dmac_type = 0; 3230 } 3231 } 3232 size -= psize; 3233 } 3234 3235 /* we're done, save away how many cookies the sgl has */ 3236 if (sgl[cnt].dmac_size == 0) { 3237 sglinfo->si_sgl_size = cnt; 3238 } else { 3239 sglinfo->si_sgl_size = cnt + 1; 3240 } 3241 } 3242 3243 /* 3244 * rootnex_bind_slowpath() 3245 * Call in the bind path if the calling driver can't use the sgl without 3246 * modifying it. We either need to use the copy buffer and/or we will end up 3247 * with a partial bind. 3248 */ 3249 static int 3250 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3251 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3252 { 3253 rootnex_sglinfo_t *sinfo; 3254 rootnex_window_t *window; 3255 ddi_dma_cookie_t *cookie; 3256 size_t copybuf_used; 3257 size_t dmac_size; 3258 boolean_t partial; 3259 off_t cur_offset; 3260 page_t *cur_pp; 3261 major_t mnum; 3262 int e; 3263 int i; 3264 3265 3266 sinfo = &dma->dp_sglinfo; 3267 copybuf_used = 0; 3268 partial = B_FALSE; 3269 3270 /* 3271 * If we're using the copybuf, set the copybuf state in dma struct. 3272 * Needs to be first since it sets the copy buffer size. 3273 */ 3274 if (sinfo->si_copybuf_req != 0) { 3275 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3276 if (e != DDI_SUCCESS) { 3277 return (e); 3278 } 3279 } else { 3280 dma->dp_copybuf_size = 0; 3281 } 3282 3283 /* 3284 * Figure out if we need to do a partial mapping. If so, figure out 3285 * if we need to trim the buffers when we munge the sgl. 3286 */ 3287 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3288 (dmao->dmao_size > dma->dp_maxxfer) || 3289 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3290 dma->dp_partial_required = B_TRUE; 3291 if (attr->dma_attr_granular != 1) { 3292 dma->dp_trim_required = B_TRUE; 3293 } 3294 } else { 3295 dma->dp_partial_required = B_FALSE; 3296 dma->dp_trim_required = B_FALSE; 3297 } 3298 3299 /* If we need to do a partial bind, make sure the driver supports it */ 3300 if (dma->dp_partial_required && 3301 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3302 3303 mnum = ddi_driver_major(dma->dp_dip); 3304 /* 3305 * patchable which allows us to print one warning per major 3306 * number. 3307 */ 3308 if ((rootnex_bind_warn) && 3309 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3310 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3311 cmn_err(CE_WARN, "!%s: coding error detected, the " 3312 "driver is using ddi_dma_attr(9S) incorrectly. " 3313 "There is a small risk of data corruption in " 3314 "particular with large I/Os. The driver should be " 3315 "replaced with a corrected version for proper " 3316 "system operation. To disable this warning, add " 3317 "'set rootnex:rootnex_bind_warn=0' to " 3318 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3319 } 3320 return (DDI_DMA_TOOBIG); 3321 } 3322 3323 /* 3324 * we might need multiple windows, setup state to handle them. In this 3325 * code path, we will have at least one window. 3326 */ 3327 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3328 if (e != DDI_SUCCESS) { 3329 rootnex_teardown_copybuf(dma); 3330 return (e); 3331 } 3332 3333 window = &dma->dp_window[0]; 3334 cookie = &dma->dp_cookies[0]; 3335 cur_offset = 0; 3336 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3337 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3338 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3339 } 3340 3341 /* loop though all the cookies we got back from get_sgl() */ 3342 for (i = 0; i < sinfo->si_sgl_size; i++) { 3343 /* 3344 * If we're using the copy buffer, check this cookie and setup 3345 * its associated copy buffer state. If this cookie uses the 3346 * copy buffer, make sure we sync this window during dma_sync. 3347 */ 3348 if (dma->dp_copybuf_size > 0) { 3349 rootnex_setup_cookie(dmao, dma, cookie, 3350 cur_offset, ©buf_used, &cur_pp); 3351 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3352 window->wd_dosync = B_TRUE; 3353 } 3354 } 3355 3356 /* 3357 * save away the cookie size, since it could be modified in 3358 * the windowing code. 3359 */ 3360 dmac_size = cookie->dmac_size; 3361 3362 /* if we went over max copybuf size */ 3363 if (dma->dp_copybuf_size && 3364 (copybuf_used > dma->dp_copybuf_size)) { 3365 partial = B_TRUE; 3366 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3367 cookie, cur_offset, ©buf_used); 3368 if (e != DDI_SUCCESS) { 3369 rootnex_teardown_copybuf(dma); 3370 rootnex_teardown_windows(dma); 3371 return (e); 3372 } 3373 3374 /* 3375 * if the coookie uses the copy buffer, make sure the 3376 * new window we just moved to is set to sync. 3377 */ 3378 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3379 window->wd_dosync = B_TRUE; 3380 } 3381 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3382 dma->dp_dip); 3383 3384 /* if the cookie cnt == max sgllen, move to the next window */ 3385 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 3386 partial = B_TRUE; 3387 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3388 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3389 cookie, attr, cur_offset); 3390 if (e != DDI_SUCCESS) { 3391 rootnex_teardown_copybuf(dma); 3392 rootnex_teardown_windows(dma); 3393 return (e); 3394 } 3395 3396 /* 3397 * if the coookie uses the copy buffer, make sure the 3398 * new window we just moved to is set to sync. 3399 */ 3400 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3401 window->wd_dosync = B_TRUE; 3402 } 3403 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3404 dma->dp_dip); 3405 3406 /* else if we will be over maxxfer */ 3407 } else if ((window->wd_size + dmac_size) > 3408 dma->dp_maxxfer) { 3409 partial = B_TRUE; 3410 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3411 cookie); 3412 if (e != DDI_SUCCESS) { 3413 rootnex_teardown_copybuf(dma); 3414 rootnex_teardown_windows(dma); 3415 return (e); 3416 } 3417 3418 /* 3419 * if the coookie uses the copy buffer, make sure the 3420 * new window we just moved to is set to sync. 3421 */ 3422 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3423 window->wd_dosync = B_TRUE; 3424 } 3425 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3426 dma->dp_dip); 3427 3428 /* else this cookie fits in the current window */ 3429 } else { 3430 window->wd_cookie_cnt++; 3431 window->wd_size += dmac_size; 3432 } 3433 3434 /* track our offset into the buffer, go to the next cookie */ 3435 ASSERT(dmac_size <= dmao->dmao_size); 3436 ASSERT(cookie->dmac_size <= dmac_size); 3437 cur_offset += dmac_size; 3438 cookie++; 3439 } 3440 3441 /* if we ended up with a zero sized window in the end, clean it up */ 3442 if (window->wd_size == 0) { 3443 hp->dmai_nwin--; 3444 window--; 3445 } 3446 3447 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3448 3449 if (!partial) { 3450 return (DDI_DMA_MAPPED); 3451 } 3452 3453 ASSERT(dma->dp_partial_required); 3454 return (DDI_DMA_PARTIAL_MAP); 3455 } 3456 3457 /* 3458 * rootnex_setup_copybuf() 3459 * Called in bind slowpath. Figures out if we're going to use the copy 3460 * buffer, and if we do, sets up the basic state to handle it. 3461 */ 3462 static int 3463 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3464 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3465 { 3466 rootnex_sglinfo_t *sinfo; 3467 ddi_dma_attr_t lattr; 3468 size_t max_copybuf; 3469 int cansleep; 3470 int e; 3471 #if !defined(__amd64) 3472 int vmflag; 3473 #endif 3474 3475 ASSERT(!dma->dp_dvma_used); 3476 3477 sinfo = &dma->dp_sglinfo; 3478 3479 /* read this first so it's consistent through the routine */ 3480 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3481 3482 /* We need to call into the rootnex on ddi_dma_sync() */ 3483 hp->dmai_rflags &= ~DMP_NOSYNC; 3484 3485 /* make sure the copybuf size <= the max size */ 3486 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3487 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3488 3489 #if !defined(__amd64) 3490 /* 3491 * if we don't have kva space to copy to/from, allocate the KVA space 3492 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3493 * the 64-bit kernel. 3494 */ 3495 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3496 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3497 3498 /* convert the sleep flags */ 3499 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3500 vmflag = VM_SLEEP; 3501 } else { 3502 vmflag = VM_NOSLEEP; 3503 } 3504 3505 /* allocate Kernel VA space that we can bcopy to/from */ 3506 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3507 vmflag); 3508 if (dma->dp_kva == NULL) { 3509 return (DDI_DMA_NORESOURCES); 3510 } 3511 } 3512 #endif 3513 3514 /* convert the sleep flags */ 3515 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3516 cansleep = 1; 3517 } else { 3518 cansleep = 0; 3519 } 3520 3521 /* 3522 * Allocate the actual copy buffer. This needs to fit within the DMA 3523 * engine limits, so we can't use kmem_alloc... We don't need 3524 * contiguous memory (sgllen) since we will be forcing windows on 3525 * sgllen anyway. 3526 */ 3527 lattr = *attr; 3528 lattr.dma_attr_align = MMU_PAGESIZE; 3529 /* 3530 * this should be < 0 to indicate no limit, but due to a bug in 3531 * the rootnex, we'll set it to the maximum positive int. 3532 */ 3533 lattr.dma_attr_sgllen = 0x7fffffff; 3534 /* 3535 * if we're using the copy buffer because of seg, use that for our 3536 * upper address limit. 3537 */ 3538 if (sinfo->si_bounce_on_seg) { 3539 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3540 } 3541 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3542 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3543 if (e != DDI_SUCCESS) { 3544 #if !defined(__amd64) 3545 if (dma->dp_kva != NULL) { 3546 vmem_free(heap_arena, dma->dp_kva, 3547 dma->dp_copybuf_size); 3548 } 3549 #endif 3550 return (DDI_DMA_NORESOURCES); 3551 } 3552 3553 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3554 size_t, dma->dp_copybuf_size); 3555 3556 return (DDI_SUCCESS); 3557 } 3558 3559 3560 /* 3561 * rootnex_setup_windows() 3562 * Called in bind slowpath to setup the window state. We always have windows 3563 * in the slowpath. Even if the window count = 1. 3564 */ 3565 static int 3566 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3567 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3568 { 3569 rootnex_window_t *windowp; 3570 rootnex_sglinfo_t *sinfo; 3571 size_t copy_state_size; 3572 size_t win_state_size; 3573 size_t state_available; 3574 size_t space_needed; 3575 uint_t copybuf_win; 3576 uint_t maxxfer_win; 3577 size_t space_used; 3578 uint_t sglwin; 3579 3580 3581 sinfo = &dma->dp_sglinfo; 3582 3583 dma->dp_current_win = 0; 3584 hp->dmai_nwin = 0; 3585 3586 /* If we don't need to do a partial, we only have one window */ 3587 if (!dma->dp_partial_required) { 3588 dma->dp_max_win = 1; 3589 3590 /* 3591 * we need multiple windows, need to figure out the worse case number 3592 * of windows. 3593 */ 3594 } else { 3595 /* 3596 * if we need windows because we need more copy buffer that 3597 * we allow, the worse case number of windows we could need 3598 * here would be (copybuf space required / copybuf space that 3599 * we have) plus one for remainder, and plus 2 to handle the 3600 * extra pages on the trim for the first and last pages of the 3601 * buffer (a page is the minimum window size so under the right 3602 * attr settings, you could have a window for each page). 3603 * The last page will only be hit here if the size is not a 3604 * multiple of the granularity (which theoretically shouldn't 3605 * be the case but never has been enforced, so we could have 3606 * broken things without it). 3607 */ 3608 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3609 ASSERT(dma->dp_copybuf_size > 0); 3610 copybuf_win = (sinfo->si_copybuf_req / 3611 dma->dp_copybuf_size) + 1 + 2; 3612 } else { 3613 copybuf_win = 0; 3614 } 3615 3616 /* 3617 * if we need windows because we have more cookies than the H/W 3618 * can handle, the number of windows we would need here would 3619 * be (cookie count / cookies count H/W supports minus 1[for 3620 * trim]) plus one for remainder. 3621 */ 3622 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3623 sglwin = (sinfo->si_sgl_size / 3624 (attr->dma_attr_sgllen - 1)) + 1; 3625 } else { 3626 sglwin = 0; 3627 } 3628 3629 /* 3630 * if we need windows because we're binding more memory than the 3631 * H/W can transfer at once, the number of windows we would need 3632 * here would be (xfer count / max xfer H/W supports) plus one 3633 * for remainder, and plus 2 to handle the extra pages on the 3634 * trim (see above comment about trim) 3635 */ 3636 if (dmao->dmao_size > dma->dp_maxxfer) { 3637 maxxfer_win = (dmao->dmao_size / 3638 dma->dp_maxxfer) + 1 + 2; 3639 } else { 3640 maxxfer_win = 0; 3641 } 3642 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3643 ASSERT(dma->dp_max_win > 0); 3644 } 3645 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3646 3647 /* 3648 * Get space for window and potential copy buffer state. Before we 3649 * go and allocate memory, see if we can get away with using what's 3650 * left in the pre-allocted state or the dynamically allocated sgl. 3651 */ 3652 space_used = (uintptr_t)(sinfo->si_sgl_size * 3653 sizeof (ddi_dma_cookie_t)); 3654 3655 /* if we dynamically allocated space for the cookies */ 3656 if (dma->dp_need_to_free_cookie) { 3657 /* if we have more space in the pre-allocted buffer, use it */ 3658 ASSERT(space_used <= dma->dp_cookie_size); 3659 if ((dma->dp_cookie_size - space_used) <= 3660 rootnex_state->r_prealloc_size) { 3661 state_available = rootnex_state->r_prealloc_size; 3662 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3663 3664 /* 3665 * else, we have more free space in the dynamically allocated 3666 * buffer, i.e. the buffer wasn't worse case fragmented so we 3667 * didn't need a lot of cookies. 3668 */ 3669 } else { 3670 state_available = dma->dp_cookie_size - space_used; 3671 windowp = (rootnex_window_t *) 3672 &dma->dp_cookies[sinfo->si_sgl_size]; 3673 } 3674 3675 /* we used the pre-alloced buffer */ 3676 } else { 3677 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3678 state_available = rootnex_state->r_prealloc_size - space_used; 3679 windowp = (rootnex_window_t *) 3680 &dma->dp_cookies[sinfo->si_sgl_size]; 3681 } 3682 3683 /* 3684 * figure out how much state we need to track the copy buffer. Add an 3685 * addition 8 bytes for pointer alignemnt later. 3686 */ 3687 if (dma->dp_copybuf_size > 0) { 3688 copy_state_size = sinfo->si_max_pages * 3689 sizeof (rootnex_pgmap_t); 3690 } else { 3691 copy_state_size = 0; 3692 } 3693 /* add an additional 8 bytes for pointer alignment */ 3694 space_needed = win_state_size + copy_state_size + 0x8; 3695 3696 /* if we have enough space already, use it */ 3697 if (state_available >= space_needed) { 3698 dma->dp_window = windowp; 3699 dma->dp_need_to_free_window = B_FALSE; 3700 3701 /* not enough space, need to allocate more. */ 3702 } else { 3703 dma->dp_window = kmem_alloc(space_needed, kmflag); 3704 if (dma->dp_window == NULL) { 3705 return (DDI_DMA_NORESOURCES); 3706 } 3707 dma->dp_need_to_free_window = B_TRUE; 3708 dma->dp_window_size = space_needed; 3709 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3710 dma->dp_dip, size_t, space_needed); 3711 } 3712 3713 /* 3714 * we allocate copy buffer state and window state at the same time. 3715 * setup our copy buffer state pointers. Make sure it's aligned. 3716 */ 3717 if (dma->dp_copybuf_size > 0) { 3718 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3719 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3720 3721 #if !defined(__amd64) 3722 /* 3723 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3724 * false/NULL. Should be quicker to bzero vs loop and set. 3725 */ 3726 bzero(dma->dp_pgmap, copy_state_size); 3727 #endif 3728 } else { 3729 dma->dp_pgmap = NULL; 3730 } 3731 3732 return (DDI_SUCCESS); 3733 } 3734 3735 3736 /* 3737 * rootnex_teardown_copybuf() 3738 * cleans up after rootnex_setup_copybuf() 3739 */ 3740 static void 3741 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3742 { 3743 #if !defined(__amd64) 3744 int i; 3745 3746 /* 3747 * if we allocated kernel heap VMEM space, go through all the pages and 3748 * map out any of the ones that we're mapped into the kernel heap VMEM 3749 * arena. Then free the VMEM space. 3750 */ 3751 if (dma->dp_kva != NULL) { 3752 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3753 if (dma->dp_pgmap[i].pm_mapped) { 3754 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3755 MMU_PAGESIZE, HAT_UNLOAD); 3756 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3757 } 3758 } 3759 3760 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3761 } 3762 3763 #endif 3764 3765 /* if we allocated a copy buffer, free it */ 3766 if (dma->dp_cbaddr != NULL) { 3767 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3768 } 3769 } 3770 3771 3772 /* 3773 * rootnex_teardown_windows() 3774 * cleans up after rootnex_setup_windows() 3775 */ 3776 static void 3777 rootnex_teardown_windows(rootnex_dma_t *dma) 3778 { 3779 /* 3780 * if we had to allocate window state on the last bind (because we 3781 * didn't have enough pre-allocated space in the handle), free it. 3782 */ 3783 if (dma->dp_need_to_free_window) { 3784 kmem_free(dma->dp_window, dma->dp_window_size); 3785 } 3786 } 3787 3788 3789 /* 3790 * rootnex_init_win() 3791 * Called in bind slow path during creation of a new window. Initializes 3792 * window state to default values. 3793 */ 3794 /*ARGSUSED*/ 3795 static void 3796 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3797 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3798 { 3799 hp->dmai_nwin++; 3800 window->wd_dosync = B_FALSE; 3801 window->wd_offset = cur_offset; 3802 window->wd_size = 0; 3803 window->wd_first_cookie = cookie; 3804 window->wd_cookie_cnt = 0; 3805 window->wd_trim.tr_trim_first = B_FALSE; 3806 window->wd_trim.tr_trim_last = B_FALSE; 3807 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3808 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3809 #if !defined(__amd64) 3810 window->wd_remap_copybuf = dma->dp_cb_remaping; 3811 #endif 3812 } 3813 3814 3815 /* 3816 * rootnex_setup_cookie() 3817 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3818 * the sgl uses the copy buffer, we need to go through each cookie, figure 3819 * out if it uses the copy buffer, and if it does, save away everything we'll 3820 * need during sync. 3821 */ 3822 static void 3823 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3824 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3825 page_t **cur_pp) 3826 { 3827 boolean_t copybuf_sz_power_2; 3828 rootnex_sglinfo_t *sinfo; 3829 paddr_t paddr; 3830 uint_t pidx; 3831 uint_t pcnt; 3832 off_t poff; 3833 #if defined(__amd64) 3834 pfn_t pfn; 3835 #else 3836 page_t **pplist; 3837 #endif 3838 3839 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3840 3841 sinfo = &dma->dp_sglinfo; 3842 3843 /* 3844 * Calculate the page index relative to the start of the buffer. The 3845 * index to the current page for our buffer is the offset into the 3846 * first page of the buffer plus our current offset into the buffer 3847 * itself, shifted of course... 3848 */ 3849 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3850 ASSERT(pidx < sinfo->si_max_pages); 3851 3852 /* if this cookie uses the copy buffer */ 3853 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3854 /* 3855 * NOTE: we know that since this cookie uses the copy buffer, it 3856 * is <= MMU_PAGESIZE. 3857 */ 3858 3859 /* 3860 * get the offset into the page. For the 64-bit kernel, get the 3861 * pfn which we'll use with seg kpm. 3862 */ 3863 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3864 #if defined(__amd64) 3865 /* mfn_to_pfn() is a NOP on i86pc */ 3866 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3867 #endif /* __amd64 */ 3868 3869 /* figure out if the copybuf size is a power of 2 */ 3870 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3871 copybuf_sz_power_2 = B_FALSE; 3872 } else { 3873 copybuf_sz_power_2 = B_TRUE; 3874 } 3875 3876 /* This page uses the copy buffer */ 3877 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3878 3879 /* 3880 * save the copy buffer KVA that we'll use with this page. 3881 * if we still fit within the copybuf, it's a simple add. 3882 * otherwise, we need to wrap over using & or % accordingly. 3883 */ 3884 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3885 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3886 *copybuf_used; 3887 } else { 3888 if (copybuf_sz_power_2) { 3889 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3890 (uintptr_t)dma->dp_cbaddr + 3891 (*copybuf_used & 3892 (dma->dp_copybuf_size - 1))); 3893 } else { 3894 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3895 (uintptr_t)dma->dp_cbaddr + 3896 (*copybuf_used % dma->dp_copybuf_size)); 3897 } 3898 } 3899 3900 /* 3901 * over write the cookie physical address with the address of 3902 * the physical address of the copy buffer page that we will 3903 * use. 3904 */ 3905 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3906 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3907 3908 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3909 3910 /* if we have a kernel VA, it's easy, just save that address */ 3911 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3912 (sinfo->si_asp == &kas)) { 3913 /* 3914 * save away the page aligned virtual address of the 3915 * driver buffer. Offsets are handled in the sync code. 3916 */ 3917 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3918 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3919 & MMU_PAGEMASK); 3920 #if !defined(__amd64) 3921 /* 3922 * we didn't need to, and will never need to map this 3923 * page. 3924 */ 3925 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3926 #endif 3927 3928 /* we don't have a kernel VA. We need one for the bcopy. */ 3929 } else { 3930 #if defined(__amd64) 3931 /* 3932 * for the 64-bit kernel, it's easy. We use seg kpm to 3933 * get a Kernel VA for the corresponding pfn. 3934 */ 3935 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3936 #else 3937 /* 3938 * for the 32-bit kernel, this is a pain. First we'll 3939 * save away the page_t or user VA for this page. This 3940 * is needed in rootnex_dma_win() when we switch to a 3941 * new window which requires us to re-map the copy 3942 * buffer. 3943 */ 3944 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3945 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3946 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3947 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3948 } else if (pplist != NULL) { 3949 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3950 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3951 } else { 3952 dma->dp_pgmap[pidx].pm_pp = NULL; 3953 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3954 (((uintptr_t) 3955 dmar_object->dmao_obj.virt_obj.v_addr + 3956 cur_offset) & MMU_PAGEMASK); 3957 } 3958 3959 /* 3960 * save away the page aligned virtual address which was 3961 * allocated from the kernel heap arena (taking into 3962 * account if we need more copy buffer than we alloced 3963 * and use multiple windows to handle this, i.e. &,%). 3964 * NOTE: there isn't and physical memory backing up this 3965 * virtual address space currently. 3966 */ 3967 if ((*copybuf_used + MMU_PAGESIZE) <= 3968 dma->dp_copybuf_size) { 3969 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3970 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3971 MMU_PAGEMASK); 3972 } else { 3973 if (copybuf_sz_power_2) { 3974 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3975 (((uintptr_t)dma->dp_kva + 3976 (*copybuf_used & 3977 (dma->dp_copybuf_size - 1))) & 3978 MMU_PAGEMASK); 3979 } else { 3980 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3981 (((uintptr_t)dma->dp_kva + 3982 (*copybuf_used % 3983 dma->dp_copybuf_size)) & 3984 MMU_PAGEMASK); 3985 } 3986 } 3987 3988 /* 3989 * if we haven't used up the available copy buffer yet, 3990 * map the kva to the physical page. 3991 */ 3992 if (!dma->dp_cb_remaping && ((*copybuf_used + 3993 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3994 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3995 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3996 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3997 dma->dp_pgmap[pidx].pm_kaddr); 3998 } else { 3999 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4000 sinfo->si_asp, 4001 dma->dp_pgmap[pidx].pm_kaddr); 4002 } 4003 4004 /* 4005 * we've used up the available copy buffer, this page 4006 * will have to be mapped during rootnex_dma_win() when 4007 * we switch to a new window which requires a re-map 4008 * the copy buffer. (32-bit kernel only) 4009 */ 4010 } else { 4011 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4012 } 4013 #endif 4014 /* go to the next page_t */ 4015 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4016 *cur_pp = (*cur_pp)->p_next; 4017 } 4018 } 4019 4020 /* add to the copy buffer count */ 4021 *copybuf_used += MMU_PAGESIZE; 4022 4023 /* 4024 * This cookie doesn't use the copy buffer. Walk through the pages this 4025 * cookie occupies to reflect this. 4026 */ 4027 } else { 4028 /* 4029 * figure out how many pages the cookie occupies. We need to 4030 * use the original page offset of the buffer and the cookies 4031 * offset in the buffer to do this. 4032 */ 4033 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4034 pcnt = mmu_btopr(cookie->dmac_size + poff); 4035 4036 while (pcnt > 0) { 4037 #if !defined(__amd64) 4038 /* 4039 * the 32-bit kernel doesn't have seg kpm, so we need 4040 * to map in the driver buffer (if it didn't come down 4041 * with a kernel VA) on the fly. Since this page doesn't 4042 * use the copy buffer, it's not, or will it ever, have 4043 * to be mapped in. 4044 */ 4045 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4046 #endif 4047 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4048 4049 /* 4050 * we need to update pidx and cur_pp or we'll loose 4051 * track of where we are. 4052 */ 4053 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4054 *cur_pp = (*cur_pp)->p_next; 4055 } 4056 pidx++; 4057 pcnt--; 4058 } 4059 } 4060 } 4061 4062 4063 /* 4064 * rootnex_sgllen_window_boundary() 4065 * Called in the bind slow path when the next cookie causes us to exceed (in 4066 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4067 * length supported by the DMA H/W. 4068 */ 4069 static int 4070 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4071 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4072 off_t cur_offset) 4073 { 4074 off_t new_offset; 4075 size_t trim_sz; 4076 off_t coffset; 4077 4078 4079 /* 4080 * if we know we'll never have to trim, it's pretty easy. Just move to 4081 * the next window and init it. We're done. 4082 */ 4083 if (!dma->dp_trim_required) { 4084 (*windowp)++; 4085 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4086 (*windowp)->wd_cookie_cnt++; 4087 (*windowp)->wd_size = cookie->dmac_size; 4088 return (DDI_SUCCESS); 4089 } 4090 4091 /* figure out how much we need to trim from the window */ 4092 ASSERT(attr->dma_attr_granular != 0); 4093 if (dma->dp_granularity_power_2) { 4094 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4095 } else { 4096 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4097 } 4098 4099 /* The window's a whole multiple of granularity. We're done */ 4100 if (trim_sz == 0) { 4101 (*windowp)++; 4102 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4103 (*windowp)->wd_cookie_cnt++; 4104 (*windowp)->wd_size = cookie->dmac_size; 4105 return (DDI_SUCCESS); 4106 } 4107 4108 /* 4109 * The window's not a whole multiple of granularity, since we know this 4110 * is due to the sgllen, we need to go back to the last cookie and trim 4111 * that one, add the left over part of the old cookie into the new 4112 * window, and then add in the new cookie into the new window. 4113 */ 4114 4115 /* 4116 * make sure the driver isn't making us do something bad... Trimming and 4117 * sgllen == 1 don't go together. 4118 */ 4119 if (attr->dma_attr_sgllen == 1) { 4120 return (DDI_DMA_NOMAPPING); 4121 } 4122 4123 /* 4124 * first, setup the current window to account for the trim. Need to go 4125 * back to the last cookie for this. 4126 */ 4127 cookie--; 4128 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4129 (*windowp)->wd_trim.tr_last_cookie = cookie; 4130 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4131 ASSERT(cookie->dmac_size > trim_sz); 4132 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4133 (*windowp)->wd_size -= trim_sz; 4134 4135 /* save the buffer offsets for the next window */ 4136 coffset = cookie->dmac_size - trim_sz; 4137 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4138 4139 /* 4140 * set this now in case this is the first window. all other cases are 4141 * set in dma_win() 4142 */ 4143 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4144 4145 /* 4146 * initialize the next window using what's left over in the previous 4147 * cookie. 4148 */ 4149 (*windowp)++; 4150 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4151 (*windowp)->wd_cookie_cnt++; 4152 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4153 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4154 (*windowp)->wd_trim.tr_first_size = trim_sz; 4155 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4156 (*windowp)->wd_dosync = B_TRUE; 4157 } 4158 4159 /* 4160 * now go back to the current cookie and add it to the new window. set 4161 * the new window size to the what was left over from the previous 4162 * cookie and what's in the current cookie. 4163 */ 4164 cookie++; 4165 (*windowp)->wd_cookie_cnt++; 4166 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4167 4168 /* 4169 * trim plus the next cookie could put us over maxxfer (a cookie can be 4170 * a max size of maxxfer). Handle that case. 4171 */ 4172 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4173 /* 4174 * maxxfer is already a whole multiple of granularity, and this 4175 * trim will be <= the previous trim (since a cookie can't be 4176 * larger than maxxfer). Make things simple here. 4177 */ 4178 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4179 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4180 (*windowp)->wd_trim.tr_last_cookie = cookie; 4181 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4182 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4183 (*windowp)->wd_size -= trim_sz; 4184 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4185 4186 /* save the buffer offsets for the next window */ 4187 coffset = cookie->dmac_size - trim_sz; 4188 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4189 4190 /* setup the next window */ 4191 (*windowp)++; 4192 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4193 (*windowp)->wd_cookie_cnt++; 4194 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4195 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4196 coffset; 4197 (*windowp)->wd_trim.tr_first_size = trim_sz; 4198 } 4199 4200 return (DDI_SUCCESS); 4201 } 4202 4203 4204 /* 4205 * rootnex_copybuf_window_boundary() 4206 * Called in bind slowpath when we get to a window boundary because we used 4207 * up all the copy buffer that we have. 4208 */ 4209 static int 4210 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4211 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4212 size_t *copybuf_used) 4213 { 4214 rootnex_sglinfo_t *sinfo; 4215 off_t new_offset; 4216 size_t trim_sz; 4217 paddr_t paddr; 4218 off_t coffset; 4219 uint_t pidx; 4220 off_t poff; 4221 4222 4223 sinfo = &dma->dp_sglinfo; 4224 4225 /* 4226 * the copy buffer should be a whole multiple of page size. We know that 4227 * this cookie is <= MMU_PAGESIZE. 4228 */ 4229 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4230 4231 /* 4232 * from now on, all new windows in this bind need to be re-mapped during 4233 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4234 * space... 4235 */ 4236 #if !defined(__amd64) 4237 dma->dp_cb_remaping = B_TRUE; 4238 #endif 4239 4240 /* reset copybuf used */ 4241 *copybuf_used = 0; 4242 4243 /* 4244 * if we don't have to trim (since granularity is set to 1), go to the 4245 * next window and add the current cookie to it. We know the current 4246 * cookie uses the copy buffer since we're in this code path. 4247 */ 4248 if (!dma->dp_trim_required) { 4249 (*windowp)++; 4250 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4251 4252 /* Add this cookie to the new window */ 4253 (*windowp)->wd_cookie_cnt++; 4254 (*windowp)->wd_size += cookie->dmac_size; 4255 *copybuf_used += MMU_PAGESIZE; 4256 return (DDI_SUCCESS); 4257 } 4258 4259 /* 4260 * *** may need to trim, figure it out. 4261 */ 4262 4263 /* figure out how much we need to trim from the window */ 4264 if (dma->dp_granularity_power_2) { 4265 trim_sz = (*windowp)->wd_size & 4266 (hp->dmai_attr.dma_attr_granular - 1); 4267 } else { 4268 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4269 } 4270 4271 /* 4272 * if the window's a whole multiple of granularity, go to the next 4273 * window, init it, then add in the current cookie. We know the current 4274 * cookie uses the copy buffer since we're in this code path. 4275 */ 4276 if (trim_sz == 0) { 4277 (*windowp)++; 4278 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4279 4280 /* Add this cookie to the new window */ 4281 (*windowp)->wd_cookie_cnt++; 4282 (*windowp)->wd_size += cookie->dmac_size; 4283 *copybuf_used += MMU_PAGESIZE; 4284 return (DDI_SUCCESS); 4285 } 4286 4287 /* 4288 * *** We figured it out, we definitly need to trim 4289 */ 4290 4291 /* 4292 * make sure the driver isn't making us do something bad... 4293 * Trimming and sgllen == 1 don't go together. 4294 */ 4295 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4296 return (DDI_DMA_NOMAPPING); 4297 } 4298 4299 /* 4300 * first, setup the current window to account for the trim. Need to go 4301 * back to the last cookie for this. Some of the last cookie will be in 4302 * the current window, and some of the last cookie will be in the new 4303 * window. All of the current cookie will be in the new window. 4304 */ 4305 cookie--; 4306 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4307 (*windowp)->wd_trim.tr_last_cookie = cookie; 4308 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4309 ASSERT(cookie->dmac_size > trim_sz); 4310 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4311 (*windowp)->wd_size -= trim_sz; 4312 4313 /* 4314 * we're trimming the last cookie (not the current cookie). So that 4315 * last cookie may have or may not have been using the copy buffer ( 4316 * we know the cookie passed in uses the copy buffer since we're in 4317 * this code path). 4318 * 4319 * If the last cookie doesn't use the copy buffer, nothing special to 4320 * do. However, if it does uses the copy buffer, it will be both the 4321 * last page in the current window and the first page in the next 4322 * window. Since we are reusing the copy buffer (and KVA space on the 4323 * 32-bit kernel), this page will use the end of the copy buffer in the 4324 * current window, and the start of the copy buffer in the next window. 4325 * Track that info... The cookie physical address was already set to 4326 * the copy buffer physical address in setup_cookie.. 4327 */ 4328 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4329 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4330 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4331 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4332 (*windowp)->wd_trim.tr_last_pidx = pidx; 4333 (*windowp)->wd_trim.tr_last_cbaddr = 4334 dma->dp_pgmap[pidx].pm_cbaddr; 4335 #if !defined(__amd64) 4336 (*windowp)->wd_trim.tr_last_kaddr = 4337 dma->dp_pgmap[pidx].pm_kaddr; 4338 #endif 4339 } 4340 4341 /* save the buffer offsets for the next window */ 4342 coffset = cookie->dmac_size - trim_sz; 4343 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4344 4345 /* 4346 * set this now in case this is the first window. all other cases are 4347 * set in dma_win() 4348 */ 4349 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4350 4351 /* 4352 * initialize the next window using what's left over in the previous 4353 * cookie. 4354 */ 4355 (*windowp)++; 4356 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4357 (*windowp)->wd_cookie_cnt++; 4358 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4359 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4360 (*windowp)->wd_trim.tr_first_size = trim_sz; 4361 4362 /* 4363 * again, we're tracking if the last cookie uses the copy buffer. 4364 * read the comment above for more info on why we need to track 4365 * additional state. 4366 * 4367 * For the first cookie in the new window, we need reset the physical 4368 * address to DMA into to the start of the copy buffer plus any 4369 * initial page offset which may be present. 4370 */ 4371 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4372 (*windowp)->wd_dosync = B_TRUE; 4373 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4374 (*windowp)->wd_trim.tr_first_pidx = pidx; 4375 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4376 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4377 4378 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4379 poff; 4380 (*windowp)->wd_trim.tr_first_paddr = 4381 ROOTNEX_PADDR_TO_RBASE(paddr); 4382 4383 #if !defined(__amd64) 4384 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4385 #endif 4386 /* account for the cookie copybuf usage in the new window */ 4387 *copybuf_used += MMU_PAGESIZE; 4388 4389 /* 4390 * every piece of code has to have a hack, and here is this 4391 * ones :-) 4392 * 4393 * There is a complex interaction between setup_cookie and the 4394 * copybuf window boundary. The complexity had to be in either 4395 * the maxxfer window, or the copybuf window, and I chose the 4396 * copybuf code. 4397 * 4398 * So in this code path, we have taken the last cookie, 4399 * virtually broken it in half due to the trim, and it happens 4400 * to use the copybuf which further complicates life. At the 4401 * same time, we have already setup the current cookie, which 4402 * is now wrong. More background info: the current cookie uses 4403 * the copybuf, so it is only a page long max. So we need to 4404 * fix the current cookies copy buffer address, physical 4405 * address, and kva for the 32-bit kernel. We due this by 4406 * bumping them by page size (of course, we can't due this on 4407 * the physical address since the copy buffer may not be 4408 * physically contiguous). 4409 */ 4410 cookie++; 4411 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4412 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4413 4414 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4415 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4416 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4417 4418 #if !defined(__amd64) 4419 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4420 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4421 #endif 4422 } else { 4423 /* go back to the current cookie */ 4424 cookie++; 4425 } 4426 4427 /* 4428 * add the current cookie to the new window. set the new window size to 4429 * the what was left over from the previous cookie and what's in the 4430 * current cookie. 4431 */ 4432 (*windowp)->wd_cookie_cnt++; 4433 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4434 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4435 4436 /* 4437 * we know that the cookie passed in always uses the copy buffer. We 4438 * wouldn't be here if it didn't. 4439 */ 4440 *copybuf_used += MMU_PAGESIZE; 4441 4442 return (DDI_SUCCESS); 4443 } 4444 4445 4446 /* 4447 * rootnex_maxxfer_window_boundary() 4448 * Called in bind slowpath when we get to a window boundary because we will 4449 * go over maxxfer. 4450 */ 4451 static int 4452 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4453 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4454 { 4455 size_t dmac_size; 4456 off_t new_offset; 4457 size_t trim_sz; 4458 off_t coffset; 4459 4460 4461 /* 4462 * calculate how much we have to trim off of the current cookie to equal 4463 * maxxfer. We don't have to account for granularity here since our 4464 * maxxfer already takes that into account. 4465 */ 4466 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4467 ASSERT(trim_sz <= cookie->dmac_size); 4468 ASSERT(trim_sz <= dma->dp_maxxfer); 4469 4470 /* save cookie size since we need it later and we might change it */ 4471 dmac_size = cookie->dmac_size; 4472 4473 /* 4474 * if we're not trimming the entire cookie, setup the current window to 4475 * account for the trim. 4476 */ 4477 if (trim_sz < cookie->dmac_size) { 4478 (*windowp)->wd_cookie_cnt++; 4479 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4480 (*windowp)->wd_trim.tr_last_cookie = cookie; 4481 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4482 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4483 (*windowp)->wd_size = dma->dp_maxxfer; 4484 4485 /* 4486 * set the adjusted cookie size now in case this is the first 4487 * window. All other windows are taken care of in get win 4488 */ 4489 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4490 } 4491 4492 /* 4493 * coffset is the current offset within the cookie, new_offset is the 4494 * current offset with the entire buffer. 4495 */ 4496 coffset = dmac_size - trim_sz; 4497 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4498 4499 /* initialize the next window */ 4500 (*windowp)++; 4501 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4502 (*windowp)->wd_cookie_cnt++; 4503 (*windowp)->wd_size = trim_sz; 4504 if (trim_sz < dmac_size) { 4505 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4506 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4507 coffset; 4508 (*windowp)->wd_trim.tr_first_size = trim_sz; 4509 } 4510 4511 return (DDI_SUCCESS); 4512 } 4513 4514 4515 /*ARGSUSED*/ 4516 static int 4517 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4518 off_t off, size_t len, uint_t cache_flags) 4519 { 4520 rootnex_sglinfo_t *sinfo; 4521 rootnex_pgmap_t *cbpage; 4522 rootnex_window_t *win; 4523 ddi_dma_impl_t *hp; 4524 rootnex_dma_t *dma; 4525 caddr_t fromaddr; 4526 caddr_t toaddr; 4527 uint_t psize; 4528 off_t offset; 4529 uint_t pidx; 4530 size_t size; 4531 off_t poff; 4532 int e; 4533 4534 4535 hp = (ddi_dma_impl_t *)handle; 4536 dma = (rootnex_dma_t *)hp->dmai_private; 4537 sinfo = &dma->dp_sglinfo; 4538 4539 /* 4540 * if we don't have any windows, we don't need to sync. A copybuf 4541 * will cause us to have at least one window. 4542 */ 4543 if (dma->dp_window == NULL) { 4544 return (DDI_SUCCESS); 4545 } 4546 4547 /* This window may not need to be sync'd */ 4548 win = &dma->dp_window[dma->dp_current_win]; 4549 if (!win->wd_dosync) { 4550 return (DDI_SUCCESS); 4551 } 4552 4553 /* handle off and len special cases */ 4554 if ((off == 0) || (rootnex_sync_ignore_params)) { 4555 offset = win->wd_offset; 4556 } else { 4557 offset = off; 4558 } 4559 if ((len == 0) || (rootnex_sync_ignore_params)) { 4560 size = win->wd_size; 4561 } else { 4562 size = len; 4563 } 4564 4565 /* check the sync args to make sure they make a little sense */ 4566 if (rootnex_sync_check_parms) { 4567 e = rootnex_valid_sync_parms(hp, win, offset, size, 4568 cache_flags); 4569 if (e != DDI_SUCCESS) { 4570 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4571 return (DDI_FAILURE); 4572 } 4573 } 4574 4575 /* 4576 * special case the first page to handle the offset into the page. The 4577 * offset to the current page for our buffer is the offset into the 4578 * first page of the buffer plus our current offset into the buffer 4579 * itself, masked of course. 4580 */ 4581 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4582 psize = MIN((MMU_PAGESIZE - poff), size); 4583 4584 /* go through all the pages that we want to sync */ 4585 while (size > 0) { 4586 /* 4587 * Calculate the page index relative to the start of the buffer. 4588 * The index to the current page for our buffer is the offset 4589 * into the first page of the buffer plus our current offset 4590 * into the buffer itself, shifted of course... 4591 */ 4592 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4593 ASSERT(pidx < sinfo->si_max_pages); 4594 4595 /* 4596 * if this page uses the copy buffer, we need to sync it, 4597 * otherwise, go on to the next page. 4598 */ 4599 cbpage = &dma->dp_pgmap[pidx]; 4600 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4601 (cbpage->pm_uses_copybuf == B_FALSE)); 4602 if (cbpage->pm_uses_copybuf) { 4603 /* cbaddr and kaddr should be page aligned */ 4604 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4605 MMU_PAGEOFFSET) == 0); 4606 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4607 MMU_PAGEOFFSET) == 0); 4608 4609 /* 4610 * if we're copying for the device, we are going to 4611 * copy from the drivers buffer and to the rootnex 4612 * allocated copy buffer. 4613 */ 4614 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4615 fromaddr = cbpage->pm_kaddr + poff; 4616 toaddr = cbpage->pm_cbaddr + poff; 4617 ROOTNEX_DPROBE2(rootnex__sync__dev, 4618 dev_info_t *, dma->dp_dip, size_t, psize); 4619 4620 /* 4621 * if we're copying for the cpu/kernel, we are going to 4622 * copy from the rootnex allocated copy buffer to the 4623 * drivers buffer. 4624 */ 4625 } else { 4626 fromaddr = cbpage->pm_cbaddr + poff; 4627 toaddr = cbpage->pm_kaddr + poff; 4628 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4629 dev_info_t *, dma->dp_dip, size_t, psize); 4630 } 4631 4632 bcopy(fromaddr, toaddr, psize); 4633 } 4634 4635 /* 4636 * decrement size until we're done, update our offset into the 4637 * buffer, and get the next page size. 4638 */ 4639 size -= psize; 4640 offset += psize; 4641 psize = MIN(MMU_PAGESIZE, size); 4642 4643 /* page offset is zero for the rest of this loop */ 4644 poff = 0; 4645 } 4646 4647 return (DDI_SUCCESS); 4648 } 4649 4650 /* 4651 * rootnex_dma_sync() 4652 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4653 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4654 * is set, ddi_dma_sync() returns immediately passing back success. 4655 */ 4656 /*ARGSUSED*/ 4657 static int 4658 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4659 off_t off, size_t len, uint_t cache_flags) 4660 { 4661 #if defined(__amd64) && !defined(__xpv) 4662 if (IOMMU_USED(rdip)) { 4663 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4664 cache_flags)); 4665 } 4666 #endif 4667 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4668 cache_flags)); 4669 } 4670 4671 /* 4672 * rootnex_valid_sync_parms() 4673 * checks the parameters passed to sync to verify they are correct. 4674 */ 4675 static int 4676 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4677 off_t offset, size_t size, uint_t cache_flags) 4678 { 4679 off_t woffset; 4680 4681 4682 /* 4683 * the first part of the test to make sure the offset passed in is 4684 * within the window. 4685 */ 4686 if (offset < win->wd_offset) { 4687 return (DDI_FAILURE); 4688 } 4689 4690 /* 4691 * second and last part of the test to make sure the offset and length 4692 * passed in is within the window. 4693 */ 4694 woffset = offset - win->wd_offset; 4695 if ((woffset + size) > win->wd_size) { 4696 return (DDI_FAILURE); 4697 } 4698 4699 /* 4700 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4701 * be set too. 4702 */ 4703 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4704 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4705 return (DDI_SUCCESS); 4706 } 4707 4708 /* 4709 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4710 * should be set. Also DDI_DMA_READ should be set in the flags. 4711 */ 4712 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4713 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4714 (hp->dmai_rflags & DDI_DMA_READ)) { 4715 return (DDI_SUCCESS); 4716 } 4717 4718 return (DDI_FAILURE); 4719 } 4720 4721 4722 /*ARGSUSED*/ 4723 static int 4724 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4725 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4726 uint_t *ccountp) 4727 { 4728 rootnex_window_t *window; 4729 rootnex_trim_t *trim; 4730 ddi_dma_impl_t *hp; 4731 rootnex_dma_t *dma; 4732 ddi_dma_obj_t *dmao; 4733 #if !defined(__amd64) 4734 rootnex_sglinfo_t *sinfo; 4735 rootnex_pgmap_t *pmap; 4736 uint_t pidx; 4737 uint_t pcnt; 4738 off_t poff; 4739 int i; 4740 #endif 4741 4742 4743 hp = (ddi_dma_impl_t *)handle; 4744 dma = (rootnex_dma_t *)hp->dmai_private; 4745 #if !defined(__amd64) 4746 sinfo = &dma->dp_sglinfo; 4747 #endif 4748 4749 /* If we try and get a window which doesn't exist, return failure */ 4750 if (win >= hp->dmai_nwin) { 4751 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4752 return (DDI_FAILURE); 4753 } 4754 4755 dmao = dma->dp_dvma_used ? &dma->dp_dma : &dma->dp_dvma; 4756 4757 /* 4758 * if we don't have any windows, and they're asking for the first 4759 * window, setup the cookie pointer to the first cookie in the bind. 4760 * setup our return values, then increment the cookie since we return 4761 * the first cookie on the stack. 4762 */ 4763 if (dma->dp_window == NULL) { 4764 if (win != 0) { 4765 ROOTNEX_DPROF_INC( 4766 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4767 return (DDI_FAILURE); 4768 } 4769 hp->dmai_cookie = dma->dp_cookies; 4770 *offp = 0; 4771 *lenp = dmao->dmao_size; 4772 *ccountp = dma->dp_sglinfo.si_sgl_size; 4773 *cookiep = hp->dmai_cookie[0]; 4774 hp->dmai_cookie++; 4775 return (DDI_SUCCESS); 4776 } 4777 4778 /* sync the old window before moving on to the new one */ 4779 window = &dma->dp_window[dma->dp_current_win]; 4780 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4781 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4782 DDI_DMA_SYNC_FORCPU); 4783 } 4784 4785 #if !defined(__amd64) 4786 /* 4787 * before we move to the next window, if we need to re-map, unmap all 4788 * the pages in this window. 4789 */ 4790 if (dma->dp_cb_remaping) { 4791 /* 4792 * If we switch to this window again, we'll need to map in 4793 * on the fly next time. 4794 */ 4795 window->wd_remap_copybuf = B_TRUE; 4796 4797 /* 4798 * calculate the page index into the buffer where this window 4799 * starts, and the number of pages this window takes up. 4800 */ 4801 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4802 MMU_PAGESHIFT; 4803 poff = (sinfo->si_buf_offset + window->wd_offset) & 4804 MMU_PAGEOFFSET; 4805 pcnt = mmu_btopr(window->wd_size + poff); 4806 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4807 4808 /* unmap pages which are currently mapped in this window */ 4809 for (i = 0; i < pcnt; i++) { 4810 if (dma->dp_pgmap[pidx].pm_mapped) { 4811 hat_unload(kas.a_hat, 4812 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4813 HAT_UNLOAD); 4814 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4815 } 4816 pidx++; 4817 } 4818 } 4819 #endif 4820 4821 /* 4822 * Move to the new window. 4823 * NOTE: current_win must be set for sync to work right 4824 */ 4825 dma->dp_current_win = win; 4826 window = &dma->dp_window[win]; 4827 4828 /* if needed, adjust the first and/or last cookies for trim */ 4829 trim = &window->wd_trim; 4830 if (trim->tr_trim_first) { 4831 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4832 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4833 #if !defined(__amd64) 4834 window->wd_first_cookie->dmac_type = 4835 (window->wd_first_cookie->dmac_type & 4836 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4837 #endif 4838 if (trim->tr_first_copybuf_win) { 4839 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4840 trim->tr_first_cbaddr; 4841 #if !defined(__amd64) 4842 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4843 trim->tr_first_kaddr; 4844 #endif 4845 } 4846 } 4847 if (trim->tr_trim_last) { 4848 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4849 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4850 if (trim->tr_last_copybuf_win) { 4851 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4852 trim->tr_last_cbaddr; 4853 #if !defined(__amd64) 4854 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4855 trim->tr_last_kaddr; 4856 #endif 4857 } 4858 } 4859 4860 /* 4861 * setup the cookie pointer to the first cookie in the window. setup 4862 * our return values, then increment the cookie since we return the 4863 * first cookie on the stack. 4864 */ 4865 hp->dmai_cookie = window->wd_first_cookie; 4866 *offp = window->wd_offset; 4867 *lenp = window->wd_size; 4868 *ccountp = window->wd_cookie_cnt; 4869 *cookiep = hp->dmai_cookie[0]; 4870 hp->dmai_cookie++; 4871 4872 #if !defined(__amd64) 4873 /* re-map copybuf if required for this window */ 4874 if (dma->dp_cb_remaping) { 4875 /* 4876 * calculate the page index into the buffer where this 4877 * window starts. 4878 */ 4879 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4880 MMU_PAGESHIFT; 4881 ASSERT(pidx < sinfo->si_max_pages); 4882 4883 /* 4884 * the first page can get unmapped if it's shared with the 4885 * previous window. Even if the rest of this window is already 4886 * mapped in, we need to still check this one. 4887 */ 4888 pmap = &dma->dp_pgmap[pidx]; 4889 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4890 if (pmap->pm_pp != NULL) { 4891 pmap->pm_mapped = B_TRUE; 4892 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4893 } else if (pmap->pm_vaddr != NULL) { 4894 pmap->pm_mapped = B_TRUE; 4895 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4896 pmap->pm_kaddr); 4897 } 4898 } 4899 pidx++; 4900 4901 /* map in the rest of the pages if required */ 4902 if (window->wd_remap_copybuf) { 4903 window->wd_remap_copybuf = B_FALSE; 4904 4905 /* figure out many pages this window takes up */ 4906 poff = (sinfo->si_buf_offset + window->wd_offset) & 4907 MMU_PAGEOFFSET; 4908 pcnt = mmu_btopr(window->wd_size + poff); 4909 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4910 4911 /* map pages which require it */ 4912 for (i = 1; i < pcnt; i++) { 4913 pmap = &dma->dp_pgmap[pidx]; 4914 if (pmap->pm_uses_copybuf) { 4915 ASSERT(pmap->pm_mapped == B_FALSE); 4916 if (pmap->pm_pp != NULL) { 4917 pmap->pm_mapped = B_TRUE; 4918 i86_pp_map(pmap->pm_pp, 4919 pmap->pm_kaddr); 4920 } else if (pmap->pm_vaddr != NULL) { 4921 pmap->pm_mapped = B_TRUE; 4922 i86_va_map(pmap->pm_vaddr, 4923 sinfo->si_asp, 4924 pmap->pm_kaddr); 4925 } 4926 } 4927 pidx++; 4928 } 4929 } 4930 } 4931 #endif 4932 4933 /* if the new window uses the copy buffer, sync it for the device */ 4934 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4935 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4936 DDI_DMA_SYNC_FORDEV); 4937 } 4938 4939 return (DDI_SUCCESS); 4940 } 4941 4942 /* 4943 * rootnex_dma_win() 4944 * called from ddi_dma_getwin() 4945 */ 4946 /*ARGSUSED*/ 4947 static int 4948 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4949 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4950 uint_t *ccountp) 4951 { 4952 #if defined(__amd64) && !defined(__xpv) 4953 if (IOMMU_USED(rdip)) { 4954 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4955 cookiep, ccountp)); 4956 } 4957 #endif 4958 4959 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4960 cookiep, ccountp)); 4961 } 4962 4963 /*ARGSUSED*/ 4964 static int 4965 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4966 ddi_dma_handle_t handle, void *v) 4967 { 4968 ddi_dma_impl_t *hp; 4969 rootnex_dma_t *dma; 4970 4971 hp = (ddi_dma_impl_t *)handle; 4972 dma = (rootnex_dma_t *)hp->dmai_private; 4973 dma->dp_iommu_private = v; 4974 4975 return (DDI_SUCCESS); 4976 } 4977 4978 /*ARGSUSED*/ 4979 static void * 4980 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4981 ddi_dma_handle_t handle) 4982 { 4983 ddi_dma_impl_t *hp; 4984 rootnex_dma_t *dma; 4985 4986 hp = (ddi_dma_impl_t *)handle; 4987 dma = (rootnex_dma_t *)hp->dmai_private; 4988 4989 return (dma->dp_iommu_private); 4990 } 4991 4992 /* 4993 * ************************ 4994 * obsoleted dma routines 4995 * ************************ 4996 */ 4997 4998 /* 4999 * rootnex_dma_map() 5000 * called from ddi_dma_setup() 5001 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 5002 */ 5003 /* ARGSUSED */ 5004 static int 5005 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 5006 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 5007 { 5008 #if defined(__amd64) 5009 /* 5010 * this interface is not supported in 64-bit x86 kernel. See comment in 5011 * rootnex_dma_mctl() 5012 */ 5013 return (DDI_DMA_NORESOURCES); 5014 5015 #else /* 32-bit x86 kernel */ 5016 ddi_dma_handle_t *lhandlep; 5017 ddi_dma_handle_t lhandle; 5018 ddi_dma_cookie_t cookie; 5019 ddi_dma_attr_t dma_attr; 5020 ddi_dma_lim_t *dma_lim; 5021 uint_t ccnt; 5022 int e; 5023 5024 5025 /* 5026 * if the driver is just testing to see if it's possible to do the bind, 5027 * we'll use local state. Otherwise, use the handle pointer passed in. 5028 */ 5029 if (handlep == NULL) { 5030 lhandlep = &lhandle; 5031 } else { 5032 lhandlep = handlep; 5033 } 5034 5035 /* convert the limit structure to a dma_attr one */ 5036 dma_lim = dmareq->dmar_limits; 5037 dma_attr.dma_attr_version = DMA_ATTR_V0; 5038 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 5039 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 5040 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 5041 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 5042 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 5043 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 5044 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 5045 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 5046 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 5047 dma_attr.dma_attr_align = MMU_PAGESIZE; 5048 dma_attr.dma_attr_flags = 0; 5049 5050 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 5051 dmareq->dmar_arg, lhandlep); 5052 if (e != DDI_SUCCESS) { 5053 return (e); 5054 } 5055 5056 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 5057 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 5058 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5059 return (e); 5060 } 5061 5062 /* 5063 * if the driver is just testing to see if it's possible to do the bind, 5064 * free up the local state and return the result. 5065 */ 5066 if (handlep == NULL) { 5067 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 5068 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5069 if (e == DDI_DMA_MAPPED) { 5070 return (DDI_DMA_MAPOK); 5071 } else { 5072 return (DDI_DMA_NOMAPPING); 5073 } 5074 } 5075 5076 return (e); 5077 #endif /* defined(__amd64) */ 5078 } 5079 5080 /* 5081 * rootnex_dma_mctl() 5082 * 5083 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 5084 */ 5085 /* ARGSUSED */ 5086 static int 5087 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5088 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5089 uint_t cache_flags) 5090 { 5091 #if defined(__amd64) 5092 /* 5093 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 5094 * common implementation in genunix, so they no longer have x86 5095 * specific functionality which called into dma_ctl. 5096 * 5097 * The rest of the obsoleted interfaces were never supported in the 5098 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 5099 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 5100 * implementation issues. 5101 * 5102 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 5103 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 5104 * reflect that now too... 5105 * 5106 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 5107 * not going to put this functionality into the 64-bit x86 kernel now. 5108 * It wasn't ported to the 64-bit kernel for s10, no reason to change 5109 * that in a future release. 5110 */ 5111 return (DDI_FAILURE); 5112 5113 #else /* 32-bit x86 kernel */ 5114 ddi_dma_cookie_t lcookie; 5115 ddi_dma_cookie_t *cookie; 5116 rootnex_window_t *window; 5117 ddi_dma_impl_t *hp; 5118 rootnex_dma_t *dma; 5119 uint_t nwin; 5120 uint_t ccnt; 5121 size_t len; 5122 off_t off; 5123 int e; 5124 5125 5126 /* 5127 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 5128 * hacky since were optimizing for the current interfaces and so we can 5129 * cleanup the mess in genunix. Hopefully we will remove the this 5130 * obsoleted routines someday soon. 5131 */ 5132 5133 switch (request) { 5134 5135 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 5136 hp = (ddi_dma_impl_t *)handle; 5137 cookie = (ddi_dma_cookie_t *)objpp; 5138 5139 /* 5140 * convert segment to cookie. We don't distinguish between the 5141 * two :-) 5142 */ 5143 *cookie = *hp->dmai_cookie; 5144 *lenp = cookie->dmac_size; 5145 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 5146 return (DDI_SUCCESS); 5147 5148 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 5149 hp = (ddi_dma_impl_t *)handle; 5150 dma = (rootnex_dma_t *)hp->dmai_private; 5151 5152 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 5153 return (DDI_DMA_STALE); 5154 } 5155 5156 /* handle the case where we don't have any windows */ 5157 if (dma->dp_window == NULL) { 5158 /* 5159 * if seg == NULL, and we don't have any windows, 5160 * return the first cookie in the sgl. 5161 */ 5162 if (*lenp == NULL) { 5163 dma->dp_current_cookie = 0; 5164 hp->dmai_cookie = dma->dp_cookies; 5165 *objpp = (caddr_t)handle; 5166 return (DDI_SUCCESS); 5167 5168 /* if we have more cookies, go to the next cookie */ 5169 } else { 5170 if ((dma->dp_current_cookie + 1) >= 5171 dma->dp_sglinfo.si_sgl_size) { 5172 return (DDI_DMA_DONE); 5173 } 5174 dma->dp_current_cookie++; 5175 hp->dmai_cookie++; 5176 return (DDI_SUCCESS); 5177 } 5178 } 5179 5180 /* We have one or more windows */ 5181 window = &dma->dp_window[dma->dp_current_win]; 5182 5183 /* 5184 * if seg == NULL, return the first cookie in the current 5185 * window 5186 */ 5187 if (*lenp == NULL) { 5188 dma->dp_current_cookie = 0; 5189 hp->dmai_cookie = window->wd_first_cookie; 5190 5191 /* 5192 * go to the next cookie in the window then see if we done with 5193 * this window. 5194 */ 5195 } else { 5196 if ((dma->dp_current_cookie + 1) >= 5197 window->wd_cookie_cnt) { 5198 return (DDI_DMA_DONE); 5199 } 5200 dma->dp_current_cookie++; 5201 hp->dmai_cookie++; 5202 } 5203 *objpp = (caddr_t)handle; 5204 return (DDI_SUCCESS); 5205 5206 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 5207 hp = (ddi_dma_impl_t *)handle; 5208 dma = (rootnex_dma_t *)hp->dmai_private; 5209 5210 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 5211 return (DDI_DMA_STALE); 5212 } 5213 5214 /* if win == NULL, return the first window in the bind */ 5215 if (*offp == NULL) { 5216 nwin = 0; 5217 5218 /* 5219 * else, go to the next window then see if we're done with all 5220 * the windows. 5221 */ 5222 } else { 5223 nwin = dma->dp_current_win + 1; 5224 if (nwin >= hp->dmai_nwin) { 5225 return (DDI_DMA_DONE); 5226 } 5227 } 5228 5229 /* switch to the next window */ 5230 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 5231 &lcookie, &ccnt); 5232 ASSERT(e == DDI_SUCCESS); 5233 if (e != DDI_SUCCESS) { 5234 return (DDI_DMA_STALE); 5235 } 5236 5237 /* reset the cookie back to the first cookie in the window */ 5238 if (dma->dp_window != NULL) { 5239 window = &dma->dp_window[dma->dp_current_win]; 5240 hp->dmai_cookie = window->wd_first_cookie; 5241 } else { 5242 hp->dmai_cookie = dma->dp_cookies; 5243 } 5244 5245 *objpp = (caddr_t)handle; 5246 return (DDI_SUCCESS); 5247 5248 case DDI_DMA_FREE: /* ddi_dma_free() */ 5249 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 5250 (void) rootnex_dma_freehdl(dip, rdip, handle); 5251 if (rootnex_state->r_dvma_call_list_id) { 5252 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 5253 } 5254 return (DDI_SUCCESS); 5255 5256 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 5257 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 5258 /* should never get here, handled in genunix */ 5259 ASSERT(0); 5260 return (DDI_FAILURE); 5261 5262 case DDI_DMA_KVADDR: 5263 case DDI_DMA_GETERR: 5264 case DDI_DMA_COFF: 5265 return (DDI_FAILURE); 5266 } 5267 5268 return (DDI_FAILURE); 5269 #endif /* defined(__amd64) */ 5270 } 5271 5272 /* 5273 * ********* 5274 * FMA Code 5275 * ********* 5276 */ 5277 5278 /* 5279 * rootnex_fm_init() 5280 * FMA init busop 5281 */ 5282 /* ARGSUSED */ 5283 static int 5284 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5285 ddi_iblock_cookie_t *ibc) 5286 { 5287 *ibc = rootnex_state->r_err_ibc; 5288 5289 return (ddi_system_fmcap); 5290 } 5291 5292 /* 5293 * rootnex_dma_check() 5294 * Function called after a dma fault occurred to find out whether the 5295 * fault address is associated with a driver that is able to handle faults 5296 * and recover from faults. 5297 */ 5298 /* ARGSUSED */ 5299 static int 5300 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5301 const void *not_used) 5302 { 5303 rootnex_window_t *window; 5304 uint64_t start_addr; 5305 uint64_t fault_addr; 5306 ddi_dma_impl_t *hp; 5307 rootnex_dma_t *dma; 5308 uint64_t end_addr; 5309 size_t csize; 5310 int i; 5311 int j; 5312 5313 5314 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5315 hp = (ddi_dma_impl_t *)handle; 5316 ASSERT(hp); 5317 5318 dma = (rootnex_dma_t *)hp->dmai_private; 5319 5320 /* Get the address that we need to search for */ 5321 fault_addr = *(uint64_t *)addr; 5322 5323 /* 5324 * if we don't have any windows, we can just walk through all the 5325 * cookies. 5326 */ 5327 if (dma->dp_window == NULL) { 5328 /* for each cookie */ 5329 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5330 /* 5331 * if the faulted address is within the physical address 5332 * range of the cookie, return DDI_FM_NONFATAL. 5333 */ 5334 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5335 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5336 dma->dp_cookies[i].dmac_size))) { 5337 return (DDI_FM_NONFATAL); 5338 } 5339 } 5340 5341 /* fault_addr not within this DMA handle */ 5342 return (DDI_FM_UNKNOWN); 5343 } 5344 5345 /* we have mutiple windows, walk through each window */ 5346 for (i = 0; i < hp->dmai_nwin; i++) { 5347 window = &dma->dp_window[i]; 5348 5349 /* Go through all the cookies in the window */ 5350 for (j = 0; j < window->wd_cookie_cnt; j++) { 5351 5352 start_addr = window->wd_first_cookie[j].dmac_laddress; 5353 csize = window->wd_first_cookie[j].dmac_size; 5354 5355 /* 5356 * if we are trimming the first cookie in the window, 5357 * and this is the first cookie, adjust the start 5358 * address and size of the cookie to account for the 5359 * trim. 5360 */ 5361 if (window->wd_trim.tr_trim_first && (j == 0)) { 5362 start_addr = window->wd_trim.tr_first_paddr; 5363 csize = window->wd_trim.tr_first_size; 5364 } 5365 5366 /* 5367 * if we are trimming the last cookie in the window, 5368 * and this is the last cookie, adjust the start 5369 * address and size of the cookie to account for the 5370 * trim. 5371 */ 5372 if (window->wd_trim.tr_trim_last && 5373 (j == (window->wd_cookie_cnt - 1))) { 5374 start_addr = window->wd_trim.tr_last_paddr; 5375 csize = window->wd_trim.tr_last_size; 5376 } 5377 5378 end_addr = start_addr + csize; 5379 5380 /* 5381 * if the faulted address is within the physical 5382 * address of the cookie, return DDI_FM_NONFATAL. 5383 */ 5384 if ((fault_addr >= start_addr) && 5385 (fault_addr <= end_addr)) { 5386 return (DDI_FM_NONFATAL); 5387 } 5388 } 5389 } 5390 5391 /* fault_addr not within this DMA handle */ 5392 return (DDI_FM_UNKNOWN); 5393 } 5394 5395 /*ARGSUSED*/ 5396 static int 5397 rootnex_quiesce(dev_info_t *dip) 5398 { 5399 #if defined(__amd64) && !defined(__xpv) 5400 return (immu_quiesce()); 5401 #else 5402 return (DDI_SUCCESS); 5403 #endif 5404 } 5405 5406 #if defined(__xpv) 5407 void 5408 immu_init(void) 5409 { 5410 ; 5411 } 5412 5413 void 5414 immu_startup(void) 5415 { 5416 ; 5417 } 5418 /*ARGSUSED*/ 5419 void 5420 immu_physmem_update(uint64_t addr, uint64_t size) 5421 { 5422 ; 5423 } 5424 #endif 5425