1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 */ 29 30 /* 31 * x86 root nexus driver 32 */ 33 34 #include <sys/sysmacros.h> 35 #include <sys/conf.h> 36 #include <sys/autoconf.h> 37 #include <sys/sysmacros.h> 38 #include <sys/debug.h> 39 #include <sys/psw.h> 40 #include <sys/ddidmareq.h> 41 #include <sys/promif.h> 42 #include <sys/devops.h> 43 #include <sys/kmem.h> 44 #include <sys/cmn_err.h> 45 #include <vm/seg.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_dev.h> 48 #include <sys/vmem.h> 49 #include <sys/mman.h> 50 #include <vm/hat.h> 51 #include <vm/as.h> 52 #include <vm/page.h> 53 #include <sys/avintr.h> 54 #include <sys/errno.h> 55 #include <sys/modctl.h> 56 #include <sys/ddi_impldefs.h> 57 #include <sys/sunddi.h> 58 #include <sys/sunndi.h> 59 #include <sys/mach_intr.h> 60 #include <sys/psm.h> 61 #include <sys/ontrap.h> 62 #include <sys/atomic.h> 63 #include <sys/sdt.h> 64 #include <sys/rootnex.h> 65 #include <vm/hat_i86.h> 66 #include <sys/ddifm.h> 67 #include <sys/ddi_isa.h> 68 #include <sys/apic.h> 69 70 #ifdef __xpv 71 #include <sys/bootinfo.h> 72 #include <sys/hypervisor.h> 73 #include <sys/bootconf.h> 74 #include <vm/kboot_mmu.h> 75 #endif 76 77 #if defined(__amd64) && !defined(__xpv) 78 #include <sys/immu.h> 79 #endif 80 81 82 /* 83 * enable/disable extra checking of function parameters. Useful for debugging 84 * drivers. 85 */ 86 #ifdef DEBUG 87 int rootnex_alloc_check_parms = 1; 88 int rootnex_bind_check_parms = 1; 89 int rootnex_bind_check_inuse = 1; 90 int rootnex_unbind_verify_buffer = 0; 91 int rootnex_sync_check_parms = 1; 92 #else 93 int rootnex_alloc_check_parms = 0; 94 int rootnex_bind_check_parms = 0; 95 int rootnex_bind_check_inuse = 0; 96 int rootnex_unbind_verify_buffer = 0; 97 int rootnex_sync_check_parms = 0; 98 #endif 99 100 boolean_t rootnex_dmar_not_setup; 101 102 /* Master Abort and Target Abort panic flag */ 103 int rootnex_fm_ma_ta_panic_flag = 0; 104 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 106 int rootnex_bind_fail = 1; 107 int rootnex_bind_warn = 1; 108 uint8_t *rootnex_warn_list; 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 110 #define ROOTNEX_BIND_WARNING (0x1 << 0) 111 112 /* 113 * revert back to old broken behavior of always sync'ing entire copy buffer. 114 * This is useful if be have a buggy driver which doesn't correctly pass in 115 * the offset and size into ddi_dma_sync(). 116 */ 117 int rootnex_sync_ignore_params = 0; 118 119 /* 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 125 * (< 8K). We will still need to allocate the copy buffer during bind though 126 * (if we need one). These can only be modified in /etc/system before rootnex 127 * attach. 128 */ 129 #if defined(__amd64) 130 int rootnex_prealloc_cookies = 65; 131 int rootnex_prealloc_windows = 4; 132 int rootnex_prealloc_copybuf = 2; 133 #else 134 int rootnex_prealloc_cookies = 33; 135 int rootnex_prealloc_windows = 4; 136 int rootnex_prealloc_copybuf = 2; 137 #endif 138 139 /* driver global state */ 140 static rootnex_state_t *rootnex_state; 141 142 #ifdef DEBUG 143 /* shortcut to rootnex counters */ 144 static uint64_t *rootnex_cnt; 145 #endif 146 147 /* 148 * XXX - does x86 even need these or are they left over from the SPARC days? 149 */ 150 /* statically defined integer/boolean properties for the root node */ 151 static rootnex_intprop_t rootnex_intprp[] = { 152 { "PAGESIZE", PAGESIZE }, 153 { "MMU_PAGESIZE", MMU_PAGESIZE }, 154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 155 { DDI_RELATIVE_ADDRESSING, 1 }, 156 }; 157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 158 159 /* 160 * If we're dom0, we're using a real device so we need to load 161 * the cookies with MFNs instead of PFNs. 162 */ 163 #ifdef __xpv 164 typedef maddr_t rootnex_addr_t; 165 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 167 #else 168 typedef paddr_t rootnex_addr_t; 169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 170 #endif 171 172 static struct cb_ops rootnex_cb_ops = { 173 nodev, /* open */ 174 nodev, /* close */ 175 nodev, /* strategy */ 176 nodev, /* print */ 177 nodev, /* dump */ 178 nodev, /* read */ 179 nodev, /* write */ 180 nodev, /* ioctl */ 181 nodev, /* devmap */ 182 nodev, /* mmap */ 183 nodev, /* segmap */ 184 nochpoll, /* chpoll */ 185 ddi_prop_op, /* cb_prop_op */ 186 NULL, /* struct streamtab */ 187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 188 CB_REV, /* Rev */ 189 nodev, /* cb_aread */ 190 nodev /* cb_awrite */ 191 }; 192 193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 194 off_t offset, off_t len, caddr_t *vaddrp); 195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 196 struct hat *hat, struct seg *seg, caddr_t addr, 197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 200 ddi_dma_handle_t *handlep); 201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 202 ddi_dma_handle_t handle); 203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 205 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 207 ddi_dma_handle_t handle); 208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 212 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 217 ddi_ctl_enum_t ctlop, void *arg, void *result); 218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 219 ddi_iblock_cookie_t *ibc); 220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 223 void *); 224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 225 226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 228 ddi_dma_handle_t *handlep); 229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 230 ddi_dma_handle_t handle); 231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 233 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 235 ddi_dma_handle_t handle); 236 #if defined(__amd64) && !defined(__xpv) 237 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 238 ddi_dma_handle_t handle); 239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 242 ddi_dma_cookie_t *cookiep, uint_t ccount); 243 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 244 ddi_dma_handle_t handle); 245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 246 #endif 247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 251 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 252 253 #if defined(__amd64) && !defined(__xpv) 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 255 ddi_dma_handle_t handle, void *v); 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 257 ddi_dma_handle_t handle); 258 #endif 259 260 261 static struct bus_ops rootnex_bus_ops = { 262 BUSO_REV, 263 rootnex_map, 264 NULL, 265 NULL, 266 NULL, 267 rootnex_map_fault, 268 0, 269 rootnex_dma_allochdl, 270 rootnex_dma_freehdl, 271 rootnex_dma_bindhdl, 272 rootnex_dma_unbindhdl, 273 rootnex_dma_sync, 274 rootnex_dma_win, 275 rootnex_dma_mctl, 276 rootnex_ctlops, 277 ddi_bus_prop_op, 278 i_ddi_rootnex_get_eventcookie, 279 i_ddi_rootnex_add_eventcall, 280 i_ddi_rootnex_remove_eventcall, 281 i_ddi_rootnex_post_event, 282 0, /* bus_intr_ctl */ 283 0, /* bus_config */ 284 0, /* bus_unconfig */ 285 rootnex_fm_init, /* bus_fm_init */ 286 NULL, /* bus_fm_fini */ 287 NULL, /* bus_fm_access_enter */ 288 NULL, /* bus_fm_access_exit */ 289 NULL, /* bus_powr */ 290 rootnex_intr_ops /* bus_intr_op */ 291 }; 292 293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 295 static int rootnex_quiesce(dev_info_t *dip); 296 297 static struct dev_ops rootnex_ops = { 298 DEVO_REV, 299 0, 300 ddi_no_info, 301 nulldev, 302 nulldev, 303 rootnex_attach, 304 rootnex_detach, 305 nulldev, 306 &rootnex_cb_ops, 307 &rootnex_bus_ops, 308 NULL, 309 rootnex_quiesce, /* quiesce */ 310 }; 311 312 static struct modldrv rootnex_modldrv = { 313 &mod_driverops, 314 "i86pc root nexus", 315 &rootnex_ops 316 }; 317 318 static struct modlinkage rootnex_modlinkage = { 319 MODREV_1, 320 (void *)&rootnex_modldrv, 321 NULL 322 }; 323 324 #if defined(__amd64) && !defined(__xpv) 325 static iommulib_nexops_t iommulib_nexops = { 326 IOMMU_NEXOPS_VERSION, 327 "Rootnex IOMMU ops Vers 1.1", 328 NULL, 329 rootnex_coredma_allochdl, 330 rootnex_coredma_freehdl, 331 rootnex_coredma_bindhdl, 332 rootnex_coredma_unbindhdl, 333 rootnex_coredma_reset_cookies, 334 rootnex_coredma_get_cookies, 335 rootnex_coredma_set_cookies, 336 rootnex_coredma_clear_cookies, 337 rootnex_coredma_get_sleep_flags, 338 rootnex_coredma_sync, 339 rootnex_coredma_win, 340 rootnex_coredma_hdl_setprivate, 341 rootnex_coredma_hdl_getprivate 342 }; 343 #endif 344 345 /* 346 * extern hacks 347 */ 348 extern struct seg_ops segdev_ops; 349 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 350 #ifdef DDI_MAP_DEBUG 351 extern int ddi_map_debug_flag; 352 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 353 #endif 354 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 355 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 356 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 357 psm_intr_op_t, int *); 358 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 359 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 360 361 /* 362 * Use device arena to use for device control register mappings. 363 * Various kernel memory walkers (debugger, dtrace) need to know 364 * to avoid this address range to prevent undesired device activity. 365 */ 366 extern void *device_arena_alloc(size_t size, int vm_flag); 367 extern void device_arena_free(void * vaddr, size_t size); 368 369 370 /* 371 * Internal functions 372 */ 373 static int rootnex_dma_init(); 374 static void rootnex_add_props(dev_info_t *); 375 static int rootnex_ctl_reportdev(dev_info_t *dip); 376 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 377 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 378 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 379 static int rootnex_map_handle(ddi_map_req_t *mp); 380 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 381 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 382 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 383 ddi_dma_attr_t *attr); 384 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 385 rootnex_sglinfo_t *sglinfo); 386 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 387 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 388 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 389 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 390 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 391 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 394 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 395 static void rootnex_teardown_windows(rootnex_dma_t *dma); 396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 397 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 398 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 399 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 400 size_t *copybuf_used, page_t **cur_pp); 401 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 402 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 403 ddi_dma_attr_t *attr, off_t cur_offset); 404 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 405 rootnex_dma_t *dma, rootnex_window_t **windowp, 406 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 407 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 409 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 410 off_t offset, size_t size, uint_t cache_flags); 411 static int rootnex_verify_buffer(rootnex_dma_t *dma); 412 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 413 const void *comp_addr, const void *not_used); 414 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 415 rootnex_sglinfo_t *sglinfo); 416 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 417 418 /* 419 * _init() 420 * 421 */ 422 int 423 _init(void) 424 { 425 426 rootnex_state = NULL; 427 return (mod_install(&rootnex_modlinkage)); 428 } 429 430 431 /* 432 * _info() 433 * 434 */ 435 int 436 _info(struct modinfo *modinfop) 437 { 438 return (mod_info(&rootnex_modlinkage, modinfop)); 439 } 440 441 442 /* 443 * _fini() 444 * 445 */ 446 int 447 _fini(void) 448 { 449 return (EBUSY); 450 } 451 452 453 /* 454 * rootnex_attach() 455 * 456 */ 457 static int 458 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 459 { 460 int fmcap; 461 int e; 462 463 switch (cmd) { 464 case DDI_ATTACH: 465 break; 466 case DDI_RESUME: 467 #if defined(__amd64) && !defined(__xpv) 468 return (immu_unquiesce()); 469 #else 470 return (DDI_SUCCESS); 471 #endif 472 default: 473 return (DDI_FAILURE); 474 } 475 476 /* 477 * We should only have one instance of rootnex. Save it away since we 478 * don't have an easy way to get it back later. 479 */ 480 ASSERT(rootnex_state == NULL); 481 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 482 483 rootnex_state->r_dip = dip; 484 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 485 rootnex_state->r_reserved_msg_printed = B_FALSE; 486 #ifdef DEBUG 487 rootnex_cnt = &rootnex_state->r_counters[0]; 488 #endif 489 490 /* 491 * Set minimum fm capability level for i86pc platforms and then 492 * initialize error handling. Since we're the rootnex, we don't 493 * care what's returned in the fmcap field. 494 */ 495 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 496 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 497 fmcap = ddi_system_fmcap; 498 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 499 500 /* initialize DMA related state */ 501 e = rootnex_dma_init(); 502 if (e != DDI_SUCCESS) { 503 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 504 return (DDI_FAILURE); 505 } 506 507 /* Add static root node properties */ 508 rootnex_add_props(dip); 509 510 /* since we can't call ddi_report_dev() */ 511 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 512 513 /* Initialize rootnex event handle */ 514 i_ddi_rootnex_init_events(dip); 515 516 #if defined(__amd64) && !defined(__xpv) 517 e = iommulib_nexus_register(dip, &iommulib_nexops, 518 &rootnex_state->r_iommulib_handle); 519 520 ASSERT(e == DDI_SUCCESS); 521 #endif 522 523 return (DDI_SUCCESS); 524 } 525 526 527 /* 528 * rootnex_detach() 529 * 530 */ 531 /*ARGSUSED*/ 532 static int 533 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 534 { 535 switch (cmd) { 536 case DDI_SUSPEND: 537 #if defined(__amd64) && !defined(__xpv) 538 return (immu_quiesce()); 539 #else 540 return (DDI_SUCCESS); 541 #endif 542 default: 543 return (DDI_FAILURE); 544 } 545 /*NOTREACHED*/ 546 547 } 548 549 550 /* 551 * rootnex_dma_init() 552 * 553 */ 554 /*ARGSUSED*/ 555 static int 556 rootnex_dma_init() 557 { 558 size_t bufsize; 559 560 561 /* 562 * size of our cookie/window/copybuf state needed in dma bind that we 563 * pre-alloc in dma_alloc_handle 564 */ 565 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 566 rootnex_state->r_prealloc_size = 567 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 568 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 569 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 570 571 /* 572 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 573 * allocate 16 extra bytes for struct pointer alignment 574 * (p->dmai_private & dma->dp_prealloc_buffer) 575 */ 576 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 577 rootnex_state->r_prealloc_size + 0x10; 578 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 579 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 580 if (rootnex_state->r_dmahdl_cache == NULL) { 581 return (DDI_FAILURE); 582 } 583 584 /* 585 * allocate array to track which major numbers we have printed warnings 586 * for. 587 */ 588 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 589 KM_SLEEP); 590 591 return (DDI_SUCCESS); 592 } 593 594 595 /* 596 * rootnex_add_props() 597 * 598 */ 599 static void 600 rootnex_add_props(dev_info_t *dip) 601 { 602 rootnex_intprop_t *rpp; 603 int i; 604 605 /* Add static integer/boolean properties to the root node */ 606 rpp = rootnex_intprp; 607 for (i = 0; i < NROOT_INTPROPS; i++) { 608 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 609 rpp[i].prop_name, rpp[i].prop_value); 610 } 611 } 612 613 614 615 /* 616 * ************************* 617 * ctlops related routines 618 * ************************* 619 */ 620 621 /* 622 * rootnex_ctlops() 623 * 624 */ 625 /*ARGSUSED*/ 626 static int 627 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 628 void *arg, void *result) 629 { 630 int n, *ptr; 631 struct ddi_parent_private_data *pdp; 632 633 switch (ctlop) { 634 case DDI_CTLOPS_DMAPMAPC: 635 /* 636 * Return 'partial' to indicate that dma mapping 637 * has to be done in the main MMU. 638 */ 639 return (DDI_DMA_PARTIAL); 640 641 case DDI_CTLOPS_BTOP: 642 /* 643 * Convert byte count input to physical page units. 644 * (byte counts that are not a page-size multiple 645 * are rounded down) 646 */ 647 *(ulong_t *)result = btop(*(ulong_t *)arg); 648 return (DDI_SUCCESS); 649 650 case DDI_CTLOPS_PTOB: 651 /* 652 * Convert size in physical pages to bytes 653 */ 654 *(ulong_t *)result = ptob(*(ulong_t *)arg); 655 return (DDI_SUCCESS); 656 657 case DDI_CTLOPS_BTOPR: 658 /* 659 * Convert byte count input to physical page units 660 * (byte counts that are not a page-size multiple 661 * are rounded up) 662 */ 663 *(ulong_t *)result = btopr(*(ulong_t *)arg); 664 return (DDI_SUCCESS); 665 666 case DDI_CTLOPS_INITCHILD: 667 return (impl_ddi_sunbus_initchild(arg)); 668 669 case DDI_CTLOPS_UNINITCHILD: 670 impl_ddi_sunbus_removechild(arg); 671 return (DDI_SUCCESS); 672 673 case DDI_CTLOPS_REPORTDEV: 674 return (rootnex_ctl_reportdev(rdip)); 675 676 case DDI_CTLOPS_IOMIN: 677 /* 678 * Nothing to do here but reflect back.. 679 */ 680 return (DDI_SUCCESS); 681 682 case DDI_CTLOPS_REGSIZE: 683 case DDI_CTLOPS_NREGS: 684 break; 685 686 case DDI_CTLOPS_SIDDEV: 687 if (ndi_dev_is_prom_node(rdip)) 688 return (DDI_SUCCESS); 689 if (ndi_dev_is_persistent_node(rdip)) 690 return (DDI_SUCCESS); 691 return (DDI_FAILURE); 692 693 case DDI_CTLOPS_POWER: 694 return ((*pm_platform_power)((power_req_t *)arg)); 695 696 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 697 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 698 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 699 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 700 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 701 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 702 if (!rootnex_state->r_reserved_msg_printed) { 703 rootnex_state->r_reserved_msg_printed = B_TRUE; 704 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 705 "1 or more reserved/obsolete operations."); 706 } 707 return (DDI_FAILURE); 708 709 default: 710 return (DDI_FAILURE); 711 } 712 /* 713 * The rest are for "hardware" properties 714 */ 715 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 716 return (DDI_FAILURE); 717 718 if (ctlop == DDI_CTLOPS_NREGS) { 719 ptr = (int *)result; 720 *ptr = pdp->par_nreg; 721 } else { 722 off_t *size = (off_t *)result; 723 724 ptr = (int *)arg; 725 n = *ptr; 726 if (n >= pdp->par_nreg) { 727 return (DDI_FAILURE); 728 } 729 *size = (off_t)pdp->par_reg[n].regspec_size; 730 } 731 return (DDI_SUCCESS); 732 } 733 734 735 /* 736 * rootnex_ctl_reportdev() 737 * 738 */ 739 static int 740 rootnex_ctl_reportdev(dev_info_t *dev) 741 { 742 int i, n, len, f_len = 0; 743 char *buf; 744 745 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 746 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 747 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 748 len = strlen(buf); 749 750 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 751 752 struct regspec *rp = sparc_pd_getreg(dev, i); 753 754 if (i == 0) 755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 756 ": "); 757 else 758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 759 " and "); 760 len = strlen(buf); 761 762 switch (rp->regspec_bustype) { 763 764 case BTEISA: 765 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 766 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 767 break; 768 769 case BTISA: 770 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 771 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 772 break; 773 774 default: 775 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 776 "space %x offset %x", 777 rp->regspec_bustype, rp->regspec_addr); 778 break; 779 } 780 len = strlen(buf); 781 } 782 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 783 int pri; 784 785 if (i != 0) { 786 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 787 ","); 788 len = strlen(buf); 789 } 790 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 791 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 792 " sparc ipl %d", pri); 793 len = strlen(buf); 794 } 795 #ifdef DEBUG 796 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 797 cmn_err(CE_NOTE, "next message is truncated: " 798 "printed length 1024, real length %d", f_len); 799 } 800 #endif /* DEBUG */ 801 cmn_err(CE_CONT, "?%s\n", buf); 802 kmem_free(buf, REPORTDEV_BUFSIZE); 803 return (DDI_SUCCESS); 804 } 805 806 807 /* 808 * ****************** 809 * map related code 810 * ****************** 811 */ 812 813 /* 814 * rootnex_map() 815 * 816 */ 817 static int 818 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 819 off_t len, caddr_t *vaddrp) 820 { 821 struct regspec *rp, tmp_reg; 822 ddi_map_req_t mr = *mp; /* Get private copy of request */ 823 int error; 824 825 mp = &mr; 826 827 switch (mp->map_op) { 828 case DDI_MO_MAP_LOCKED: 829 case DDI_MO_UNMAP: 830 case DDI_MO_MAP_HANDLE: 831 break; 832 default: 833 #ifdef DDI_MAP_DEBUG 834 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 835 mp->map_op); 836 #endif /* DDI_MAP_DEBUG */ 837 return (DDI_ME_UNIMPLEMENTED); 838 } 839 840 if (mp->map_flags & DDI_MF_USER_MAPPING) { 841 #ifdef DDI_MAP_DEBUG 842 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 843 #endif /* DDI_MAP_DEBUG */ 844 return (DDI_ME_UNIMPLEMENTED); 845 } 846 847 /* 848 * First, if given an rnumber, convert it to a regspec... 849 * (Presumably, this is on behalf of a child of the root node?) 850 */ 851 852 if (mp->map_type == DDI_MT_RNUMBER) { 853 854 int rnumber = mp->map_obj.rnumber; 855 #ifdef DDI_MAP_DEBUG 856 static char *out_of_range = 857 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 858 #endif /* DDI_MAP_DEBUG */ 859 860 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 861 if (rp == NULL) { 862 #ifdef DDI_MAP_DEBUG 863 cmn_err(CE_WARN, out_of_range, rnumber, 864 ddi_get_name(rdip)); 865 #endif /* DDI_MAP_DEBUG */ 866 return (DDI_ME_RNUMBER_RANGE); 867 } 868 869 /* 870 * Convert the given ddi_map_req_t from rnumber to regspec... 871 */ 872 873 mp->map_type = DDI_MT_REGSPEC; 874 mp->map_obj.rp = rp; 875 } 876 877 /* 878 * Adjust offset and length correspnding to called values... 879 * XXX: A non-zero length means override the one in the regspec 880 * XXX: (regardless of what's in the parent's range?) 881 */ 882 883 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 884 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 885 886 #ifdef DDI_MAP_DEBUG 887 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 888 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 889 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 890 len, mp->map_handlep); 891 #endif /* DDI_MAP_DEBUG */ 892 893 /* 894 * I/O or memory mapping: 895 * 896 * <bustype=0, addr=x, len=x>: memory 897 * <bustype=1, addr=x, len=x>: i/o 898 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 899 */ 900 901 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 902 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 903 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 904 ddi_get_name(rdip), rp->regspec_bustype, 905 rp->regspec_addr, rp->regspec_size); 906 return (DDI_ME_INVAL); 907 } 908 909 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 910 /* 911 * compatibility i/o mapping 912 */ 913 rp->regspec_bustype += (uint_t)offset; 914 } else { 915 /* 916 * Normal memory or i/o mapping 917 */ 918 rp->regspec_addr += (uint_t)offset; 919 } 920 921 if (len != 0) 922 rp->regspec_size = (uint_t)len; 923 924 #ifdef DDI_MAP_DEBUG 925 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 926 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 927 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 928 offset, len, mp->map_handlep); 929 #endif /* DDI_MAP_DEBUG */ 930 931 /* 932 * Apply any parent ranges at this level, if applicable. 933 * (This is where nexus specific regspec translation takes place. 934 * Use of this function is implicit agreement that translation is 935 * provided via ddi_apply_range.) 936 */ 937 938 #ifdef DDI_MAP_DEBUG 939 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 940 ddi_get_name(dip), ddi_get_name(rdip)); 941 #endif /* DDI_MAP_DEBUG */ 942 943 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 944 return (error); 945 946 switch (mp->map_op) { 947 case DDI_MO_MAP_LOCKED: 948 949 /* 950 * Set up the locked down kernel mapping to the regspec... 951 */ 952 953 return (rootnex_map_regspec(mp, vaddrp)); 954 955 case DDI_MO_UNMAP: 956 957 /* 958 * Release mapping... 959 */ 960 961 return (rootnex_unmap_regspec(mp, vaddrp)); 962 963 case DDI_MO_MAP_HANDLE: 964 965 return (rootnex_map_handle(mp)); 966 967 default: 968 return (DDI_ME_UNIMPLEMENTED); 969 } 970 } 971 972 973 /* 974 * rootnex_map_fault() 975 * 976 * fault in mappings for requestors 977 */ 978 /*ARGSUSED*/ 979 static int 980 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 981 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 982 uint_t lock) 983 { 984 985 #ifdef DDI_MAP_DEBUG 986 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 987 ddi_map_debug(" Seg <%s>\n", 988 seg->s_ops == &segdev_ops ? "segdev" : 989 seg == &kvseg ? "segkmem" : "NONE!"); 990 #endif /* DDI_MAP_DEBUG */ 991 992 /* 993 * This is all terribly broken, but it is a start 994 * 995 * XXX Note that this test means that segdev_ops 996 * must be exported from seg_dev.c. 997 * XXX What about devices with their own segment drivers? 998 */ 999 if (seg->s_ops == &segdev_ops) { 1000 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1001 1002 if (hat == NULL) { 1003 /* 1004 * This is one plausible interpretation of 1005 * a null hat i.e. use the first hat on the 1006 * address space hat list which by convention is 1007 * the hat of the system MMU. At alternative 1008 * would be to panic .. this might well be better .. 1009 */ 1010 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1011 hat = seg->s_as->a_hat; 1012 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1013 } 1014 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1015 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1016 } else if (seg == &kvseg && dp == NULL) { 1017 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1018 HAT_LOAD_LOCK); 1019 } else 1020 return (DDI_FAILURE); 1021 return (DDI_SUCCESS); 1022 } 1023 1024 1025 /* 1026 * rootnex_map_regspec() 1027 * we don't support mapping of I/O cards above 4Gb 1028 */ 1029 static int 1030 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1031 { 1032 rootnex_addr_t rbase; 1033 void *cvaddr; 1034 uint_t npages, pgoffset; 1035 struct regspec *rp; 1036 ddi_acc_hdl_t *hp; 1037 ddi_acc_impl_t *ap; 1038 uint_t hat_acc_flags; 1039 paddr_t pbase; 1040 1041 rp = mp->map_obj.rp; 1042 hp = mp->map_handlep; 1043 1044 #ifdef DDI_MAP_DEBUG 1045 ddi_map_debug( 1046 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1047 rp->regspec_bustype, rp->regspec_addr, 1048 rp->regspec_size, mp->map_handlep); 1049 #endif /* DDI_MAP_DEBUG */ 1050 1051 /* 1052 * I/O or memory mapping 1053 * 1054 * <bustype=0, addr=x, len=x>: memory 1055 * <bustype=1, addr=x, len=x>: i/o 1056 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1057 */ 1058 1059 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1060 cmn_err(CE_WARN, "rootnex: invalid register spec" 1061 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1062 rp->regspec_addr, rp->regspec_size); 1063 return (DDI_FAILURE); 1064 } 1065 1066 if (rp->regspec_bustype != 0) { 1067 /* 1068 * I/O space - needs a handle. 1069 */ 1070 if (hp == NULL) { 1071 return (DDI_FAILURE); 1072 } 1073 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1074 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1075 impl_acc_hdl_init(hp); 1076 1077 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1078 #ifdef DDI_MAP_DEBUG 1079 ddi_map_debug("rootnex_map_regspec: mmap() " 1080 "to I/O space is not supported.\n"); 1081 #endif /* DDI_MAP_DEBUG */ 1082 return (DDI_ME_INVAL); 1083 } else { 1084 /* 1085 * 1275-compliant vs. compatibility i/o mapping 1086 */ 1087 *vaddrp = 1088 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1089 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1090 ((caddr_t)(uintptr_t)rp->regspec_addr); 1091 #ifdef __xpv 1092 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1093 hp->ah_pfn = xen_assign_pfn( 1094 mmu_btop((ulong_t)rp->regspec_addr & 1095 MMU_PAGEMASK)); 1096 } else { 1097 hp->ah_pfn = mmu_btop( 1098 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1099 } 1100 #else 1101 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1102 MMU_PAGEMASK); 1103 #endif 1104 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1105 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1106 } 1107 1108 #ifdef DDI_MAP_DEBUG 1109 ddi_map_debug( 1110 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1111 rp->regspec_size, *vaddrp); 1112 #endif /* DDI_MAP_DEBUG */ 1113 return (DDI_SUCCESS); 1114 } 1115 1116 /* 1117 * Memory space 1118 */ 1119 1120 if (hp != NULL) { 1121 /* 1122 * hat layer ignores 1123 * hp->ah_acc.devacc_attr_endian_flags. 1124 */ 1125 switch (hp->ah_acc.devacc_attr_dataorder) { 1126 case DDI_STRICTORDER_ACC: 1127 hat_acc_flags = HAT_STRICTORDER; 1128 break; 1129 case DDI_UNORDERED_OK_ACC: 1130 hat_acc_flags = HAT_UNORDERED_OK; 1131 break; 1132 case DDI_MERGING_OK_ACC: 1133 hat_acc_flags = HAT_MERGING_OK; 1134 break; 1135 case DDI_LOADCACHING_OK_ACC: 1136 hat_acc_flags = HAT_LOADCACHING_OK; 1137 break; 1138 case DDI_STORECACHING_OK_ACC: 1139 hat_acc_flags = HAT_STORECACHING_OK; 1140 break; 1141 } 1142 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1143 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1144 impl_acc_hdl_init(hp); 1145 hp->ah_hat_flags = hat_acc_flags; 1146 } else { 1147 hat_acc_flags = HAT_STRICTORDER; 1148 } 1149 1150 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1151 #ifdef __xpv 1152 /* 1153 * If we're dom0, we're using a real device so we need to translate 1154 * the MA to a PA. 1155 */ 1156 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1157 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1158 } else { 1159 pbase = rbase; 1160 } 1161 #else 1162 pbase = rbase; 1163 #endif 1164 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1165 1166 if (rp->regspec_size == 0) { 1167 #ifdef DDI_MAP_DEBUG 1168 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1169 #endif /* DDI_MAP_DEBUG */ 1170 return (DDI_ME_INVAL); 1171 } 1172 1173 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1174 /* extra cast to make gcc happy */ 1175 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1176 } else { 1177 npages = mmu_btopr(rp->regspec_size + pgoffset); 1178 1179 #ifdef DDI_MAP_DEBUG 1180 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1181 "physical %llx", npages, pbase); 1182 #endif /* DDI_MAP_DEBUG */ 1183 1184 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1185 if (cvaddr == NULL) 1186 return (DDI_ME_NORESOURCES); 1187 1188 /* 1189 * Now map in the pages we've allocated... 1190 */ 1191 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1192 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1193 HAT_LOAD_LOCK); 1194 *vaddrp = (caddr_t)cvaddr + pgoffset; 1195 1196 /* save away pfn and npages for FMA */ 1197 hp = mp->map_handlep; 1198 if (hp) { 1199 hp->ah_pfn = mmu_btop(pbase); 1200 hp->ah_pnum = npages; 1201 } 1202 } 1203 1204 #ifdef DDI_MAP_DEBUG 1205 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1206 #endif /* DDI_MAP_DEBUG */ 1207 return (DDI_SUCCESS); 1208 } 1209 1210 1211 /* 1212 * rootnex_unmap_regspec() 1213 * 1214 */ 1215 static int 1216 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1217 { 1218 caddr_t addr = (caddr_t)*vaddrp; 1219 uint_t npages, pgoffset; 1220 struct regspec *rp; 1221 1222 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1223 return (0); 1224 1225 rp = mp->map_obj.rp; 1226 1227 if (rp->regspec_size == 0) { 1228 #ifdef DDI_MAP_DEBUG 1229 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1230 #endif /* DDI_MAP_DEBUG */ 1231 return (DDI_ME_INVAL); 1232 } 1233 1234 /* 1235 * I/O or memory mapping: 1236 * 1237 * <bustype=0, addr=x, len=x>: memory 1238 * <bustype=1, addr=x, len=x>: i/o 1239 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1240 */ 1241 if (rp->regspec_bustype != 0) { 1242 /* 1243 * This is I/O space, which requires no particular 1244 * processing on unmap since it isn't mapped in the 1245 * first place. 1246 */ 1247 return (DDI_SUCCESS); 1248 } 1249 1250 /* 1251 * Memory space 1252 */ 1253 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1254 npages = mmu_btopr(rp->regspec_size + pgoffset); 1255 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1256 device_arena_free(addr - pgoffset, ptob(npages)); 1257 1258 /* 1259 * Destroy the pointer - the mapping has logically gone 1260 */ 1261 *vaddrp = NULL; 1262 1263 return (DDI_SUCCESS); 1264 } 1265 1266 1267 /* 1268 * rootnex_map_handle() 1269 * 1270 */ 1271 static int 1272 rootnex_map_handle(ddi_map_req_t *mp) 1273 { 1274 rootnex_addr_t rbase; 1275 ddi_acc_hdl_t *hp; 1276 uint_t pgoffset; 1277 struct regspec *rp; 1278 paddr_t pbase; 1279 1280 rp = mp->map_obj.rp; 1281 1282 #ifdef DDI_MAP_DEBUG 1283 ddi_map_debug( 1284 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1285 rp->regspec_bustype, rp->regspec_addr, 1286 rp->regspec_size, mp->map_handlep); 1287 #endif /* DDI_MAP_DEBUG */ 1288 1289 /* 1290 * I/O or memory mapping: 1291 * 1292 * <bustype=0, addr=x, len=x>: memory 1293 * <bustype=1, addr=x, len=x>: i/o 1294 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1295 */ 1296 if (rp->regspec_bustype != 0) { 1297 /* 1298 * This refers to I/O space, and we don't support "mapping" 1299 * I/O space to a user. 1300 */ 1301 return (DDI_FAILURE); 1302 } 1303 1304 /* 1305 * Set up the hat_flags for the mapping. 1306 */ 1307 hp = mp->map_handlep; 1308 1309 switch (hp->ah_acc.devacc_attr_endian_flags) { 1310 case DDI_NEVERSWAP_ACC: 1311 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1312 break; 1313 case DDI_STRUCTURE_LE_ACC: 1314 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1315 break; 1316 case DDI_STRUCTURE_BE_ACC: 1317 return (DDI_FAILURE); 1318 default: 1319 return (DDI_REGS_ACC_CONFLICT); 1320 } 1321 1322 switch (hp->ah_acc.devacc_attr_dataorder) { 1323 case DDI_STRICTORDER_ACC: 1324 break; 1325 case DDI_UNORDERED_OK_ACC: 1326 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1327 break; 1328 case DDI_MERGING_OK_ACC: 1329 hp->ah_hat_flags |= HAT_MERGING_OK; 1330 break; 1331 case DDI_LOADCACHING_OK_ACC: 1332 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1333 break; 1334 case DDI_STORECACHING_OK_ACC: 1335 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1336 break; 1337 default: 1338 return (DDI_FAILURE); 1339 } 1340 1341 rbase = (rootnex_addr_t)rp->regspec_addr & 1342 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1343 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1344 1345 if (rp->regspec_size == 0) 1346 return (DDI_ME_INVAL); 1347 1348 #ifdef __xpv 1349 /* 1350 * If we're dom0, we're using a real device so we need to translate 1351 * the MA to a PA. 1352 */ 1353 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1354 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1355 (rbase & MMU_PAGEOFFSET); 1356 } else { 1357 pbase = rbase; 1358 } 1359 #else 1360 pbase = rbase; 1361 #endif 1362 1363 hp->ah_pfn = mmu_btop(pbase); 1364 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1365 1366 return (DDI_SUCCESS); 1367 } 1368 1369 1370 1371 /* 1372 * ************************ 1373 * interrupt related code 1374 * ************************ 1375 */ 1376 1377 /* 1378 * rootnex_intr_ops() 1379 * bus_intr_op() function for interrupt support 1380 */ 1381 /* ARGSUSED */ 1382 static int 1383 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1384 ddi_intr_handle_impl_t *hdlp, void *result) 1385 { 1386 struct intrspec *ispec; 1387 1388 DDI_INTR_NEXDBG((CE_CONT, 1389 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1390 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1391 1392 /* Process the interrupt operation */ 1393 switch (intr_op) { 1394 case DDI_INTROP_GETCAP: 1395 /* First check with pcplusmp */ 1396 if (psm_intr_ops == NULL) 1397 return (DDI_FAILURE); 1398 1399 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1400 *(int *)result = 0; 1401 return (DDI_FAILURE); 1402 } 1403 break; 1404 case DDI_INTROP_SETCAP: 1405 if (psm_intr_ops == NULL) 1406 return (DDI_FAILURE); 1407 1408 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1409 return (DDI_FAILURE); 1410 break; 1411 case DDI_INTROP_ALLOC: 1412 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1413 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1414 case DDI_INTROP_FREE: 1415 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1416 return (rootnex_free_intr_fixed(rdip, hdlp)); 1417 case DDI_INTROP_GETPRI: 1418 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1419 return (DDI_FAILURE); 1420 *(int *)result = ispec->intrspec_pri; 1421 break; 1422 case DDI_INTROP_SETPRI: 1423 /* Validate the interrupt priority passed to us */ 1424 if (*(int *)result > LOCK_LEVEL) 1425 return (DDI_FAILURE); 1426 1427 /* Ensure that PSM is all initialized and ispec is ok */ 1428 if ((psm_intr_ops == NULL) || 1429 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1430 return (DDI_FAILURE); 1431 1432 /* Change the priority */ 1433 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1434 PSM_FAILURE) 1435 return (DDI_FAILURE); 1436 1437 /* update the ispec with the new priority */ 1438 ispec->intrspec_pri = *(int *)result; 1439 break; 1440 case DDI_INTROP_ADDISR: 1441 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1442 return (DDI_FAILURE); 1443 ispec->intrspec_func = hdlp->ih_cb_func; 1444 break; 1445 case DDI_INTROP_REMISR: 1446 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1447 return (DDI_FAILURE); 1448 ispec->intrspec_func = (uint_t (*)()) 0; 1449 break; 1450 case DDI_INTROP_ENABLE: 1451 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1452 return (DDI_FAILURE); 1453 1454 /* Call psmi to translate irq with the dip */ 1455 if (psm_intr_ops == NULL) 1456 return (DDI_FAILURE); 1457 1458 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1459 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1460 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1461 return (DDI_FAILURE); 1462 1463 /* Add the interrupt handler */ 1464 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1465 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1466 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1467 return (DDI_FAILURE); 1468 break; 1469 case DDI_INTROP_DISABLE: 1470 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1471 return (DDI_FAILURE); 1472 1473 /* Call psm_ops() to translate irq with the dip */ 1474 if (psm_intr_ops == NULL) 1475 return (DDI_FAILURE); 1476 1477 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1478 (void) (*psm_intr_ops)(rdip, hdlp, 1479 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1480 1481 /* Remove the interrupt handler */ 1482 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1483 hdlp->ih_cb_func, hdlp->ih_vector); 1484 break; 1485 case DDI_INTROP_SETMASK: 1486 if (psm_intr_ops == NULL) 1487 return (DDI_FAILURE); 1488 1489 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1490 return (DDI_FAILURE); 1491 break; 1492 case DDI_INTROP_CLRMASK: 1493 if (psm_intr_ops == NULL) 1494 return (DDI_FAILURE); 1495 1496 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1497 return (DDI_FAILURE); 1498 break; 1499 case DDI_INTROP_GETPENDING: 1500 if (psm_intr_ops == NULL) 1501 return (DDI_FAILURE); 1502 1503 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1504 result)) { 1505 *(int *)result = 0; 1506 return (DDI_FAILURE); 1507 } 1508 break; 1509 case DDI_INTROP_NAVAIL: 1510 case DDI_INTROP_NINTRS: 1511 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1512 if (*(int *)result == 0) { 1513 /* 1514 * Special case for 'pcic' driver' only. This driver 1515 * driver is a child of 'isa' and 'rootnex' drivers. 1516 * 1517 * See detailed comments on this in the function 1518 * rootnex_get_ispec(). 1519 * 1520 * Children of 'pcic' send 'NINITR' request all the 1521 * way to rootnex driver. But, the 'pdp->par_nintr' 1522 * field may not initialized. So, we fake it here 1523 * to return 1 (a la what PCMCIA nexus does). 1524 */ 1525 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1526 *(int *)result = 1; 1527 else 1528 return (DDI_FAILURE); 1529 } 1530 break; 1531 case DDI_INTROP_SUPPORTED_TYPES: 1532 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1533 break; 1534 default: 1535 return (DDI_FAILURE); 1536 } 1537 1538 return (DDI_SUCCESS); 1539 } 1540 1541 1542 /* 1543 * rootnex_get_ispec() 1544 * convert an interrupt number to an interrupt specification. 1545 * The interrupt number determines which interrupt spec will be 1546 * returned if more than one exists. 1547 * 1548 * Look into the parent private data area of the 'rdip' to find out 1549 * the interrupt specification. First check to make sure there is 1550 * one that matchs "inumber" and then return a pointer to it. 1551 * 1552 * Return NULL if one could not be found. 1553 * 1554 * NOTE: This is needed for rootnex_intr_ops() 1555 */ 1556 static struct intrspec * 1557 rootnex_get_ispec(dev_info_t *rdip, int inum) 1558 { 1559 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1560 1561 /* 1562 * Special case handling for drivers that provide their own 1563 * intrspec structures instead of relying on the DDI framework. 1564 * 1565 * A broken hardware driver in ON could potentially provide its 1566 * own intrspec structure, instead of relying on the hardware. 1567 * If these drivers are children of 'rootnex' then we need to 1568 * continue to provide backward compatibility to them here. 1569 * 1570 * Following check is a special case for 'pcic' driver which 1571 * was found to have broken hardwre andby provides its own intrspec. 1572 * 1573 * Verbatim comments from this driver are shown here: 1574 * "Don't use the ddi_add_intr since we don't have a 1575 * default intrspec in all cases." 1576 * 1577 * Since an 'ispec' may not be always created for it, 1578 * check for that and create one if so. 1579 * 1580 * NOTE: Currently 'pcic' is the only driver found to do this. 1581 */ 1582 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1583 pdp->par_nintr = 1; 1584 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1585 pdp->par_nintr, KM_SLEEP); 1586 } 1587 1588 /* Validate the interrupt number */ 1589 if (inum >= pdp->par_nintr) 1590 return (NULL); 1591 1592 /* Get the interrupt structure pointer and return that */ 1593 return ((struct intrspec *)&pdp->par_intr[inum]); 1594 } 1595 1596 /* 1597 * Allocate interrupt vector for FIXED (legacy) type. 1598 */ 1599 static int 1600 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1601 void *result) 1602 { 1603 struct intrspec *ispec; 1604 ddi_intr_handle_impl_t info_hdl; 1605 int ret; 1606 int free_phdl = 0; 1607 apic_get_type_t type_info; 1608 1609 if (psm_intr_ops == NULL) 1610 return (DDI_FAILURE); 1611 1612 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1613 return (DDI_FAILURE); 1614 1615 /* 1616 * If the PSM module is "APIX" then pass the request for it 1617 * to allocate the vector now. 1618 */ 1619 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1620 info_hdl.ih_private = &type_info; 1621 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1622 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1623 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1624 free_phdl = 1; 1625 i_ddi_alloc_intr_phdl(hdlp); 1626 } 1627 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1628 ret = (*psm_intr_ops)(rdip, hdlp, 1629 PSM_INTR_OP_ALLOC_VECTORS, result); 1630 if (free_phdl) { /* free up the phdl structure */ 1631 free_phdl = 0; 1632 i_ddi_free_intr_phdl(hdlp); 1633 hdlp->ih_private = NULL; 1634 } 1635 } else { 1636 /* 1637 * No APIX module; fall back to the old scheme where the 1638 * interrupt vector is allocated during ddi_enable_intr() call. 1639 */ 1640 hdlp->ih_pri = ispec->intrspec_pri; 1641 *(int *)result = hdlp->ih_scratch1; 1642 ret = DDI_SUCCESS; 1643 } 1644 1645 return (ret); 1646 } 1647 1648 /* 1649 * Free up interrupt vector for FIXED (legacy) type. 1650 */ 1651 static int 1652 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1653 { 1654 struct intrspec *ispec; 1655 struct ddi_parent_private_data *pdp; 1656 ddi_intr_handle_impl_t info_hdl; 1657 int ret; 1658 apic_get_type_t type_info; 1659 1660 if (psm_intr_ops == NULL) 1661 return (DDI_FAILURE); 1662 1663 /* 1664 * If the PSM module is "APIX" then pass the request for it 1665 * to free up the vector now. 1666 */ 1667 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1668 info_hdl.ih_private = &type_info; 1669 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1670 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1671 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1672 return (DDI_FAILURE); 1673 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1674 ret = (*psm_intr_ops)(rdip, hdlp, 1675 PSM_INTR_OP_FREE_VECTORS, NULL); 1676 } else { 1677 /* 1678 * No APIX module; fall back to the old scheme where 1679 * the interrupt vector was already freed during 1680 * ddi_disable_intr() call. 1681 */ 1682 ret = DDI_SUCCESS; 1683 } 1684 1685 pdp = ddi_get_parent_data(rdip); 1686 1687 /* 1688 * Special case for 'pcic' driver' only. 1689 * If an intrspec was created for it, clean it up here 1690 * See detailed comments on this in the function 1691 * rootnex_get_ispec(). 1692 */ 1693 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1694 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1695 pdp->par_nintr); 1696 /* 1697 * Set it to zero; so that 1698 * DDI framework doesn't free it again 1699 */ 1700 pdp->par_intr = NULL; 1701 pdp->par_nintr = 0; 1702 } 1703 1704 return (ret); 1705 } 1706 1707 1708 /* 1709 * ****************** 1710 * dma related code 1711 * ****************** 1712 */ 1713 1714 /*ARGSUSED*/ 1715 static int 1716 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1717 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1718 ddi_dma_handle_t *handlep) 1719 { 1720 uint64_t maxsegmentsize_ll; 1721 uint_t maxsegmentsize; 1722 ddi_dma_impl_t *hp; 1723 rootnex_dma_t *dma; 1724 uint64_t count_max; 1725 uint64_t seg; 1726 int kmflag; 1727 int e; 1728 1729 1730 /* convert our sleep flags */ 1731 if (waitfp == DDI_DMA_SLEEP) { 1732 kmflag = KM_SLEEP; 1733 } else { 1734 kmflag = KM_NOSLEEP; 1735 } 1736 1737 /* 1738 * We try to do only one memory allocation here. We'll do a little 1739 * pointer manipulation later. If the bind ends up taking more than 1740 * our prealloc's space, we'll have to allocate more memory in the 1741 * bind operation. Not great, but much better than before and the 1742 * best we can do with the current bind interfaces. 1743 */ 1744 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1745 if (hp == NULL) 1746 return (DDI_DMA_NORESOURCES); 1747 1748 /* Do our pointer manipulation now, align the structures */ 1749 hp->dmai_private = (void *)(((uintptr_t)hp + 1750 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1751 dma = (rootnex_dma_t *)hp->dmai_private; 1752 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1753 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1754 1755 /* setup the handle */ 1756 rootnex_clean_dmahdl(hp); 1757 hp->dmai_error.err_fep = NULL; 1758 hp->dmai_error.err_cf = NULL; 1759 dma->dp_dip = rdip; 1760 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1761 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1762 1763 /* 1764 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1765 * is being used. Set the upper limit to the seg value. 1766 * There will be enough DVMA space to always get addresses 1767 * that will match the constraints. 1768 */ 1769 if (IOMMU_USED(rdip) && 1770 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1771 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1772 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1773 } else 1774 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1775 1776 hp->dmai_minxfer = attr->dma_attr_minxfer; 1777 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1778 hp->dmai_rdip = rdip; 1779 hp->dmai_attr = *attr; 1780 1781 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1782 dma->dp_sglinfo.si_cancross = B_FALSE; 1783 else 1784 dma->dp_sglinfo.si_cancross = B_TRUE; 1785 1786 /* we don't need to worry about the SPL since we do a tryenter */ 1787 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1788 1789 /* 1790 * Figure out our maximum segment size. If the segment size is greater 1791 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1792 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1793 * dma_attr_count_max are size-1 type values. 1794 * 1795 * Maximum segment size is the largest physically contiguous chunk of 1796 * memory that we can return from a bind (i.e. the maximum size of a 1797 * single cookie). 1798 */ 1799 1800 /* handle the rollover cases */ 1801 seg = attr->dma_attr_seg + 1; 1802 if (seg < attr->dma_attr_seg) { 1803 seg = attr->dma_attr_seg; 1804 } 1805 count_max = attr->dma_attr_count_max + 1; 1806 if (count_max < attr->dma_attr_count_max) { 1807 count_max = attr->dma_attr_count_max; 1808 } 1809 1810 /* 1811 * granularity may or may not be a power of two. If it isn't, we can't 1812 * use a simple mask. 1813 */ 1814 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1815 dma->dp_granularity_power_2 = B_FALSE; 1816 } else { 1817 dma->dp_granularity_power_2 = B_TRUE; 1818 } 1819 1820 /* 1821 * maxxfer should be a whole multiple of granularity. If we're going to 1822 * break up a window because we're greater than maxxfer, we might as 1823 * well make sure it's maxxfer is a whole multiple so we don't have to 1824 * worry about triming the window later on for this case. 1825 */ 1826 if (attr->dma_attr_granular > 1) { 1827 if (dma->dp_granularity_power_2) { 1828 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1829 (attr->dma_attr_maxxfer & 1830 (attr->dma_attr_granular - 1)); 1831 } else { 1832 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1833 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1834 } 1835 } else { 1836 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1837 } 1838 1839 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1840 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1841 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1842 maxsegmentsize = 0xFFFFFFFF; 1843 } else { 1844 maxsegmentsize = maxsegmentsize_ll; 1845 } 1846 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1847 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1848 1849 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1850 if (rootnex_alloc_check_parms) { 1851 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1852 if (e != DDI_SUCCESS) { 1853 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1854 (void) rootnex_dma_freehdl(dip, rdip, 1855 (ddi_dma_handle_t)hp); 1856 return (e); 1857 } 1858 } 1859 1860 *handlep = (ddi_dma_handle_t)hp; 1861 1862 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1863 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1864 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1865 1866 return (DDI_SUCCESS); 1867 } 1868 1869 1870 /* 1871 * rootnex_dma_allochdl() 1872 * called from ddi_dma_alloc_handle(). 1873 */ 1874 static int 1875 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1876 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1877 { 1878 int retval = DDI_SUCCESS; 1879 #if defined(__amd64) && !defined(__xpv) 1880 1881 if (IOMMU_UNITIALIZED(rdip)) { 1882 retval = iommulib_nex_open(dip, rdip); 1883 1884 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1885 return (retval); 1886 } 1887 1888 if (IOMMU_UNUSED(rdip)) { 1889 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1890 handlep); 1891 } else { 1892 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1893 waitfp, arg, handlep); 1894 } 1895 #else 1896 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1897 handlep); 1898 #endif 1899 switch (retval) { 1900 case DDI_DMA_NORESOURCES: 1901 if (waitfp != DDI_DMA_DONTWAIT) { 1902 ddi_set_callback(waitfp, arg, 1903 &rootnex_state->r_dvma_call_list_id); 1904 } 1905 break; 1906 case DDI_SUCCESS: 1907 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1908 break; 1909 default: 1910 break; 1911 } 1912 return (retval); 1913 } 1914 1915 /*ARGSUSED*/ 1916 static int 1917 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1918 ddi_dma_handle_t handle) 1919 { 1920 ddi_dma_impl_t *hp; 1921 rootnex_dma_t *dma; 1922 1923 1924 hp = (ddi_dma_impl_t *)handle; 1925 dma = (rootnex_dma_t *)hp->dmai_private; 1926 1927 /* unbind should have been called first */ 1928 ASSERT(!dma->dp_inuse); 1929 1930 mutex_destroy(&dma->dp_mutex); 1931 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1932 1933 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1934 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1935 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1936 1937 return (DDI_SUCCESS); 1938 } 1939 1940 /* 1941 * rootnex_dma_freehdl() 1942 * called from ddi_dma_free_handle(). 1943 */ 1944 static int 1945 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1946 { 1947 int ret; 1948 1949 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1950 #if defined(__amd64) && !defined(__xpv) 1951 if (IOMMU_USED(rdip)) 1952 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1953 else 1954 #endif 1955 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1956 1957 if (rootnex_state->r_dvma_call_list_id) 1958 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1959 1960 return (ret); 1961 } 1962 1963 /*ARGSUSED*/ 1964 static int 1965 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1966 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1967 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1968 { 1969 rootnex_sglinfo_t *sinfo; 1970 ddi_dma_obj_t *dmao; 1971 #if defined(__amd64) && !defined(__xpv) 1972 struct dvmaseg *dvs; 1973 ddi_dma_cookie_t *cookie; 1974 #endif 1975 ddi_dma_attr_t *attr; 1976 ddi_dma_impl_t *hp; 1977 rootnex_dma_t *dma; 1978 int kmflag; 1979 int e; 1980 uint_t ncookies; 1981 1982 hp = (ddi_dma_impl_t *)handle; 1983 dma = (rootnex_dma_t *)hp->dmai_private; 1984 dmao = &dma->dp_dma; 1985 sinfo = &dma->dp_sglinfo; 1986 attr = &hp->dmai_attr; 1987 1988 /* convert the sleep flags */ 1989 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1990 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1991 } else { 1992 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1993 } 1994 1995 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1996 1997 /* 1998 * This is useful for debugging a driver. Not as useful in a production 1999 * system. The only time this will fail is if you have a driver bug. 2000 */ 2001 if (rootnex_bind_check_inuse) { 2002 /* 2003 * No one else should ever have this lock unless someone else 2004 * is trying to use this handle. So contention on the lock 2005 * is the same as inuse being set. 2006 */ 2007 e = mutex_tryenter(&dma->dp_mutex); 2008 if (e == 0) { 2009 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2010 return (DDI_DMA_INUSE); 2011 } 2012 if (dma->dp_inuse) { 2013 mutex_exit(&dma->dp_mutex); 2014 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2015 return (DDI_DMA_INUSE); 2016 } 2017 dma->dp_inuse = B_TRUE; 2018 mutex_exit(&dma->dp_mutex); 2019 } 2020 2021 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2022 if (rootnex_bind_check_parms) { 2023 e = rootnex_valid_bind_parms(dmareq, attr); 2024 if (e != DDI_SUCCESS) { 2025 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2026 rootnex_clean_dmahdl(hp); 2027 return (e); 2028 } 2029 } 2030 2031 /* save away the original bind info */ 2032 dma->dp_dma = dmareq->dmar_object; 2033 2034 #if defined(__amd64) && !defined(__xpv) 2035 if (IOMMU_USED(rdip)) { 2036 dmao = &dma->dp_dvma; 2037 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2038 switch (e) { 2039 case DDI_SUCCESS: 2040 if (sinfo->si_cancross || 2041 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2042 dmao->dmao_size > sinfo->si_max_cookie_size) { 2043 dma->dp_dvma_used = B_TRUE; 2044 break; 2045 } 2046 sinfo->si_sgl_size = 1; 2047 hp->dmai_rflags |= DMP_NOSYNC; 2048 2049 dma->dp_dvma_used = B_TRUE; 2050 dma->dp_need_to_free_cookie = B_FALSE; 2051 2052 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2053 cookie = hp->dmai_cookie = dma->dp_cookies = 2054 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2055 cookie->dmac_laddress = dvs->dvs_start + 2056 dmao->dmao_obj.dvma_obj.dv_off; 2057 cookie->dmac_size = dvs->dvs_len; 2058 cookie->dmac_type = 0; 2059 2060 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2061 rdip); 2062 goto fast; 2063 case DDI_ENOTSUP: 2064 break; 2065 default: 2066 rootnex_clean_dmahdl(hp); 2067 return (e); 2068 } 2069 } 2070 #endif 2071 2072 /* 2073 * Figure out a rough estimate of what maximum number of pages 2074 * this buffer could use (a high estimate of course). 2075 */ 2076 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2077 2078 if (dma->dp_dvma_used) { 2079 /* 2080 * The number of physical pages is the worst case. 2081 * 2082 * For DVMA, the worst case is the length divided 2083 * by the maximum cookie length, plus 1. Add to that 2084 * the number of segment boundaries potentially crossed, and 2085 * the additional number of DVMA segments that was returned. 2086 * 2087 * In the normal case, for modern devices, si_cancross will 2088 * be false, and dv_nseg will be 1, and the fast path will 2089 * have been taken above. 2090 */ 2091 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2092 + 1; 2093 if (sinfo->si_cancross) 2094 ncookies += 2095 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2096 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2097 2098 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2099 } 2100 2101 /* 2102 * We'll use the pre-allocated cookies for any bind that will *always* 2103 * fit (more important to be consistent, we don't want to create 2104 * additional degenerate cases). 2105 */ 2106 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2107 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2108 dma->dp_need_to_free_cookie = B_FALSE; 2109 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2110 uint_t, sinfo->si_max_pages); 2111 2112 /* 2113 * For anything larger than that, we'll go ahead and allocate the 2114 * maximum number of pages we expect to see. Hopefuly, we won't be 2115 * seeing this path in the fast path for high performance devices very 2116 * frequently. 2117 * 2118 * a ddi bind interface that allowed the driver to provide storage to 2119 * the bind interface would speed this case up. 2120 */ 2121 } else { 2122 /* 2123 * Save away how much memory we allocated. If we're doing a 2124 * nosleep, the alloc could fail... 2125 */ 2126 dma->dp_cookie_size = sinfo->si_max_pages * 2127 sizeof (ddi_dma_cookie_t); 2128 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2129 if (dma->dp_cookies == NULL) { 2130 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2131 rootnex_clean_dmahdl(hp); 2132 return (DDI_DMA_NORESOURCES); 2133 } 2134 dma->dp_need_to_free_cookie = B_TRUE; 2135 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2136 uint_t, sinfo->si_max_pages); 2137 } 2138 hp->dmai_cookie = dma->dp_cookies; 2139 2140 /* 2141 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2142 * looking at the constraints in the dma structure. It will then put 2143 * some additional state about the sgl in the dma struct (i.e. is 2144 * the sgl clean, or do we need to do some munging; how many pages 2145 * need to be copied, etc.) 2146 */ 2147 if (dma->dp_dvma_used) 2148 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2149 else 2150 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2151 2152 out: 2153 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2154 /* if we don't need a copy buffer, we don't need to sync */ 2155 if (sinfo->si_copybuf_req == 0) { 2156 hp->dmai_rflags |= DMP_NOSYNC; 2157 } 2158 2159 /* 2160 * if we don't need the copybuf and we don't need to do a partial, we 2161 * hit the fast path. All the high performance devices should be trying 2162 * to hit this path. To hit this path, a device should be able to reach 2163 * all of memory, shouldn't try to bind more than it can transfer, and 2164 * the buffer shouldn't require more cookies than the driver/device can 2165 * handle [sgllen]). 2166 * 2167 * Note that negative values of dma_attr_sgllen are supposed 2168 * to mean unlimited, but we just cast them to mean a 2169 * "ridiculous large limit". This saves some extra checks on 2170 * hot paths. 2171 */ 2172 if ((sinfo->si_copybuf_req == 0) && 2173 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2174 (dmao->dmao_size < dma->dp_maxxfer)) { 2175 fast: 2176 /* 2177 * If the driver supports FMA, insert the handle in the FMA DMA 2178 * handle cache. 2179 */ 2180 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2181 hp->dmai_error.err_cf = rootnex_dma_check; 2182 2183 /* 2184 * copy out the first cookie and ccountp, set the cookie 2185 * pointer to the second cookie. The first cookie is passed 2186 * back on the stack. Additional cookies are accessed via 2187 * ddi_dma_nextcookie() 2188 */ 2189 *cookiep = dma->dp_cookies[0]; 2190 *ccountp = sinfo->si_sgl_size; 2191 hp->dmai_cookie++; 2192 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2193 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2194 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2195 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2196 uint_t, dmao->dmao_size, uint_t, *ccountp); 2197 2198 2199 return (DDI_DMA_MAPPED); 2200 } 2201 2202 /* 2203 * go to the slow path, we may need to alloc more memory, create 2204 * multiple windows, and munge up a sgl to make the device happy. 2205 */ 2206 2207 /* 2208 * With the IOMMU mapobject method used, we should never hit 2209 * the slow path. If we do, something is seriously wrong. 2210 * Clean up and return an error. 2211 */ 2212 2213 #if defined(__amd64) && !defined(__xpv) 2214 2215 if (dma->dp_dvma_used) { 2216 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2217 &dma->dp_dvma); 2218 e = DDI_DMA_NOMAPPING; 2219 } else { 2220 #endif 2221 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2222 kmflag); 2223 #if defined(__amd64) && !defined(__xpv) 2224 } 2225 #endif 2226 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2227 if (dma->dp_need_to_free_cookie) { 2228 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2229 } 2230 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2231 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2232 return (e); 2233 } 2234 2235 /* 2236 * If the driver supports FMA, insert the handle in the FMA DMA handle 2237 * cache. 2238 */ 2239 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2240 hp->dmai_error.err_cf = rootnex_dma_check; 2241 2242 /* if the first window uses the copy buffer, sync it for the device */ 2243 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2244 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2245 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2246 DDI_DMA_SYNC_FORDEV); 2247 } 2248 2249 /* 2250 * copy out the first cookie and ccountp, set the cookie pointer to the 2251 * second cookie. Make sure the partial flag is set/cleared correctly. 2252 * If we have a partial map (i.e. multiple windows), the number of 2253 * cookies we return is the number of cookies in the first window. 2254 */ 2255 if (e == DDI_DMA_MAPPED) { 2256 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2257 *ccountp = sinfo->si_sgl_size; 2258 hp->dmai_nwin = 1; 2259 } else { 2260 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2261 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2262 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2263 } 2264 *cookiep = dma->dp_cookies[0]; 2265 hp->dmai_cookie++; 2266 2267 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2268 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2269 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2270 dmao->dmao_size, uint_t, *ccountp); 2271 return (e); 2272 } 2273 2274 /* 2275 * rootnex_dma_bindhdl() 2276 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2277 */ 2278 static int 2279 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2280 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2281 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2282 { 2283 int ret; 2284 #if defined(__amd64) && !defined(__xpv) 2285 if (IOMMU_USED(rdip)) 2286 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2287 cookiep, ccountp); 2288 else 2289 #endif 2290 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2291 cookiep, ccountp); 2292 2293 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2294 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2295 &rootnex_state->r_dvma_call_list_id); 2296 } 2297 2298 return (ret); 2299 } 2300 2301 2302 2303 /*ARGSUSED*/ 2304 static int 2305 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2306 ddi_dma_handle_t handle) 2307 { 2308 ddi_dma_impl_t *hp; 2309 rootnex_dma_t *dma; 2310 int e; 2311 2312 hp = (ddi_dma_impl_t *)handle; 2313 dma = (rootnex_dma_t *)hp->dmai_private; 2314 2315 /* make sure the buffer wasn't free'd before calling unbind */ 2316 if (rootnex_unbind_verify_buffer) { 2317 e = rootnex_verify_buffer(dma); 2318 if (e != DDI_SUCCESS) { 2319 ASSERT(0); 2320 return (DDI_FAILURE); 2321 } 2322 } 2323 2324 /* sync the current window before unbinding the buffer */ 2325 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2326 (hp->dmai_rflags & DDI_DMA_READ)) { 2327 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2328 DDI_DMA_SYNC_FORCPU); 2329 } 2330 2331 /* 2332 * cleanup and copy buffer or window state. if we didn't use the copy 2333 * buffer or windows, there won't be much to do :-) 2334 */ 2335 rootnex_teardown_copybuf(dma); 2336 rootnex_teardown_windows(dma); 2337 2338 #if defined(__amd64) && !defined(__xpv) 2339 if (IOMMU_USED(rdip)) 2340 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2341 &dma->dp_dvma); 2342 #endif 2343 2344 /* 2345 * If we had to allocate space to for the worse case sgl (it didn't 2346 * fit into our pre-allocate buffer), free that up now 2347 */ 2348 if (dma->dp_need_to_free_cookie) { 2349 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2350 } 2351 2352 /* 2353 * clean up the handle so it's ready for the next bind (i.e. if the 2354 * handle is reused). 2355 */ 2356 rootnex_clean_dmahdl(hp); 2357 hp->dmai_error.err_cf = NULL; 2358 2359 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2360 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2361 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2362 2363 return (DDI_SUCCESS); 2364 } 2365 2366 /* 2367 * rootnex_dma_unbindhdl() 2368 * called from ddi_dma_unbind_handle() 2369 */ 2370 /*ARGSUSED*/ 2371 static int 2372 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2373 ddi_dma_handle_t handle) 2374 { 2375 int ret; 2376 2377 #if defined(__amd64) && !defined(__xpv) 2378 if (IOMMU_USED(rdip)) 2379 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2380 else 2381 #endif 2382 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2383 2384 if (rootnex_state->r_dvma_call_list_id) 2385 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2386 2387 return (ret); 2388 } 2389 2390 #if defined(__amd64) && !defined(__xpv) 2391 2392 static int 2393 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2394 { 2395 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2396 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2397 2398 if (dma->dp_sleep_flags != KM_SLEEP && 2399 dma->dp_sleep_flags != KM_NOSLEEP) 2400 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2401 return (dma->dp_sleep_flags); 2402 } 2403 /*ARGSUSED*/ 2404 static void 2405 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2406 { 2407 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2408 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2409 rootnex_window_t *window; 2410 2411 if (dma->dp_window) { 2412 window = &dma->dp_window[dma->dp_current_win]; 2413 hp->dmai_cookie = window->wd_first_cookie; 2414 } else { 2415 hp->dmai_cookie = dma->dp_cookies; 2416 } 2417 hp->dmai_cookie++; 2418 } 2419 2420 /*ARGSUSED*/ 2421 static int 2422 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2423 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2424 { 2425 int i; 2426 int km_flags; 2427 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2428 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2429 rootnex_window_t *window; 2430 ddi_dma_cookie_t *cp; 2431 ddi_dma_cookie_t *cookie; 2432 2433 ASSERT(*cookiepp == NULL); 2434 ASSERT(*ccountp == 0); 2435 2436 if (dma->dp_window) { 2437 window = &dma->dp_window[dma->dp_current_win]; 2438 cp = window->wd_first_cookie; 2439 *ccountp = window->wd_cookie_cnt; 2440 } else { 2441 cp = dma->dp_cookies; 2442 *ccountp = dma->dp_sglinfo.si_sgl_size; 2443 } 2444 2445 km_flags = rootnex_coredma_get_sleep_flags(handle); 2446 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2447 if (cookie == NULL) { 2448 return (DDI_DMA_NORESOURCES); 2449 } 2450 2451 for (i = 0; i < *ccountp; i++) { 2452 cookie[i].dmac_notused = cp[i].dmac_notused; 2453 cookie[i].dmac_type = cp[i].dmac_type; 2454 cookie[i].dmac_address = cp[i].dmac_address; 2455 cookie[i].dmac_size = cp[i].dmac_size; 2456 } 2457 2458 *cookiepp = cookie; 2459 2460 return (DDI_SUCCESS); 2461 } 2462 2463 /*ARGSUSED*/ 2464 static int 2465 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2466 ddi_dma_cookie_t *cookiep, uint_t ccount) 2467 { 2468 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2469 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2470 rootnex_window_t *window; 2471 ddi_dma_cookie_t *cur_cookiep; 2472 2473 ASSERT(cookiep); 2474 ASSERT(ccount != 0); 2475 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2476 2477 if (dma->dp_window) { 2478 window = &dma->dp_window[dma->dp_current_win]; 2479 dma->dp_saved_cookies = window->wd_first_cookie; 2480 window->wd_first_cookie = cookiep; 2481 ASSERT(ccount == window->wd_cookie_cnt); 2482 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2483 + window->wd_first_cookie; 2484 } else { 2485 dma->dp_saved_cookies = dma->dp_cookies; 2486 dma->dp_cookies = cookiep; 2487 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2488 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2489 + dma->dp_cookies; 2490 } 2491 2492 dma->dp_need_to_switch_cookies = B_TRUE; 2493 hp->dmai_cookie = cur_cookiep; 2494 2495 return (DDI_SUCCESS); 2496 } 2497 2498 /*ARGSUSED*/ 2499 static int 2500 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2501 { 2502 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2503 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2504 rootnex_window_t *window; 2505 ddi_dma_cookie_t *cur_cookiep; 2506 ddi_dma_cookie_t *cookie_array; 2507 uint_t ccount; 2508 2509 /* check if cookies have not been switched */ 2510 if (dma->dp_need_to_switch_cookies == B_FALSE) 2511 return (DDI_SUCCESS); 2512 2513 ASSERT(dma->dp_saved_cookies); 2514 2515 if (dma->dp_window) { 2516 window = &dma->dp_window[dma->dp_current_win]; 2517 cookie_array = window->wd_first_cookie; 2518 window->wd_first_cookie = dma->dp_saved_cookies; 2519 dma->dp_saved_cookies = NULL; 2520 ccount = window->wd_cookie_cnt; 2521 cur_cookiep = (hp->dmai_cookie - cookie_array) 2522 + window->wd_first_cookie; 2523 } else { 2524 cookie_array = dma->dp_cookies; 2525 dma->dp_cookies = dma->dp_saved_cookies; 2526 dma->dp_saved_cookies = NULL; 2527 ccount = dma->dp_sglinfo.si_sgl_size; 2528 cur_cookiep = (hp->dmai_cookie - cookie_array) 2529 + dma->dp_cookies; 2530 } 2531 2532 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2533 2534 hp->dmai_cookie = cur_cookiep; 2535 2536 dma->dp_need_to_switch_cookies = B_FALSE; 2537 2538 return (DDI_SUCCESS); 2539 } 2540 2541 #endif 2542 2543 static struct as * 2544 rootnex_get_as(ddi_dma_obj_t *dmao) 2545 { 2546 struct as *asp; 2547 2548 switch (dmao->dmao_type) { 2549 case DMA_OTYP_VADDR: 2550 case DMA_OTYP_BUFVADDR: 2551 asp = dmao->dmao_obj.virt_obj.v_as; 2552 if (asp == NULL) 2553 asp = &kas; 2554 break; 2555 default: 2556 asp = NULL; 2557 break; 2558 } 2559 return (asp); 2560 } 2561 2562 /* 2563 * rootnex_verify_buffer() 2564 * verify buffer wasn't free'd 2565 */ 2566 static int 2567 rootnex_verify_buffer(rootnex_dma_t *dma) 2568 { 2569 page_t **pplist; 2570 caddr_t vaddr; 2571 uint_t pcnt; 2572 uint_t poff; 2573 page_t *pp; 2574 char b; 2575 int i; 2576 2577 /* Figure out how many pages this buffer occupies */ 2578 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2579 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2580 } else { 2581 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2582 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2583 } 2584 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2585 2586 switch (dma->dp_dma.dmao_type) { 2587 case DMA_OTYP_PAGES: 2588 /* 2589 * for a linked list of pp's walk through them to make sure 2590 * they're locked and not free. 2591 */ 2592 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2593 for (i = 0; i < pcnt; i++) { 2594 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2595 return (DDI_FAILURE); 2596 } 2597 pp = pp->p_next; 2598 } 2599 break; 2600 2601 case DMA_OTYP_VADDR: 2602 case DMA_OTYP_BUFVADDR: 2603 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2604 /* 2605 * for an array of pp's walk through them to make sure they're 2606 * not free. It's possible that they may not be locked. 2607 */ 2608 if (pplist) { 2609 for (i = 0; i < pcnt; i++) { 2610 if (PP_ISFREE(pplist[i])) { 2611 return (DDI_FAILURE); 2612 } 2613 } 2614 2615 /* For a virtual address, try to peek at each page */ 2616 } else { 2617 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2618 for (i = 0; i < pcnt; i++) { 2619 if (ddi_peek8(NULL, vaddr, &b) == 2620 DDI_FAILURE) 2621 return (DDI_FAILURE); 2622 vaddr += MMU_PAGESIZE; 2623 } 2624 } 2625 } 2626 break; 2627 2628 default: 2629 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2630 break; 2631 } 2632 2633 return (DDI_SUCCESS); 2634 } 2635 2636 2637 /* 2638 * rootnex_clean_dmahdl() 2639 * Clean the dma handle. This should be called on a handle alloc and an 2640 * unbind handle. Set the handle state to the default settings. 2641 */ 2642 static void 2643 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2644 { 2645 rootnex_dma_t *dma; 2646 2647 2648 dma = (rootnex_dma_t *)hp->dmai_private; 2649 2650 hp->dmai_nwin = 0; 2651 dma->dp_current_cookie = 0; 2652 dma->dp_copybuf_size = 0; 2653 dma->dp_window = NULL; 2654 dma->dp_cbaddr = NULL; 2655 dma->dp_inuse = B_FALSE; 2656 dma->dp_dvma_used = B_FALSE; 2657 dma->dp_need_to_free_cookie = B_FALSE; 2658 dma->dp_need_to_switch_cookies = B_FALSE; 2659 dma->dp_saved_cookies = NULL; 2660 dma->dp_sleep_flags = KM_PANIC; 2661 dma->dp_need_to_free_window = B_FALSE; 2662 dma->dp_partial_required = B_FALSE; 2663 dma->dp_trim_required = B_FALSE; 2664 dma->dp_sglinfo.si_copybuf_req = 0; 2665 #if !defined(__amd64) 2666 dma->dp_cb_remaping = B_FALSE; 2667 dma->dp_kva = NULL; 2668 #endif 2669 2670 /* FMA related initialization */ 2671 hp->dmai_fault = 0; 2672 hp->dmai_fault_check = NULL; 2673 hp->dmai_fault_notify = NULL; 2674 hp->dmai_error.err_ena = 0; 2675 hp->dmai_error.err_status = DDI_FM_OK; 2676 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2677 hp->dmai_error.err_ontrap = NULL; 2678 } 2679 2680 2681 /* 2682 * rootnex_valid_alloc_parms() 2683 * Called in ddi_dma_alloc_handle path to validate its parameters. 2684 */ 2685 static int 2686 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2687 { 2688 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2689 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2690 (attr->dma_attr_granular > MMU_PAGESIZE) || 2691 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2692 return (DDI_DMA_BADATTR); 2693 } 2694 2695 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2696 return (DDI_DMA_BADATTR); 2697 } 2698 2699 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2700 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2701 attr->dma_attr_sgllen == 0) { 2702 return (DDI_DMA_BADATTR); 2703 } 2704 2705 /* We should be able to DMA into every byte offset in a page */ 2706 if (maxsegmentsize < MMU_PAGESIZE) { 2707 return (DDI_DMA_BADATTR); 2708 } 2709 2710 /* if we're bouncing on seg, seg must be <= addr_hi */ 2711 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2712 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2713 return (DDI_DMA_BADATTR); 2714 } 2715 return (DDI_SUCCESS); 2716 } 2717 2718 /* 2719 * rootnex_valid_bind_parms() 2720 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2721 */ 2722 /* ARGSUSED */ 2723 static int 2724 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2725 { 2726 #if !defined(__amd64) 2727 /* 2728 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2729 * we can track the offset for the obsoleted interfaces. 2730 */ 2731 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2732 return (DDI_DMA_TOOBIG); 2733 } 2734 #endif 2735 2736 return (DDI_SUCCESS); 2737 } 2738 2739 2740 /* 2741 * rootnex_need_bounce_seg() 2742 * check to see if the buffer lives on both side of the seg. 2743 */ 2744 static boolean_t 2745 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2746 { 2747 ddi_dma_atyp_t buftype; 2748 rootnex_addr_t raddr; 2749 boolean_t lower_addr; 2750 boolean_t upper_addr; 2751 uint64_t offset; 2752 page_t **pplist; 2753 uint64_t paddr; 2754 uint32_t psize; 2755 uint32_t size; 2756 caddr_t vaddr; 2757 uint_t pcnt; 2758 page_t *pp; 2759 2760 2761 /* shortcuts */ 2762 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2763 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2764 buftype = dmar_object->dmao_type; 2765 size = dmar_object->dmao_size; 2766 2767 lower_addr = B_FALSE; 2768 upper_addr = B_FALSE; 2769 pcnt = 0; 2770 2771 /* 2772 * Process the first page to handle the initial offset of the buffer. 2773 * We'll use the base address we get later when we loop through all 2774 * the pages. 2775 */ 2776 if (buftype == DMA_OTYP_PAGES) { 2777 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2778 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2779 MMU_PAGEOFFSET; 2780 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2781 psize = MIN(size, (MMU_PAGESIZE - offset)); 2782 pp = pp->p_next; 2783 sglinfo->si_asp = NULL; 2784 } else if (pplist != NULL) { 2785 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2786 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2787 if (sglinfo->si_asp == NULL) { 2788 sglinfo->si_asp = &kas; 2789 } 2790 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2791 paddr += offset; 2792 psize = MIN(size, (MMU_PAGESIZE - offset)); 2793 pcnt++; 2794 } else { 2795 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2796 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2797 if (sglinfo->si_asp == NULL) { 2798 sglinfo->si_asp = &kas; 2799 } 2800 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2801 paddr += offset; 2802 psize = MIN(size, (MMU_PAGESIZE - offset)); 2803 vaddr += psize; 2804 } 2805 2806 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2807 2808 if ((raddr + psize) > sglinfo->si_segmask) { 2809 upper_addr = B_TRUE; 2810 } else { 2811 lower_addr = B_TRUE; 2812 } 2813 size -= psize; 2814 2815 /* 2816 * Walk through the rest of the pages in the buffer. Track to see 2817 * if we have pages on both sides of the segment boundary. 2818 */ 2819 while (size > 0) { 2820 /* partial or full page */ 2821 psize = MIN(size, MMU_PAGESIZE); 2822 2823 if (buftype == DMA_OTYP_PAGES) { 2824 /* get the paddr from the page_t */ 2825 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2826 paddr = pfn_to_pa(pp->p_pagenum); 2827 pp = pp->p_next; 2828 } else if (pplist != NULL) { 2829 /* index into the array of page_t's to get the paddr */ 2830 ASSERT(!PP_ISFREE(pplist[pcnt])); 2831 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2832 pcnt++; 2833 } else { 2834 /* call into the VM to get the paddr */ 2835 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2836 vaddr)); 2837 vaddr += psize; 2838 } 2839 2840 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2841 2842 if ((raddr + psize) > sglinfo->si_segmask) { 2843 upper_addr = B_TRUE; 2844 } else { 2845 lower_addr = B_TRUE; 2846 } 2847 /* 2848 * if the buffer lives both above and below the segment 2849 * boundary, or the current page is the page immediately 2850 * after the segment, we will use a copy/bounce buffer for 2851 * all pages > seg. 2852 */ 2853 if ((lower_addr && upper_addr) || 2854 (raddr == (sglinfo->si_segmask + 1))) { 2855 return (B_TRUE); 2856 } 2857 2858 size -= psize; 2859 } 2860 2861 return (B_FALSE); 2862 } 2863 2864 /* 2865 * rootnex_get_sgl() 2866 * Called in bind fastpath to get the sgl. Most of this will be replaced 2867 * with a call to the vm layer when vm2.0 comes around... 2868 */ 2869 static void 2870 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2871 rootnex_sglinfo_t *sglinfo) 2872 { 2873 ddi_dma_atyp_t buftype; 2874 rootnex_addr_t raddr; 2875 uint64_t last_page; 2876 uint64_t offset; 2877 uint64_t addrhi; 2878 uint64_t addrlo; 2879 uint64_t maxseg; 2880 page_t **pplist; 2881 uint64_t paddr; 2882 uint32_t psize; 2883 uint32_t size; 2884 caddr_t vaddr; 2885 uint_t pcnt; 2886 page_t *pp; 2887 uint_t cnt; 2888 2889 2890 /* shortcuts */ 2891 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2892 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2893 maxseg = sglinfo->si_max_cookie_size; 2894 buftype = dmar_object->dmao_type; 2895 addrhi = sglinfo->si_max_addr; 2896 addrlo = sglinfo->si_min_addr; 2897 size = dmar_object->dmao_size; 2898 2899 pcnt = 0; 2900 cnt = 0; 2901 2902 2903 /* 2904 * check to see if we need to use the copy buffer for pages over 2905 * the segment attr. 2906 */ 2907 sglinfo->si_bounce_on_seg = B_FALSE; 2908 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2909 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2910 dmar_object, sglinfo); 2911 } 2912 2913 /* 2914 * if we were passed down a linked list of pages, i.e. pointer to 2915 * page_t, use this to get our physical address and buf offset. 2916 */ 2917 if (buftype == DMA_OTYP_PAGES) { 2918 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2919 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2920 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2921 MMU_PAGEOFFSET; 2922 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2923 psize = MIN(size, (MMU_PAGESIZE - offset)); 2924 pp = pp->p_next; 2925 sglinfo->si_asp = NULL; 2926 2927 /* 2928 * We weren't passed down a linked list of pages, but if we were passed 2929 * down an array of pages, use this to get our physical address and buf 2930 * offset. 2931 */ 2932 } else if (pplist != NULL) { 2933 ASSERT((buftype == DMA_OTYP_VADDR) || 2934 (buftype == DMA_OTYP_BUFVADDR)); 2935 2936 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2937 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2938 if (sglinfo->si_asp == NULL) { 2939 sglinfo->si_asp = &kas; 2940 } 2941 2942 ASSERT(!PP_ISFREE(pplist[pcnt])); 2943 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2944 paddr += offset; 2945 psize = MIN(size, (MMU_PAGESIZE - offset)); 2946 pcnt++; 2947 2948 /* 2949 * All we have is a virtual address, we'll need to call into the VM 2950 * to get the physical address. 2951 */ 2952 } else { 2953 ASSERT((buftype == DMA_OTYP_VADDR) || 2954 (buftype == DMA_OTYP_BUFVADDR)); 2955 2956 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2957 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2958 if (sglinfo->si_asp == NULL) { 2959 sglinfo->si_asp = &kas; 2960 } 2961 2962 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2963 paddr += offset; 2964 psize = MIN(size, (MMU_PAGESIZE - offset)); 2965 vaddr += psize; 2966 } 2967 2968 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2969 2970 /* 2971 * Setup the first cookie with the physical address of the page and the 2972 * size of the page (which takes into account the initial offset into 2973 * the page. 2974 */ 2975 sgl[cnt].dmac_laddress = raddr; 2976 sgl[cnt].dmac_size = psize; 2977 sgl[cnt].dmac_type = 0; 2978 2979 /* 2980 * Save away the buffer offset into the page. We'll need this later in 2981 * the copy buffer code to help figure out the page index within the 2982 * buffer and the offset into the current page. 2983 */ 2984 sglinfo->si_buf_offset = offset; 2985 2986 /* 2987 * If we are using the copy buffer for anything over the segment 2988 * boundary, and this page is over the segment boundary. 2989 * OR 2990 * if the DMA engine can't reach the physical address. 2991 */ 2992 if (((sglinfo->si_bounce_on_seg) && 2993 ((raddr + psize) > sglinfo->si_segmask)) || 2994 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2995 /* 2996 * Increase how much copy buffer we use. We always increase by 2997 * pagesize so we don't have to worry about converting offsets. 2998 * Set a flag in the cookies dmac_type to indicate that it uses 2999 * the copy buffer. If this isn't the last cookie, go to the 3000 * next cookie (since we separate each page which uses the copy 3001 * buffer in case the copy buffer is not physically contiguous. 3002 */ 3003 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3004 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3005 if ((cnt + 1) < sglinfo->si_max_pages) { 3006 cnt++; 3007 sgl[cnt].dmac_laddress = 0; 3008 sgl[cnt].dmac_size = 0; 3009 sgl[cnt].dmac_type = 0; 3010 } 3011 } 3012 3013 /* 3014 * save this page's physical address so we can figure out if the next 3015 * page is physically contiguous. Keep decrementing size until we are 3016 * done with the buffer. 3017 */ 3018 last_page = raddr & MMU_PAGEMASK; 3019 size -= psize; 3020 3021 while (size > 0) { 3022 /* Get the size for this page (i.e. partial or full page) */ 3023 psize = MIN(size, MMU_PAGESIZE); 3024 3025 if (buftype == DMA_OTYP_PAGES) { 3026 /* get the paddr from the page_t */ 3027 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3028 paddr = pfn_to_pa(pp->p_pagenum); 3029 pp = pp->p_next; 3030 } else if (pplist != NULL) { 3031 /* index into the array of page_t's to get the paddr */ 3032 ASSERT(!PP_ISFREE(pplist[pcnt])); 3033 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3034 pcnt++; 3035 } else { 3036 /* call into the VM to get the paddr */ 3037 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3038 vaddr)); 3039 vaddr += psize; 3040 } 3041 3042 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3043 3044 /* 3045 * If we are using the copy buffer for anything over the 3046 * segment boundary, and this page is over the segment 3047 * boundary. 3048 * OR 3049 * if the DMA engine can't reach the physical address. 3050 */ 3051 if (((sglinfo->si_bounce_on_seg) && 3052 ((raddr + psize) > sglinfo->si_segmask)) || 3053 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3054 3055 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3056 3057 /* 3058 * if there is something in the current cookie, go to 3059 * the next one. We only want one page in a cookie which 3060 * uses the copybuf since the copybuf doesn't have to 3061 * be physically contiguous. 3062 */ 3063 if (sgl[cnt].dmac_size != 0) { 3064 cnt++; 3065 } 3066 sgl[cnt].dmac_laddress = raddr; 3067 sgl[cnt].dmac_size = psize; 3068 #if defined(__amd64) 3069 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3070 #else 3071 /* 3072 * save the buf offset for 32-bit kernel. used in the 3073 * obsoleted interfaces. 3074 */ 3075 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3076 (dmar_object->dmao_size - size); 3077 #endif 3078 /* if this isn't the last cookie, go to the next one */ 3079 if ((cnt + 1) < sglinfo->si_max_pages) { 3080 cnt++; 3081 sgl[cnt].dmac_laddress = 0; 3082 sgl[cnt].dmac_size = 0; 3083 sgl[cnt].dmac_type = 0; 3084 } 3085 3086 /* 3087 * this page didn't need the copy buffer, if it's not physically 3088 * contiguous, or it would put us over a segment boundary, or it 3089 * puts us over the max cookie size, or the current sgl doesn't 3090 * have anything in it. 3091 */ 3092 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3093 !(raddr & sglinfo->si_segmask) || 3094 ((sgl[cnt].dmac_size + psize) > maxseg) || 3095 (sgl[cnt].dmac_size == 0)) { 3096 /* 3097 * if we're not already in a new cookie, go to the next 3098 * cookie. 3099 */ 3100 if (sgl[cnt].dmac_size != 0) { 3101 cnt++; 3102 } 3103 3104 /* save the cookie information */ 3105 sgl[cnt].dmac_laddress = raddr; 3106 sgl[cnt].dmac_size = psize; 3107 #if defined(__amd64) 3108 sgl[cnt].dmac_type = 0; 3109 #else 3110 /* 3111 * save the buf offset for 32-bit kernel. used in the 3112 * obsoleted interfaces. 3113 */ 3114 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3115 #endif 3116 3117 /* 3118 * this page didn't need the copy buffer, it is physically 3119 * contiguous with the last page, and it's <= the max cookie 3120 * size. 3121 */ 3122 } else { 3123 sgl[cnt].dmac_size += psize; 3124 3125 /* 3126 * if this exactly == the maximum cookie size, and 3127 * it isn't the last cookie, go to the next cookie. 3128 */ 3129 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3130 ((cnt + 1) < sglinfo->si_max_pages)) { 3131 cnt++; 3132 sgl[cnt].dmac_laddress = 0; 3133 sgl[cnt].dmac_size = 0; 3134 sgl[cnt].dmac_type = 0; 3135 } 3136 } 3137 3138 /* 3139 * save this page's physical address so we can figure out if the 3140 * next page is physically contiguous. Keep decrementing size 3141 * until we are done with the buffer. 3142 */ 3143 last_page = raddr; 3144 size -= psize; 3145 } 3146 3147 /* we're done, save away how many cookies the sgl has */ 3148 if (sgl[cnt].dmac_size == 0) { 3149 ASSERT(cnt < sglinfo->si_max_pages); 3150 sglinfo->si_sgl_size = cnt; 3151 } else { 3152 sglinfo->si_sgl_size = cnt + 1; 3153 } 3154 } 3155 3156 static void 3157 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3158 rootnex_sglinfo_t *sglinfo) 3159 { 3160 uint64_t offset; 3161 uint64_t maxseg; 3162 uint64_t dvaddr; 3163 struct dvmaseg *dvs; 3164 uint64_t paddr; 3165 uint32_t psize, ssize; 3166 uint32_t size; 3167 uint_t cnt; 3168 int physcontig; 3169 3170 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3171 3172 /* shortcuts */ 3173 maxseg = sglinfo->si_max_cookie_size; 3174 size = dmar_object->dmao_size; 3175 3176 cnt = 0; 3177 sglinfo->si_bounce_on_seg = B_FALSE; 3178 3179 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3180 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3181 ssize = dvs->dvs_len; 3182 paddr = dvs->dvs_start; 3183 paddr += offset; 3184 psize = MIN(ssize, (maxseg - offset)); 3185 dvaddr = paddr + psize; 3186 ssize -= psize; 3187 3188 sgl[cnt].dmac_laddress = paddr; 3189 sgl[cnt].dmac_size = psize; 3190 sgl[cnt].dmac_type = 0; 3191 3192 size -= psize; 3193 while (size > 0) { 3194 if (ssize == 0) { 3195 dvs++; 3196 ssize = dvs->dvs_len; 3197 dvaddr = dvs->dvs_start; 3198 physcontig = 0; 3199 } else 3200 physcontig = 1; 3201 3202 paddr = dvaddr; 3203 psize = MIN(ssize, maxseg); 3204 dvaddr += psize; 3205 ssize -= psize; 3206 3207 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3208 ((sgl[cnt].dmac_size + psize) > maxseg) || 3209 (sgl[cnt].dmac_size == 0)) { 3210 /* 3211 * if we're not already in a new cookie, go to the next 3212 * cookie. 3213 */ 3214 if (sgl[cnt].dmac_size != 0) { 3215 cnt++; 3216 } 3217 3218 /* save the cookie information */ 3219 sgl[cnt].dmac_laddress = paddr; 3220 sgl[cnt].dmac_size = psize; 3221 sgl[cnt].dmac_type = 0; 3222 } else { 3223 sgl[cnt].dmac_size += psize; 3224 3225 /* 3226 * if this exactly == the maximum cookie size, and 3227 * it isn't the last cookie, go to the next cookie. 3228 */ 3229 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3230 ((cnt + 1) < sglinfo->si_max_pages)) { 3231 cnt++; 3232 sgl[cnt].dmac_laddress = 0; 3233 sgl[cnt].dmac_size = 0; 3234 sgl[cnt].dmac_type = 0; 3235 } 3236 } 3237 size -= psize; 3238 } 3239 3240 /* we're done, save away how many cookies the sgl has */ 3241 if (sgl[cnt].dmac_size == 0) { 3242 sglinfo->si_sgl_size = cnt; 3243 } else { 3244 sglinfo->si_sgl_size = cnt + 1; 3245 } 3246 } 3247 3248 /* 3249 * rootnex_bind_slowpath() 3250 * Call in the bind path if the calling driver can't use the sgl without 3251 * modifying it. We either need to use the copy buffer and/or we will end up 3252 * with a partial bind. 3253 */ 3254 static int 3255 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3256 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3257 { 3258 rootnex_sglinfo_t *sinfo; 3259 rootnex_window_t *window; 3260 ddi_dma_cookie_t *cookie; 3261 size_t copybuf_used; 3262 size_t dmac_size; 3263 boolean_t partial; 3264 off_t cur_offset; 3265 page_t *cur_pp; 3266 major_t mnum; 3267 int e; 3268 int i; 3269 3270 3271 sinfo = &dma->dp_sglinfo; 3272 copybuf_used = 0; 3273 partial = B_FALSE; 3274 3275 /* 3276 * If we're using the copybuf, set the copybuf state in dma struct. 3277 * Needs to be first since it sets the copy buffer size. 3278 */ 3279 if (sinfo->si_copybuf_req != 0) { 3280 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3281 if (e != DDI_SUCCESS) { 3282 return (e); 3283 } 3284 } else { 3285 dma->dp_copybuf_size = 0; 3286 } 3287 3288 /* 3289 * Figure out if we need to do a partial mapping. If so, figure out 3290 * if we need to trim the buffers when we munge the sgl. 3291 */ 3292 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3293 (dmao->dmao_size > dma->dp_maxxfer) || 3294 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3295 dma->dp_partial_required = B_TRUE; 3296 if (attr->dma_attr_granular != 1) { 3297 dma->dp_trim_required = B_TRUE; 3298 } 3299 } else { 3300 dma->dp_partial_required = B_FALSE; 3301 dma->dp_trim_required = B_FALSE; 3302 } 3303 3304 /* If we need to do a partial bind, make sure the driver supports it */ 3305 if (dma->dp_partial_required && 3306 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3307 3308 mnum = ddi_driver_major(dma->dp_dip); 3309 /* 3310 * patchable which allows us to print one warning per major 3311 * number. 3312 */ 3313 if ((rootnex_bind_warn) && 3314 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3315 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3316 cmn_err(CE_WARN, "!%s: coding error detected, the " 3317 "driver is using ddi_dma_attr(9S) incorrectly. " 3318 "There is a small risk of data corruption in " 3319 "particular with large I/Os. The driver should be " 3320 "replaced with a corrected version for proper " 3321 "system operation. To disable this warning, add " 3322 "'set rootnex:rootnex_bind_warn=0' to " 3323 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3324 } 3325 return (DDI_DMA_TOOBIG); 3326 } 3327 3328 /* 3329 * we might need multiple windows, setup state to handle them. In this 3330 * code path, we will have at least one window. 3331 */ 3332 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3333 if (e != DDI_SUCCESS) { 3334 rootnex_teardown_copybuf(dma); 3335 return (e); 3336 } 3337 3338 window = &dma->dp_window[0]; 3339 cookie = &dma->dp_cookies[0]; 3340 cur_offset = 0; 3341 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3342 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3343 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3344 } 3345 3346 /* loop though all the cookies we got back from get_sgl() */ 3347 for (i = 0; i < sinfo->si_sgl_size; i++) { 3348 /* 3349 * If we're using the copy buffer, check this cookie and setup 3350 * its associated copy buffer state. If this cookie uses the 3351 * copy buffer, make sure we sync this window during dma_sync. 3352 */ 3353 if (dma->dp_copybuf_size > 0) { 3354 rootnex_setup_cookie(dmao, dma, cookie, 3355 cur_offset, ©buf_used, &cur_pp); 3356 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3357 window->wd_dosync = B_TRUE; 3358 } 3359 } 3360 3361 /* 3362 * save away the cookie size, since it could be modified in 3363 * the windowing code. 3364 */ 3365 dmac_size = cookie->dmac_size; 3366 3367 /* if we went over max copybuf size */ 3368 if (dma->dp_copybuf_size && 3369 (copybuf_used > dma->dp_copybuf_size)) { 3370 partial = B_TRUE; 3371 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3372 cookie, cur_offset, ©buf_used); 3373 if (e != DDI_SUCCESS) { 3374 rootnex_teardown_copybuf(dma); 3375 rootnex_teardown_windows(dma); 3376 return (e); 3377 } 3378 3379 /* 3380 * if the coookie uses the copy buffer, make sure the 3381 * new window we just moved to is set to sync. 3382 */ 3383 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3384 window->wd_dosync = B_TRUE; 3385 } 3386 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3387 dma->dp_dip); 3388 3389 /* if the cookie cnt == max sgllen, move to the next window */ 3390 } else if (window->wd_cookie_cnt >= 3391 (unsigned)attr->dma_attr_sgllen) { 3392 partial = B_TRUE; 3393 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3394 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3395 cookie, attr, cur_offset); 3396 if (e != DDI_SUCCESS) { 3397 rootnex_teardown_copybuf(dma); 3398 rootnex_teardown_windows(dma); 3399 return (e); 3400 } 3401 3402 /* 3403 * if the coookie uses the copy buffer, make sure the 3404 * new window we just moved to is set to sync. 3405 */ 3406 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3407 window->wd_dosync = B_TRUE; 3408 } 3409 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3410 dma->dp_dip); 3411 3412 /* else if we will be over maxxfer */ 3413 } else if ((window->wd_size + dmac_size) > 3414 dma->dp_maxxfer) { 3415 partial = B_TRUE; 3416 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3417 cookie); 3418 if (e != DDI_SUCCESS) { 3419 rootnex_teardown_copybuf(dma); 3420 rootnex_teardown_windows(dma); 3421 return (e); 3422 } 3423 3424 /* 3425 * if the coookie uses the copy buffer, make sure the 3426 * new window we just moved to is set to sync. 3427 */ 3428 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3429 window->wd_dosync = B_TRUE; 3430 } 3431 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3432 dma->dp_dip); 3433 3434 /* else this cookie fits in the current window */ 3435 } else { 3436 window->wd_cookie_cnt++; 3437 window->wd_size += dmac_size; 3438 } 3439 3440 /* track our offset into the buffer, go to the next cookie */ 3441 ASSERT(dmac_size <= dmao->dmao_size); 3442 ASSERT(cookie->dmac_size <= dmac_size); 3443 cur_offset += dmac_size; 3444 cookie++; 3445 } 3446 3447 /* if we ended up with a zero sized window in the end, clean it up */ 3448 if (window->wd_size == 0) { 3449 hp->dmai_nwin--; 3450 window--; 3451 } 3452 3453 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3454 3455 if (!partial) { 3456 return (DDI_DMA_MAPPED); 3457 } 3458 3459 ASSERT(dma->dp_partial_required); 3460 return (DDI_DMA_PARTIAL_MAP); 3461 } 3462 3463 /* 3464 * rootnex_setup_copybuf() 3465 * Called in bind slowpath. Figures out if we're going to use the copy 3466 * buffer, and if we do, sets up the basic state to handle it. 3467 */ 3468 static int 3469 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3470 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3471 { 3472 rootnex_sglinfo_t *sinfo; 3473 ddi_dma_attr_t lattr; 3474 size_t max_copybuf; 3475 int cansleep; 3476 int e; 3477 #if !defined(__amd64) 3478 int vmflag; 3479 #endif 3480 3481 ASSERT(!dma->dp_dvma_used); 3482 3483 sinfo = &dma->dp_sglinfo; 3484 3485 /* read this first so it's consistent through the routine */ 3486 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3487 3488 /* We need to call into the rootnex on ddi_dma_sync() */ 3489 hp->dmai_rflags &= ~DMP_NOSYNC; 3490 3491 /* make sure the copybuf size <= the max size */ 3492 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3493 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3494 3495 #if !defined(__amd64) 3496 /* 3497 * if we don't have kva space to copy to/from, allocate the KVA space 3498 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3499 * the 64-bit kernel. 3500 */ 3501 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3502 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3503 3504 /* convert the sleep flags */ 3505 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3506 vmflag = VM_SLEEP; 3507 } else { 3508 vmflag = VM_NOSLEEP; 3509 } 3510 3511 /* allocate Kernel VA space that we can bcopy to/from */ 3512 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3513 vmflag); 3514 if (dma->dp_kva == NULL) { 3515 return (DDI_DMA_NORESOURCES); 3516 } 3517 } 3518 #endif 3519 3520 /* convert the sleep flags */ 3521 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3522 cansleep = 1; 3523 } else { 3524 cansleep = 0; 3525 } 3526 3527 /* 3528 * Allocate the actual copy buffer. This needs to fit within the DMA 3529 * engine limits, so we can't use kmem_alloc... We don't need 3530 * contiguous memory (sgllen) since we will be forcing windows on 3531 * sgllen anyway. 3532 */ 3533 lattr = *attr; 3534 lattr.dma_attr_align = MMU_PAGESIZE; 3535 lattr.dma_attr_sgllen = -1; /* no limit */ 3536 /* 3537 * if we're using the copy buffer because of seg, use that for our 3538 * upper address limit. 3539 */ 3540 if (sinfo->si_bounce_on_seg) { 3541 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3542 } 3543 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3544 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3545 if (e != DDI_SUCCESS) { 3546 #if !defined(__amd64) 3547 if (dma->dp_kva != NULL) { 3548 vmem_free(heap_arena, dma->dp_kva, 3549 dma->dp_copybuf_size); 3550 } 3551 #endif 3552 return (DDI_DMA_NORESOURCES); 3553 } 3554 3555 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3556 size_t, dma->dp_copybuf_size); 3557 3558 return (DDI_SUCCESS); 3559 } 3560 3561 3562 /* 3563 * rootnex_setup_windows() 3564 * Called in bind slowpath to setup the window state. We always have windows 3565 * in the slowpath. Even if the window count = 1. 3566 */ 3567 static int 3568 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3569 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3570 { 3571 rootnex_window_t *windowp; 3572 rootnex_sglinfo_t *sinfo; 3573 size_t copy_state_size; 3574 size_t win_state_size; 3575 size_t state_available; 3576 size_t space_needed; 3577 uint_t copybuf_win; 3578 uint_t maxxfer_win; 3579 size_t space_used; 3580 uint_t sglwin; 3581 3582 3583 sinfo = &dma->dp_sglinfo; 3584 3585 dma->dp_current_win = 0; 3586 hp->dmai_nwin = 0; 3587 3588 /* If we don't need to do a partial, we only have one window */ 3589 if (!dma->dp_partial_required) { 3590 dma->dp_max_win = 1; 3591 3592 /* 3593 * we need multiple windows, need to figure out the worse case number 3594 * of windows. 3595 */ 3596 } else { 3597 /* 3598 * if we need windows because we need more copy buffer that 3599 * we allow, the worse case number of windows we could need 3600 * here would be (copybuf space required / copybuf space that 3601 * we have) plus one for remainder, and plus 2 to handle the 3602 * extra pages on the trim for the first and last pages of the 3603 * buffer (a page is the minimum window size so under the right 3604 * attr settings, you could have a window for each page). 3605 * The last page will only be hit here if the size is not a 3606 * multiple of the granularity (which theoretically shouldn't 3607 * be the case but never has been enforced, so we could have 3608 * broken things without it). 3609 */ 3610 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3611 ASSERT(dma->dp_copybuf_size > 0); 3612 copybuf_win = (sinfo->si_copybuf_req / 3613 dma->dp_copybuf_size) + 1 + 2; 3614 } else { 3615 copybuf_win = 0; 3616 } 3617 3618 /* 3619 * if we need windows because we have more cookies than the H/W 3620 * can handle, the number of windows we would need here would 3621 * be (cookie count / cookies count H/W supports minus 1[for 3622 * trim]) plus one for remainder. 3623 */ 3624 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3625 sglwin = (sinfo->si_sgl_size / 3626 (attr->dma_attr_sgllen - 1)) + 1; 3627 } else { 3628 sglwin = 0; 3629 } 3630 3631 /* 3632 * if we need windows because we're binding more memory than the 3633 * H/W can transfer at once, the number of windows we would need 3634 * here would be (xfer count / max xfer H/W supports) plus one 3635 * for remainder, and plus 2 to handle the extra pages on the 3636 * trim (see above comment about trim) 3637 */ 3638 if (dmao->dmao_size > dma->dp_maxxfer) { 3639 maxxfer_win = (dmao->dmao_size / 3640 dma->dp_maxxfer) + 1 + 2; 3641 } else { 3642 maxxfer_win = 0; 3643 } 3644 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3645 ASSERT(dma->dp_max_win > 0); 3646 } 3647 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3648 3649 /* 3650 * Get space for window and potential copy buffer state. Before we 3651 * go and allocate memory, see if we can get away with using what's 3652 * left in the pre-allocted state or the dynamically allocated sgl. 3653 */ 3654 space_used = (uintptr_t)(sinfo->si_sgl_size * 3655 sizeof (ddi_dma_cookie_t)); 3656 3657 /* if we dynamically allocated space for the cookies */ 3658 if (dma->dp_need_to_free_cookie) { 3659 /* if we have more space in the pre-allocted buffer, use it */ 3660 ASSERT(space_used <= dma->dp_cookie_size); 3661 if ((dma->dp_cookie_size - space_used) <= 3662 rootnex_state->r_prealloc_size) { 3663 state_available = rootnex_state->r_prealloc_size; 3664 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3665 3666 /* 3667 * else, we have more free space in the dynamically allocated 3668 * buffer, i.e. the buffer wasn't worse case fragmented so we 3669 * didn't need a lot of cookies. 3670 */ 3671 } else { 3672 state_available = dma->dp_cookie_size - space_used; 3673 windowp = (rootnex_window_t *) 3674 &dma->dp_cookies[sinfo->si_sgl_size]; 3675 } 3676 3677 /* we used the pre-alloced buffer */ 3678 } else { 3679 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3680 state_available = rootnex_state->r_prealloc_size - space_used; 3681 windowp = (rootnex_window_t *) 3682 &dma->dp_cookies[sinfo->si_sgl_size]; 3683 } 3684 3685 /* 3686 * figure out how much state we need to track the copy buffer. Add an 3687 * addition 8 bytes for pointer alignemnt later. 3688 */ 3689 if (dma->dp_copybuf_size > 0) { 3690 copy_state_size = sinfo->si_max_pages * 3691 sizeof (rootnex_pgmap_t); 3692 } else { 3693 copy_state_size = 0; 3694 } 3695 /* add an additional 8 bytes for pointer alignment */ 3696 space_needed = win_state_size + copy_state_size + 0x8; 3697 3698 /* if we have enough space already, use it */ 3699 if (state_available >= space_needed) { 3700 dma->dp_window = windowp; 3701 dma->dp_need_to_free_window = B_FALSE; 3702 3703 /* not enough space, need to allocate more. */ 3704 } else { 3705 dma->dp_window = kmem_alloc(space_needed, kmflag); 3706 if (dma->dp_window == NULL) { 3707 return (DDI_DMA_NORESOURCES); 3708 } 3709 dma->dp_need_to_free_window = B_TRUE; 3710 dma->dp_window_size = space_needed; 3711 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3712 dma->dp_dip, size_t, space_needed); 3713 } 3714 3715 /* 3716 * we allocate copy buffer state and window state at the same time. 3717 * setup our copy buffer state pointers. Make sure it's aligned. 3718 */ 3719 if (dma->dp_copybuf_size > 0) { 3720 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3721 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3722 3723 #if !defined(__amd64) 3724 /* 3725 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3726 * false/NULL. Should be quicker to bzero vs loop and set. 3727 */ 3728 bzero(dma->dp_pgmap, copy_state_size); 3729 #endif 3730 } else { 3731 dma->dp_pgmap = NULL; 3732 } 3733 3734 return (DDI_SUCCESS); 3735 } 3736 3737 3738 /* 3739 * rootnex_teardown_copybuf() 3740 * cleans up after rootnex_setup_copybuf() 3741 */ 3742 static void 3743 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3744 { 3745 #if !defined(__amd64) 3746 int i; 3747 3748 /* 3749 * if we allocated kernel heap VMEM space, go through all the pages and 3750 * map out any of the ones that we're mapped into the kernel heap VMEM 3751 * arena. Then free the VMEM space. 3752 */ 3753 if (dma->dp_kva != NULL) { 3754 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3755 if (dma->dp_pgmap[i].pm_mapped) { 3756 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3757 MMU_PAGESIZE, HAT_UNLOAD); 3758 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3759 } 3760 } 3761 3762 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3763 } 3764 3765 #endif 3766 3767 /* if we allocated a copy buffer, free it */ 3768 if (dma->dp_cbaddr != NULL) { 3769 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3770 } 3771 } 3772 3773 3774 /* 3775 * rootnex_teardown_windows() 3776 * cleans up after rootnex_setup_windows() 3777 */ 3778 static void 3779 rootnex_teardown_windows(rootnex_dma_t *dma) 3780 { 3781 /* 3782 * if we had to allocate window state on the last bind (because we 3783 * didn't have enough pre-allocated space in the handle), free it. 3784 */ 3785 if (dma->dp_need_to_free_window) { 3786 kmem_free(dma->dp_window, dma->dp_window_size); 3787 } 3788 } 3789 3790 3791 /* 3792 * rootnex_init_win() 3793 * Called in bind slow path during creation of a new window. Initializes 3794 * window state to default values. 3795 */ 3796 /*ARGSUSED*/ 3797 static void 3798 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3799 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3800 { 3801 hp->dmai_nwin++; 3802 window->wd_dosync = B_FALSE; 3803 window->wd_offset = cur_offset; 3804 window->wd_size = 0; 3805 window->wd_first_cookie = cookie; 3806 window->wd_cookie_cnt = 0; 3807 window->wd_trim.tr_trim_first = B_FALSE; 3808 window->wd_trim.tr_trim_last = B_FALSE; 3809 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3810 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3811 #if !defined(__amd64) 3812 window->wd_remap_copybuf = dma->dp_cb_remaping; 3813 #endif 3814 } 3815 3816 3817 /* 3818 * rootnex_setup_cookie() 3819 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3820 * the sgl uses the copy buffer, we need to go through each cookie, figure 3821 * out if it uses the copy buffer, and if it does, save away everything we'll 3822 * need during sync. 3823 */ 3824 static void 3825 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3826 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3827 page_t **cur_pp) 3828 { 3829 boolean_t copybuf_sz_power_2; 3830 rootnex_sglinfo_t *sinfo; 3831 paddr_t paddr; 3832 uint_t pidx; 3833 uint_t pcnt; 3834 off_t poff; 3835 #if defined(__amd64) 3836 pfn_t pfn; 3837 #else 3838 page_t **pplist; 3839 #endif 3840 3841 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3842 3843 sinfo = &dma->dp_sglinfo; 3844 3845 /* 3846 * Calculate the page index relative to the start of the buffer. The 3847 * index to the current page for our buffer is the offset into the 3848 * first page of the buffer plus our current offset into the buffer 3849 * itself, shifted of course... 3850 */ 3851 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3852 ASSERT(pidx < sinfo->si_max_pages); 3853 3854 /* if this cookie uses the copy buffer */ 3855 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3856 /* 3857 * NOTE: we know that since this cookie uses the copy buffer, it 3858 * is <= MMU_PAGESIZE. 3859 */ 3860 3861 /* 3862 * get the offset into the page. For the 64-bit kernel, get the 3863 * pfn which we'll use with seg kpm. 3864 */ 3865 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3866 #if defined(__amd64) 3867 /* mfn_to_pfn() is a NOP on i86pc */ 3868 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3869 #endif /* __amd64 */ 3870 3871 /* figure out if the copybuf size is a power of 2 */ 3872 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3873 copybuf_sz_power_2 = B_FALSE; 3874 } else { 3875 copybuf_sz_power_2 = B_TRUE; 3876 } 3877 3878 /* This page uses the copy buffer */ 3879 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3880 3881 /* 3882 * save the copy buffer KVA that we'll use with this page. 3883 * if we still fit within the copybuf, it's a simple add. 3884 * otherwise, we need to wrap over using & or % accordingly. 3885 */ 3886 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3887 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3888 *copybuf_used; 3889 } else { 3890 if (copybuf_sz_power_2) { 3891 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3892 (uintptr_t)dma->dp_cbaddr + 3893 (*copybuf_used & 3894 (dma->dp_copybuf_size - 1))); 3895 } else { 3896 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3897 (uintptr_t)dma->dp_cbaddr + 3898 (*copybuf_used % dma->dp_copybuf_size)); 3899 } 3900 } 3901 3902 /* 3903 * over write the cookie physical address with the address of 3904 * the physical address of the copy buffer page that we will 3905 * use. 3906 */ 3907 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3908 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3909 3910 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3911 3912 /* if we have a kernel VA, it's easy, just save that address */ 3913 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3914 (sinfo->si_asp == &kas)) { 3915 /* 3916 * save away the page aligned virtual address of the 3917 * driver buffer. Offsets are handled in the sync code. 3918 */ 3919 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3920 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3921 & MMU_PAGEMASK); 3922 #if !defined(__amd64) 3923 /* 3924 * we didn't need to, and will never need to map this 3925 * page. 3926 */ 3927 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3928 #endif 3929 3930 /* we don't have a kernel VA. We need one for the bcopy. */ 3931 } else { 3932 #if defined(__amd64) 3933 /* 3934 * for the 64-bit kernel, it's easy. We use seg kpm to 3935 * get a Kernel VA for the corresponding pfn. 3936 */ 3937 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3938 #else 3939 /* 3940 * for the 32-bit kernel, this is a pain. First we'll 3941 * save away the page_t or user VA for this page. This 3942 * is needed in rootnex_dma_win() when we switch to a 3943 * new window which requires us to re-map the copy 3944 * buffer. 3945 */ 3946 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3947 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3948 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3949 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3950 } else if (pplist != NULL) { 3951 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3952 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3953 } else { 3954 dma->dp_pgmap[pidx].pm_pp = NULL; 3955 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3956 (((uintptr_t) 3957 dmar_object->dmao_obj.virt_obj.v_addr + 3958 cur_offset) & MMU_PAGEMASK); 3959 } 3960 3961 /* 3962 * save away the page aligned virtual address which was 3963 * allocated from the kernel heap arena (taking into 3964 * account if we need more copy buffer than we alloced 3965 * and use multiple windows to handle this, i.e. &,%). 3966 * NOTE: there isn't and physical memory backing up this 3967 * virtual address space currently. 3968 */ 3969 if ((*copybuf_used + MMU_PAGESIZE) <= 3970 dma->dp_copybuf_size) { 3971 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3972 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3973 MMU_PAGEMASK); 3974 } else { 3975 if (copybuf_sz_power_2) { 3976 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3977 (((uintptr_t)dma->dp_kva + 3978 (*copybuf_used & 3979 (dma->dp_copybuf_size - 1))) & 3980 MMU_PAGEMASK); 3981 } else { 3982 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3983 (((uintptr_t)dma->dp_kva + 3984 (*copybuf_used % 3985 dma->dp_copybuf_size)) & 3986 MMU_PAGEMASK); 3987 } 3988 } 3989 3990 /* 3991 * if we haven't used up the available copy buffer yet, 3992 * map the kva to the physical page. 3993 */ 3994 if (!dma->dp_cb_remaping && ((*copybuf_used + 3995 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3996 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3997 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3998 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3999 dma->dp_pgmap[pidx].pm_kaddr); 4000 } else { 4001 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4002 sinfo->si_asp, 4003 dma->dp_pgmap[pidx].pm_kaddr); 4004 } 4005 4006 /* 4007 * we've used up the available copy buffer, this page 4008 * will have to be mapped during rootnex_dma_win() when 4009 * we switch to a new window which requires a re-map 4010 * the copy buffer. (32-bit kernel only) 4011 */ 4012 } else { 4013 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4014 } 4015 #endif 4016 /* go to the next page_t */ 4017 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4018 *cur_pp = (*cur_pp)->p_next; 4019 } 4020 } 4021 4022 /* add to the copy buffer count */ 4023 *copybuf_used += MMU_PAGESIZE; 4024 4025 /* 4026 * This cookie doesn't use the copy buffer. Walk through the pages this 4027 * cookie occupies to reflect this. 4028 */ 4029 } else { 4030 /* 4031 * figure out how many pages the cookie occupies. We need to 4032 * use the original page offset of the buffer and the cookies 4033 * offset in the buffer to do this. 4034 */ 4035 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4036 pcnt = mmu_btopr(cookie->dmac_size + poff); 4037 4038 while (pcnt > 0) { 4039 #if !defined(__amd64) 4040 /* 4041 * the 32-bit kernel doesn't have seg kpm, so we need 4042 * to map in the driver buffer (if it didn't come down 4043 * with a kernel VA) on the fly. Since this page doesn't 4044 * use the copy buffer, it's not, or will it ever, have 4045 * to be mapped in. 4046 */ 4047 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4048 #endif 4049 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4050 4051 /* 4052 * we need to update pidx and cur_pp or we'll loose 4053 * track of where we are. 4054 */ 4055 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4056 *cur_pp = (*cur_pp)->p_next; 4057 } 4058 pidx++; 4059 pcnt--; 4060 } 4061 } 4062 } 4063 4064 4065 /* 4066 * rootnex_sgllen_window_boundary() 4067 * Called in the bind slow path when the next cookie causes us to exceed (in 4068 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4069 * length supported by the DMA H/W. 4070 */ 4071 static int 4072 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4073 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4074 off_t cur_offset) 4075 { 4076 off_t new_offset; 4077 size_t trim_sz; 4078 off_t coffset; 4079 4080 4081 /* 4082 * if we know we'll never have to trim, it's pretty easy. Just move to 4083 * the next window and init it. We're done. 4084 */ 4085 if (!dma->dp_trim_required) { 4086 (*windowp)++; 4087 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4088 (*windowp)->wd_cookie_cnt++; 4089 (*windowp)->wd_size = cookie->dmac_size; 4090 return (DDI_SUCCESS); 4091 } 4092 4093 /* figure out how much we need to trim from the window */ 4094 ASSERT(attr->dma_attr_granular != 0); 4095 if (dma->dp_granularity_power_2) { 4096 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4097 } else { 4098 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4099 } 4100 4101 /* The window's a whole multiple of granularity. We're done */ 4102 if (trim_sz == 0) { 4103 (*windowp)++; 4104 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4105 (*windowp)->wd_cookie_cnt++; 4106 (*windowp)->wd_size = cookie->dmac_size; 4107 return (DDI_SUCCESS); 4108 } 4109 4110 /* 4111 * The window's not a whole multiple of granularity, since we know this 4112 * is due to the sgllen, we need to go back to the last cookie and trim 4113 * that one, add the left over part of the old cookie into the new 4114 * window, and then add in the new cookie into the new window. 4115 */ 4116 4117 /* 4118 * make sure the driver isn't making us do something bad... Trimming and 4119 * sgllen == 1 don't go together. 4120 */ 4121 if (attr->dma_attr_sgllen == 1) { 4122 return (DDI_DMA_NOMAPPING); 4123 } 4124 4125 /* 4126 * first, setup the current window to account for the trim. Need to go 4127 * back to the last cookie for this. 4128 */ 4129 cookie--; 4130 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4131 (*windowp)->wd_trim.tr_last_cookie = cookie; 4132 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4133 ASSERT(cookie->dmac_size > trim_sz); 4134 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4135 (*windowp)->wd_size -= trim_sz; 4136 4137 /* save the buffer offsets for the next window */ 4138 coffset = cookie->dmac_size - trim_sz; 4139 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4140 4141 /* 4142 * set this now in case this is the first window. all other cases are 4143 * set in dma_win() 4144 */ 4145 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4146 4147 /* 4148 * initialize the next window using what's left over in the previous 4149 * cookie. 4150 */ 4151 (*windowp)++; 4152 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4153 (*windowp)->wd_cookie_cnt++; 4154 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4155 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4156 (*windowp)->wd_trim.tr_first_size = trim_sz; 4157 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4158 (*windowp)->wd_dosync = B_TRUE; 4159 } 4160 4161 /* 4162 * now go back to the current cookie and add it to the new window. set 4163 * the new window size to the what was left over from the previous 4164 * cookie and what's in the current cookie. 4165 */ 4166 cookie++; 4167 (*windowp)->wd_cookie_cnt++; 4168 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4169 4170 /* 4171 * trim plus the next cookie could put us over maxxfer (a cookie can be 4172 * a max size of maxxfer). Handle that case. 4173 */ 4174 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4175 /* 4176 * maxxfer is already a whole multiple of granularity, and this 4177 * trim will be <= the previous trim (since a cookie can't be 4178 * larger than maxxfer). Make things simple here. 4179 */ 4180 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4181 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4182 (*windowp)->wd_trim.tr_last_cookie = cookie; 4183 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4184 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4185 (*windowp)->wd_size -= trim_sz; 4186 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4187 4188 /* save the buffer offsets for the next window */ 4189 coffset = cookie->dmac_size - trim_sz; 4190 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4191 4192 /* setup the next window */ 4193 (*windowp)++; 4194 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4195 (*windowp)->wd_cookie_cnt++; 4196 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4197 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4198 coffset; 4199 (*windowp)->wd_trim.tr_first_size = trim_sz; 4200 } 4201 4202 return (DDI_SUCCESS); 4203 } 4204 4205 4206 /* 4207 * rootnex_copybuf_window_boundary() 4208 * Called in bind slowpath when we get to a window boundary because we used 4209 * up all the copy buffer that we have. 4210 */ 4211 static int 4212 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4213 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4214 size_t *copybuf_used) 4215 { 4216 rootnex_sglinfo_t *sinfo; 4217 off_t new_offset; 4218 size_t trim_sz; 4219 paddr_t paddr; 4220 off_t coffset; 4221 uint_t pidx; 4222 off_t poff; 4223 4224 4225 sinfo = &dma->dp_sglinfo; 4226 4227 /* 4228 * the copy buffer should be a whole multiple of page size. We know that 4229 * this cookie is <= MMU_PAGESIZE. 4230 */ 4231 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4232 4233 /* 4234 * from now on, all new windows in this bind need to be re-mapped during 4235 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4236 * space... 4237 */ 4238 #if !defined(__amd64) 4239 dma->dp_cb_remaping = B_TRUE; 4240 #endif 4241 4242 /* reset copybuf used */ 4243 *copybuf_used = 0; 4244 4245 /* 4246 * if we don't have to trim (since granularity is set to 1), go to the 4247 * next window and add the current cookie to it. We know the current 4248 * cookie uses the copy buffer since we're in this code path. 4249 */ 4250 if (!dma->dp_trim_required) { 4251 (*windowp)++; 4252 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4253 4254 /* Add this cookie to the new window */ 4255 (*windowp)->wd_cookie_cnt++; 4256 (*windowp)->wd_size += cookie->dmac_size; 4257 *copybuf_used += MMU_PAGESIZE; 4258 return (DDI_SUCCESS); 4259 } 4260 4261 /* 4262 * *** may need to trim, figure it out. 4263 */ 4264 4265 /* figure out how much we need to trim from the window */ 4266 if (dma->dp_granularity_power_2) { 4267 trim_sz = (*windowp)->wd_size & 4268 (hp->dmai_attr.dma_attr_granular - 1); 4269 } else { 4270 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4271 } 4272 4273 /* 4274 * if the window's a whole multiple of granularity, go to the next 4275 * window, init it, then add in the current cookie. We know the current 4276 * cookie uses the copy buffer since we're in this code path. 4277 */ 4278 if (trim_sz == 0) { 4279 (*windowp)++; 4280 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4281 4282 /* Add this cookie to the new window */ 4283 (*windowp)->wd_cookie_cnt++; 4284 (*windowp)->wd_size += cookie->dmac_size; 4285 *copybuf_used += MMU_PAGESIZE; 4286 return (DDI_SUCCESS); 4287 } 4288 4289 /* 4290 * *** We figured it out, we definitly need to trim 4291 */ 4292 4293 /* 4294 * make sure the driver isn't making us do something bad... 4295 * Trimming and sgllen == 1 don't go together. 4296 */ 4297 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4298 return (DDI_DMA_NOMAPPING); 4299 } 4300 4301 /* 4302 * first, setup the current window to account for the trim. Need to go 4303 * back to the last cookie for this. Some of the last cookie will be in 4304 * the current window, and some of the last cookie will be in the new 4305 * window. All of the current cookie will be in the new window. 4306 */ 4307 cookie--; 4308 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4309 (*windowp)->wd_trim.tr_last_cookie = cookie; 4310 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4311 ASSERT(cookie->dmac_size > trim_sz); 4312 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4313 (*windowp)->wd_size -= trim_sz; 4314 4315 /* 4316 * we're trimming the last cookie (not the current cookie). So that 4317 * last cookie may have or may not have been using the copy buffer ( 4318 * we know the cookie passed in uses the copy buffer since we're in 4319 * this code path). 4320 * 4321 * If the last cookie doesn't use the copy buffer, nothing special to 4322 * do. However, if it does uses the copy buffer, it will be both the 4323 * last page in the current window and the first page in the next 4324 * window. Since we are reusing the copy buffer (and KVA space on the 4325 * 32-bit kernel), this page will use the end of the copy buffer in the 4326 * current window, and the start of the copy buffer in the next window. 4327 * Track that info... The cookie physical address was already set to 4328 * the copy buffer physical address in setup_cookie.. 4329 */ 4330 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4331 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4332 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4333 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4334 (*windowp)->wd_trim.tr_last_pidx = pidx; 4335 (*windowp)->wd_trim.tr_last_cbaddr = 4336 dma->dp_pgmap[pidx].pm_cbaddr; 4337 #if !defined(__amd64) 4338 (*windowp)->wd_trim.tr_last_kaddr = 4339 dma->dp_pgmap[pidx].pm_kaddr; 4340 #endif 4341 } 4342 4343 /* save the buffer offsets for the next window */ 4344 coffset = cookie->dmac_size - trim_sz; 4345 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4346 4347 /* 4348 * set this now in case this is the first window. all other cases are 4349 * set in dma_win() 4350 */ 4351 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4352 4353 /* 4354 * initialize the next window using what's left over in the previous 4355 * cookie. 4356 */ 4357 (*windowp)++; 4358 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4359 (*windowp)->wd_cookie_cnt++; 4360 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4361 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4362 (*windowp)->wd_trim.tr_first_size = trim_sz; 4363 4364 /* 4365 * again, we're tracking if the last cookie uses the copy buffer. 4366 * read the comment above for more info on why we need to track 4367 * additional state. 4368 * 4369 * For the first cookie in the new window, we need reset the physical 4370 * address to DMA into to the start of the copy buffer plus any 4371 * initial page offset which may be present. 4372 */ 4373 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4374 (*windowp)->wd_dosync = B_TRUE; 4375 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4376 (*windowp)->wd_trim.tr_first_pidx = pidx; 4377 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4378 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4379 4380 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4381 poff; 4382 (*windowp)->wd_trim.tr_first_paddr = 4383 ROOTNEX_PADDR_TO_RBASE(paddr); 4384 4385 #if !defined(__amd64) 4386 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4387 #endif 4388 /* account for the cookie copybuf usage in the new window */ 4389 *copybuf_used += MMU_PAGESIZE; 4390 4391 /* 4392 * every piece of code has to have a hack, and here is this 4393 * ones :-) 4394 * 4395 * There is a complex interaction between setup_cookie and the 4396 * copybuf window boundary. The complexity had to be in either 4397 * the maxxfer window, or the copybuf window, and I chose the 4398 * copybuf code. 4399 * 4400 * So in this code path, we have taken the last cookie, 4401 * virtually broken it in half due to the trim, and it happens 4402 * to use the copybuf which further complicates life. At the 4403 * same time, we have already setup the current cookie, which 4404 * is now wrong. More background info: the current cookie uses 4405 * the copybuf, so it is only a page long max. So we need to 4406 * fix the current cookies copy buffer address, physical 4407 * address, and kva for the 32-bit kernel. We due this by 4408 * bumping them by page size (of course, we can't due this on 4409 * the physical address since the copy buffer may not be 4410 * physically contiguous). 4411 */ 4412 cookie++; 4413 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4414 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4415 4416 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4417 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4418 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4419 4420 #if !defined(__amd64) 4421 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4422 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4423 #endif 4424 } else { 4425 /* go back to the current cookie */ 4426 cookie++; 4427 } 4428 4429 /* 4430 * add the current cookie to the new window. set the new window size to 4431 * the what was left over from the previous cookie and what's in the 4432 * current cookie. 4433 */ 4434 (*windowp)->wd_cookie_cnt++; 4435 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4436 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4437 4438 /* 4439 * we know that the cookie passed in always uses the copy buffer. We 4440 * wouldn't be here if it didn't. 4441 */ 4442 *copybuf_used += MMU_PAGESIZE; 4443 4444 return (DDI_SUCCESS); 4445 } 4446 4447 4448 /* 4449 * rootnex_maxxfer_window_boundary() 4450 * Called in bind slowpath when we get to a window boundary because we will 4451 * go over maxxfer. 4452 */ 4453 static int 4454 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4455 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4456 { 4457 size_t dmac_size; 4458 off_t new_offset; 4459 size_t trim_sz; 4460 off_t coffset; 4461 4462 4463 /* 4464 * calculate how much we have to trim off of the current cookie to equal 4465 * maxxfer. We don't have to account for granularity here since our 4466 * maxxfer already takes that into account. 4467 */ 4468 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4469 ASSERT(trim_sz <= cookie->dmac_size); 4470 ASSERT(trim_sz <= dma->dp_maxxfer); 4471 4472 /* save cookie size since we need it later and we might change it */ 4473 dmac_size = cookie->dmac_size; 4474 4475 /* 4476 * if we're not trimming the entire cookie, setup the current window to 4477 * account for the trim. 4478 */ 4479 if (trim_sz < cookie->dmac_size) { 4480 (*windowp)->wd_cookie_cnt++; 4481 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4482 (*windowp)->wd_trim.tr_last_cookie = cookie; 4483 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4484 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4485 (*windowp)->wd_size = dma->dp_maxxfer; 4486 4487 /* 4488 * set the adjusted cookie size now in case this is the first 4489 * window. All other windows are taken care of in get win 4490 */ 4491 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4492 } 4493 4494 /* 4495 * coffset is the current offset within the cookie, new_offset is the 4496 * current offset with the entire buffer. 4497 */ 4498 coffset = dmac_size - trim_sz; 4499 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4500 4501 /* initialize the next window */ 4502 (*windowp)++; 4503 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4504 (*windowp)->wd_cookie_cnt++; 4505 (*windowp)->wd_size = trim_sz; 4506 if (trim_sz < dmac_size) { 4507 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4508 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4509 coffset; 4510 (*windowp)->wd_trim.tr_first_size = trim_sz; 4511 } 4512 4513 return (DDI_SUCCESS); 4514 } 4515 4516 4517 /*ARGSUSED*/ 4518 static int 4519 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4520 off_t off, size_t len, uint_t cache_flags) 4521 { 4522 rootnex_sglinfo_t *sinfo; 4523 rootnex_pgmap_t *cbpage; 4524 rootnex_window_t *win; 4525 ddi_dma_impl_t *hp; 4526 rootnex_dma_t *dma; 4527 caddr_t fromaddr; 4528 caddr_t toaddr; 4529 uint_t psize; 4530 off_t offset; 4531 uint_t pidx; 4532 size_t size; 4533 off_t poff; 4534 int e; 4535 4536 4537 hp = (ddi_dma_impl_t *)handle; 4538 dma = (rootnex_dma_t *)hp->dmai_private; 4539 sinfo = &dma->dp_sglinfo; 4540 4541 /* 4542 * if we don't have any windows, we don't need to sync. A copybuf 4543 * will cause us to have at least one window. 4544 */ 4545 if (dma->dp_window == NULL) { 4546 return (DDI_SUCCESS); 4547 } 4548 4549 /* This window may not need to be sync'd */ 4550 win = &dma->dp_window[dma->dp_current_win]; 4551 if (!win->wd_dosync) { 4552 return (DDI_SUCCESS); 4553 } 4554 4555 /* handle off and len special cases */ 4556 if ((off == 0) || (rootnex_sync_ignore_params)) { 4557 offset = win->wd_offset; 4558 } else { 4559 offset = off; 4560 } 4561 if ((len == 0) || (rootnex_sync_ignore_params)) { 4562 size = win->wd_size; 4563 } else { 4564 size = len; 4565 } 4566 4567 /* check the sync args to make sure they make a little sense */ 4568 if (rootnex_sync_check_parms) { 4569 e = rootnex_valid_sync_parms(hp, win, offset, size, 4570 cache_flags); 4571 if (e != DDI_SUCCESS) { 4572 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4573 return (DDI_FAILURE); 4574 } 4575 } 4576 4577 /* 4578 * special case the first page to handle the offset into the page. The 4579 * offset to the current page for our buffer is the offset into the 4580 * first page of the buffer plus our current offset into the buffer 4581 * itself, masked of course. 4582 */ 4583 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4584 psize = MIN((MMU_PAGESIZE - poff), size); 4585 4586 /* go through all the pages that we want to sync */ 4587 while (size > 0) { 4588 /* 4589 * Calculate the page index relative to the start of the buffer. 4590 * The index to the current page for our buffer is the offset 4591 * into the first page of the buffer plus our current offset 4592 * into the buffer itself, shifted of course... 4593 */ 4594 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4595 ASSERT(pidx < sinfo->si_max_pages); 4596 4597 /* 4598 * if this page uses the copy buffer, we need to sync it, 4599 * otherwise, go on to the next page. 4600 */ 4601 cbpage = &dma->dp_pgmap[pidx]; 4602 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4603 (cbpage->pm_uses_copybuf == B_FALSE)); 4604 if (cbpage->pm_uses_copybuf) { 4605 /* cbaddr and kaddr should be page aligned */ 4606 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4607 MMU_PAGEOFFSET) == 0); 4608 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4609 MMU_PAGEOFFSET) == 0); 4610 4611 /* 4612 * if we're copying for the device, we are going to 4613 * copy from the drivers buffer and to the rootnex 4614 * allocated copy buffer. 4615 */ 4616 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4617 fromaddr = cbpage->pm_kaddr + poff; 4618 toaddr = cbpage->pm_cbaddr + poff; 4619 ROOTNEX_DPROBE2(rootnex__sync__dev, 4620 dev_info_t *, dma->dp_dip, size_t, psize); 4621 4622 /* 4623 * if we're copying for the cpu/kernel, we are going to 4624 * copy from the rootnex allocated copy buffer to the 4625 * drivers buffer. 4626 */ 4627 } else { 4628 fromaddr = cbpage->pm_cbaddr + poff; 4629 toaddr = cbpage->pm_kaddr + poff; 4630 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4631 dev_info_t *, dma->dp_dip, size_t, psize); 4632 } 4633 4634 bcopy(fromaddr, toaddr, psize); 4635 } 4636 4637 /* 4638 * decrement size until we're done, update our offset into the 4639 * buffer, and get the next page size. 4640 */ 4641 size -= psize; 4642 offset += psize; 4643 psize = MIN(MMU_PAGESIZE, size); 4644 4645 /* page offset is zero for the rest of this loop */ 4646 poff = 0; 4647 } 4648 4649 return (DDI_SUCCESS); 4650 } 4651 4652 /* 4653 * rootnex_dma_sync() 4654 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4655 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4656 * is set, ddi_dma_sync() returns immediately passing back success. 4657 */ 4658 /*ARGSUSED*/ 4659 static int 4660 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4661 off_t off, size_t len, uint_t cache_flags) 4662 { 4663 #if defined(__amd64) && !defined(__xpv) 4664 if (IOMMU_USED(rdip)) { 4665 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4666 cache_flags)); 4667 } 4668 #endif 4669 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4670 cache_flags)); 4671 } 4672 4673 /* 4674 * rootnex_valid_sync_parms() 4675 * checks the parameters passed to sync to verify they are correct. 4676 */ 4677 static int 4678 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4679 off_t offset, size_t size, uint_t cache_flags) 4680 { 4681 off_t woffset; 4682 4683 4684 /* 4685 * the first part of the test to make sure the offset passed in is 4686 * within the window. 4687 */ 4688 if (offset < win->wd_offset) { 4689 return (DDI_FAILURE); 4690 } 4691 4692 /* 4693 * second and last part of the test to make sure the offset and length 4694 * passed in is within the window. 4695 */ 4696 woffset = offset - win->wd_offset; 4697 if ((woffset + size) > win->wd_size) { 4698 return (DDI_FAILURE); 4699 } 4700 4701 /* 4702 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4703 * be set too. 4704 */ 4705 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4706 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4707 return (DDI_SUCCESS); 4708 } 4709 4710 /* 4711 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4712 * should be set. Also DDI_DMA_READ should be set in the flags. 4713 */ 4714 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4715 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4716 (hp->dmai_rflags & DDI_DMA_READ)) { 4717 return (DDI_SUCCESS); 4718 } 4719 4720 return (DDI_FAILURE); 4721 } 4722 4723 4724 /*ARGSUSED*/ 4725 static int 4726 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4727 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4728 uint_t *ccountp) 4729 { 4730 rootnex_window_t *window; 4731 rootnex_trim_t *trim; 4732 ddi_dma_impl_t *hp; 4733 rootnex_dma_t *dma; 4734 ddi_dma_obj_t *dmao; 4735 #if !defined(__amd64) 4736 rootnex_sglinfo_t *sinfo; 4737 rootnex_pgmap_t *pmap; 4738 uint_t pidx; 4739 uint_t pcnt; 4740 off_t poff; 4741 int i; 4742 #endif 4743 4744 4745 hp = (ddi_dma_impl_t *)handle; 4746 dma = (rootnex_dma_t *)hp->dmai_private; 4747 #if !defined(__amd64) 4748 sinfo = &dma->dp_sglinfo; 4749 #endif 4750 4751 /* If we try and get a window which doesn't exist, return failure */ 4752 if (win >= hp->dmai_nwin) { 4753 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4754 return (DDI_FAILURE); 4755 } 4756 4757 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4758 4759 /* 4760 * if we don't have any windows, and they're asking for the first 4761 * window, setup the cookie pointer to the first cookie in the bind. 4762 * setup our return values, then increment the cookie since we return 4763 * the first cookie on the stack. 4764 */ 4765 if (dma->dp_window == NULL) { 4766 if (win != 0) { 4767 ROOTNEX_DPROF_INC( 4768 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4769 return (DDI_FAILURE); 4770 } 4771 hp->dmai_cookie = dma->dp_cookies; 4772 *offp = 0; 4773 *lenp = dmao->dmao_size; 4774 *ccountp = dma->dp_sglinfo.si_sgl_size; 4775 *cookiep = hp->dmai_cookie[0]; 4776 hp->dmai_cookie++; 4777 return (DDI_SUCCESS); 4778 } 4779 4780 /* sync the old window before moving on to the new one */ 4781 window = &dma->dp_window[dma->dp_current_win]; 4782 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4783 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4784 DDI_DMA_SYNC_FORCPU); 4785 } 4786 4787 #if !defined(__amd64) 4788 /* 4789 * before we move to the next window, if we need to re-map, unmap all 4790 * the pages in this window. 4791 */ 4792 if (dma->dp_cb_remaping) { 4793 /* 4794 * If we switch to this window again, we'll need to map in 4795 * on the fly next time. 4796 */ 4797 window->wd_remap_copybuf = B_TRUE; 4798 4799 /* 4800 * calculate the page index into the buffer where this window 4801 * starts, and the number of pages this window takes up. 4802 */ 4803 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4804 MMU_PAGESHIFT; 4805 poff = (sinfo->si_buf_offset + window->wd_offset) & 4806 MMU_PAGEOFFSET; 4807 pcnt = mmu_btopr(window->wd_size + poff); 4808 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4809 4810 /* unmap pages which are currently mapped in this window */ 4811 for (i = 0; i < pcnt; i++) { 4812 if (dma->dp_pgmap[pidx].pm_mapped) { 4813 hat_unload(kas.a_hat, 4814 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4815 HAT_UNLOAD); 4816 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4817 } 4818 pidx++; 4819 } 4820 } 4821 #endif 4822 4823 /* 4824 * Move to the new window. 4825 * NOTE: current_win must be set for sync to work right 4826 */ 4827 dma->dp_current_win = win; 4828 window = &dma->dp_window[win]; 4829 4830 /* if needed, adjust the first and/or last cookies for trim */ 4831 trim = &window->wd_trim; 4832 if (trim->tr_trim_first) { 4833 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4834 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4835 #if !defined(__amd64) 4836 window->wd_first_cookie->dmac_type = 4837 (window->wd_first_cookie->dmac_type & 4838 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4839 #endif 4840 if (trim->tr_first_copybuf_win) { 4841 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4842 trim->tr_first_cbaddr; 4843 #if !defined(__amd64) 4844 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4845 trim->tr_first_kaddr; 4846 #endif 4847 } 4848 } 4849 if (trim->tr_trim_last) { 4850 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4851 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4852 if (trim->tr_last_copybuf_win) { 4853 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4854 trim->tr_last_cbaddr; 4855 #if !defined(__amd64) 4856 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4857 trim->tr_last_kaddr; 4858 #endif 4859 } 4860 } 4861 4862 /* 4863 * setup the cookie pointer to the first cookie in the window. setup 4864 * our return values, then increment the cookie since we return the 4865 * first cookie on the stack. 4866 */ 4867 hp->dmai_cookie = window->wd_first_cookie; 4868 *offp = window->wd_offset; 4869 *lenp = window->wd_size; 4870 *ccountp = window->wd_cookie_cnt; 4871 *cookiep = hp->dmai_cookie[0]; 4872 hp->dmai_cookie++; 4873 4874 #if !defined(__amd64) 4875 /* re-map copybuf if required for this window */ 4876 if (dma->dp_cb_remaping) { 4877 /* 4878 * calculate the page index into the buffer where this 4879 * window starts. 4880 */ 4881 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4882 MMU_PAGESHIFT; 4883 ASSERT(pidx < sinfo->si_max_pages); 4884 4885 /* 4886 * the first page can get unmapped if it's shared with the 4887 * previous window. Even if the rest of this window is already 4888 * mapped in, we need to still check this one. 4889 */ 4890 pmap = &dma->dp_pgmap[pidx]; 4891 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4892 if (pmap->pm_pp != NULL) { 4893 pmap->pm_mapped = B_TRUE; 4894 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4895 } else if (pmap->pm_vaddr != NULL) { 4896 pmap->pm_mapped = B_TRUE; 4897 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4898 pmap->pm_kaddr); 4899 } 4900 } 4901 pidx++; 4902 4903 /* map in the rest of the pages if required */ 4904 if (window->wd_remap_copybuf) { 4905 window->wd_remap_copybuf = B_FALSE; 4906 4907 /* figure out many pages this window takes up */ 4908 poff = (sinfo->si_buf_offset + window->wd_offset) & 4909 MMU_PAGEOFFSET; 4910 pcnt = mmu_btopr(window->wd_size + poff); 4911 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4912 4913 /* map pages which require it */ 4914 for (i = 1; i < pcnt; i++) { 4915 pmap = &dma->dp_pgmap[pidx]; 4916 if (pmap->pm_uses_copybuf) { 4917 ASSERT(pmap->pm_mapped == B_FALSE); 4918 if (pmap->pm_pp != NULL) { 4919 pmap->pm_mapped = B_TRUE; 4920 i86_pp_map(pmap->pm_pp, 4921 pmap->pm_kaddr); 4922 } else if (pmap->pm_vaddr != NULL) { 4923 pmap->pm_mapped = B_TRUE; 4924 i86_va_map(pmap->pm_vaddr, 4925 sinfo->si_asp, 4926 pmap->pm_kaddr); 4927 } 4928 } 4929 pidx++; 4930 } 4931 } 4932 } 4933 #endif 4934 4935 /* if the new window uses the copy buffer, sync it for the device */ 4936 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4937 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4938 DDI_DMA_SYNC_FORDEV); 4939 } 4940 4941 return (DDI_SUCCESS); 4942 } 4943 4944 /* 4945 * rootnex_dma_win() 4946 * called from ddi_dma_getwin() 4947 */ 4948 /*ARGSUSED*/ 4949 static int 4950 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4951 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4952 uint_t *ccountp) 4953 { 4954 #if defined(__amd64) && !defined(__xpv) 4955 if (IOMMU_USED(rdip)) { 4956 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4957 cookiep, ccountp)); 4958 } 4959 #endif 4960 4961 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4962 cookiep, ccountp)); 4963 } 4964 4965 #if defined(__amd64) && !defined(__xpv) 4966 /*ARGSUSED*/ 4967 static int 4968 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4969 ddi_dma_handle_t handle, void *v) 4970 { 4971 ddi_dma_impl_t *hp; 4972 rootnex_dma_t *dma; 4973 4974 hp = (ddi_dma_impl_t *)handle; 4975 dma = (rootnex_dma_t *)hp->dmai_private; 4976 dma->dp_iommu_private = v; 4977 4978 return (DDI_SUCCESS); 4979 } 4980 4981 /*ARGSUSED*/ 4982 static void * 4983 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4984 ddi_dma_handle_t handle) 4985 { 4986 ddi_dma_impl_t *hp; 4987 rootnex_dma_t *dma; 4988 4989 hp = (ddi_dma_impl_t *)handle; 4990 dma = (rootnex_dma_t *)hp->dmai_private; 4991 4992 return (dma->dp_iommu_private); 4993 } 4994 #endif 4995 4996 /* 4997 * ************************ 4998 * obsoleted dma routines 4999 * ************************ 5000 */ 5001 5002 /* 5003 * rootnex_dma_mctl() 5004 * 5005 * We don't support this legacy interface any more on x86. 5006 */ 5007 /* ARGSUSED */ 5008 static int 5009 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5010 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5011 uint_t cache_flags) 5012 { 5013 /* 5014 * The only thing dma_mctl is usef for anymore is legacy SPARC 5015 * dvma and sbus-specific routines. 5016 */ 5017 return (DDI_FAILURE); 5018 } 5019 5020 /* 5021 * ********* 5022 * FMA Code 5023 * ********* 5024 */ 5025 5026 /* 5027 * rootnex_fm_init() 5028 * FMA init busop 5029 */ 5030 /* ARGSUSED */ 5031 static int 5032 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5033 ddi_iblock_cookie_t *ibc) 5034 { 5035 *ibc = rootnex_state->r_err_ibc; 5036 5037 return (ddi_system_fmcap); 5038 } 5039 5040 /* 5041 * rootnex_dma_check() 5042 * Function called after a dma fault occurred to find out whether the 5043 * fault address is associated with a driver that is able to handle faults 5044 * and recover from faults. 5045 */ 5046 /* ARGSUSED */ 5047 static int 5048 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5049 const void *not_used) 5050 { 5051 rootnex_window_t *window; 5052 uint64_t start_addr; 5053 uint64_t fault_addr; 5054 ddi_dma_impl_t *hp; 5055 rootnex_dma_t *dma; 5056 uint64_t end_addr; 5057 size_t csize; 5058 int i; 5059 int j; 5060 5061 5062 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5063 hp = (ddi_dma_impl_t *)handle; 5064 ASSERT(hp); 5065 5066 dma = (rootnex_dma_t *)hp->dmai_private; 5067 5068 /* Get the address that we need to search for */ 5069 fault_addr = *(uint64_t *)addr; 5070 5071 /* 5072 * if we don't have any windows, we can just walk through all the 5073 * cookies. 5074 */ 5075 if (dma->dp_window == NULL) { 5076 /* for each cookie */ 5077 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5078 /* 5079 * if the faulted address is within the physical address 5080 * range of the cookie, return DDI_FM_NONFATAL. 5081 */ 5082 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5083 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5084 dma->dp_cookies[i].dmac_size))) { 5085 return (DDI_FM_NONFATAL); 5086 } 5087 } 5088 5089 /* fault_addr not within this DMA handle */ 5090 return (DDI_FM_UNKNOWN); 5091 } 5092 5093 /* we have mutiple windows, walk through each window */ 5094 for (i = 0; i < hp->dmai_nwin; i++) { 5095 window = &dma->dp_window[i]; 5096 5097 /* Go through all the cookies in the window */ 5098 for (j = 0; j < window->wd_cookie_cnt; j++) { 5099 5100 start_addr = window->wd_first_cookie[j].dmac_laddress; 5101 csize = window->wd_first_cookie[j].dmac_size; 5102 5103 /* 5104 * if we are trimming the first cookie in the window, 5105 * and this is the first cookie, adjust the start 5106 * address and size of the cookie to account for the 5107 * trim. 5108 */ 5109 if (window->wd_trim.tr_trim_first && (j == 0)) { 5110 start_addr = window->wd_trim.tr_first_paddr; 5111 csize = window->wd_trim.tr_first_size; 5112 } 5113 5114 /* 5115 * if we are trimming the last cookie in the window, 5116 * and this is the last cookie, adjust the start 5117 * address and size of the cookie to account for the 5118 * trim. 5119 */ 5120 if (window->wd_trim.tr_trim_last && 5121 (j == (window->wd_cookie_cnt - 1))) { 5122 start_addr = window->wd_trim.tr_last_paddr; 5123 csize = window->wd_trim.tr_last_size; 5124 } 5125 5126 end_addr = start_addr + csize; 5127 5128 /* 5129 * if the faulted address is within the physical 5130 * address of the cookie, return DDI_FM_NONFATAL. 5131 */ 5132 if ((fault_addr >= start_addr) && 5133 (fault_addr <= end_addr)) { 5134 return (DDI_FM_NONFATAL); 5135 } 5136 } 5137 } 5138 5139 /* fault_addr not within this DMA handle */ 5140 return (DDI_FM_UNKNOWN); 5141 } 5142 5143 /*ARGSUSED*/ 5144 static int 5145 rootnex_quiesce(dev_info_t *dip) 5146 { 5147 #if defined(__amd64) && !defined(__xpv) 5148 return (immu_quiesce()); 5149 #else 5150 return (DDI_SUCCESS); 5151 #endif 5152 } 5153 5154 #if defined(__xpv) 5155 void 5156 immu_init(void) 5157 { 5158 ; 5159 } 5160 5161 void 5162 immu_startup(void) 5163 { 5164 ; 5165 } 5166 /*ARGSUSED*/ 5167 void 5168 immu_physmem_update(uint64_t addr, uint64_t size) 5169 { 5170 ; 5171 } 5172 #endif 5173