1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 */ 29 30 /* 31 * x86 root nexus driver 32 */ 33 34 #include <sys/sysmacros.h> 35 #include <sys/conf.h> 36 #include <sys/autoconf.h> 37 #include <sys/sysmacros.h> 38 #include <sys/debug.h> 39 #include <sys/psw.h> 40 #include <sys/ddidmareq.h> 41 #include <sys/promif.h> 42 #include <sys/devops.h> 43 #include <sys/kmem.h> 44 #include <sys/cmn_err.h> 45 #include <vm/seg.h> 46 #include <vm/seg_kmem.h> 47 #include <vm/seg_dev.h> 48 #include <sys/vmem.h> 49 #include <sys/mman.h> 50 #include <vm/hat.h> 51 #include <vm/as.h> 52 #include <vm/page.h> 53 #include <sys/avintr.h> 54 #include <sys/errno.h> 55 #include <sys/modctl.h> 56 #include <sys/ddi_impldefs.h> 57 #include <sys/sunddi.h> 58 #include <sys/sunndi.h> 59 #include <sys/mach_intr.h> 60 #include <sys/psm.h> 61 #include <sys/ontrap.h> 62 #include <sys/atomic.h> 63 #include <sys/sdt.h> 64 #include <sys/rootnex.h> 65 #include <vm/hat_i86.h> 66 #include <sys/ddifm.h> 67 #include <sys/ddi_isa.h> 68 #include <sys/apic.h> 69 70 #ifdef __xpv 71 #include <sys/bootinfo.h> 72 #include <sys/hypervisor.h> 73 #include <sys/bootconf.h> 74 #include <vm/kboot_mmu.h> 75 #endif 76 77 #if defined(__amd64) && !defined(__xpv) 78 #include <sys/immu.h> 79 #endif 80 81 82 /* 83 * enable/disable extra checking of function parameters. Useful for debugging 84 * drivers. 85 */ 86 #ifdef DEBUG 87 int rootnex_alloc_check_parms = 1; 88 int rootnex_bind_check_parms = 1; 89 int rootnex_bind_check_inuse = 1; 90 int rootnex_unbind_verify_buffer = 0; 91 int rootnex_sync_check_parms = 1; 92 #else 93 int rootnex_alloc_check_parms = 0; 94 int rootnex_bind_check_parms = 0; 95 int rootnex_bind_check_inuse = 0; 96 int rootnex_unbind_verify_buffer = 0; 97 int rootnex_sync_check_parms = 0; 98 #endif 99 100 boolean_t rootnex_dmar_not_setup; 101 102 /* Master Abort and Target Abort panic flag */ 103 int rootnex_fm_ma_ta_panic_flag = 0; 104 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 106 int rootnex_bind_fail = 1; 107 int rootnex_bind_warn = 1; 108 uint8_t *rootnex_warn_list; 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 110 #define ROOTNEX_BIND_WARNING (0x1 << 0) 111 112 /* 113 * revert back to old broken behavior of always sync'ing entire copy buffer. 114 * This is useful if be have a buggy driver which doesn't correctly pass in 115 * the offset and size into ddi_dma_sync(). 116 */ 117 int rootnex_sync_ignore_params = 0; 118 119 /* 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 121 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 122 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 123 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 124 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 125 * (< 8K). We will still need to allocate the copy buffer during bind though 126 * (if we need one). These can only be modified in /etc/system before rootnex 127 * attach. 128 */ 129 #if defined(__amd64) 130 int rootnex_prealloc_cookies = 65; 131 int rootnex_prealloc_windows = 4; 132 int rootnex_prealloc_copybuf = 2; 133 #else 134 int rootnex_prealloc_cookies = 33; 135 int rootnex_prealloc_windows = 4; 136 int rootnex_prealloc_copybuf = 2; 137 #endif 138 139 /* driver global state */ 140 static rootnex_state_t *rootnex_state; 141 142 #ifdef DEBUG 143 /* shortcut to rootnex counters */ 144 static uint64_t *rootnex_cnt; 145 #endif 146 147 /* 148 * XXX - does x86 even need these or are they left over from the SPARC days? 149 */ 150 /* statically defined integer/boolean properties for the root node */ 151 static rootnex_intprop_t rootnex_intprp[] = { 152 { "PAGESIZE", PAGESIZE }, 153 { "MMU_PAGESIZE", MMU_PAGESIZE }, 154 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 155 { DDI_RELATIVE_ADDRESSING, 1 }, 156 }; 157 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 158 159 /* 160 * If we're dom0, we're using a real device so we need to load 161 * the cookies with MFNs instead of PFNs. 162 */ 163 #ifdef __xpv 164 typedef maddr_t rootnex_addr_t; 165 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 166 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 167 #else 168 typedef paddr_t rootnex_addr_t; 169 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 170 #endif 171 172 static struct cb_ops rootnex_cb_ops = { 173 nodev, /* open */ 174 nodev, /* close */ 175 nodev, /* strategy */ 176 nodev, /* print */ 177 nodev, /* dump */ 178 nodev, /* read */ 179 nodev, /* write */ 180 nodev, /* ioctl */ 181 nodev, /* devmap */ 182 nodev, /* mmap */ 183 nodev, /* segmap */ 184 nochpoll, /* chpoll */ 185 ddi_prop_op, /* cb_prop_op */ 186 NULL, /* struct streamtab */ 187 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 188 CB_REV, /* Rev */ 189 nodev, /* cb_aread */ 190 nodev /* cb_awrite */ 191 }; 192 193 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 194 off_t offset, off_t len, caddr_t *vaddrp); 195 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 196 struct hat *hat, struct seg *seg, caddr_t addr, 197 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 198 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 200 ddi_dma_handle_t *handlep); 201 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 202 ddi_dma_handle_t handle); 203 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 205 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 206 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 207 ddi_dma_handle_t handle); 208 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 209 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 210 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 211 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 212 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 213 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 214 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 215 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 216 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 217 ddi_ctl_enum_t ctlop, void *arg, void *result); 218 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 219 ddi_iblock_cookie_t *ibc); 220 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 221 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 222 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 223 void *); 224 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 225 226 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 227 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 228 ddi_dma_handle_t *handlep); 229 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 230 ddi_dma_handle_t handle); 231 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 232 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 233 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 234 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 235 ddi_dma_handle_t handle); 236 #if defined(__amd64) && !defined(__xpv) 237 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 238 ddi_dma_handle_t handle); 239 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 240 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 241 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 242 ddi_dma_cookie_t *cookiep, uint_t ccount); 243 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 244 ddi_dma_handle_t handle); 245 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 246 #endif 247 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 248 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 249 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 250 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 251 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 252 253 #if defined(__amd64) && !defined(__xpv) 254 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 255 ddi_dma_handle_t handle, void *v); 256 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 257 ddi_dma_handle_t handle); 258 #endif 259 260 261 static struct bus_ops rootnex_bus_ops = { 262 BUSO_REV, 263 rootnex_map, 264 NULL, 265 NULL, 266 NULL, 267 rootnex_map_fault, 268 0, 269 rootnex_dma_allochdl, 270 rootnex_dma_freehdl, 271 rootnex_dma_bindhdl, 272 rootnex_dma_unbindhdl, 273 rootnex_dma_sync, 274 rootnex_dma_win, 275 rootnex_dma_mctl, 276 rootnex_ctlops, 277 ddi_bus_prop_op, 278 i_ddi_rootnex_get_eventcookie, 279 i_ddi_rootnex_add_eventcall, 280 i_ddi_rootnex_remove_eventcall, 281 i_ddi_rootnex_post_event, 282 0, /* bus_intr_ctl */ 283 0, /* bus_config */ 284 0, /* bus_unconfig */ 285 rootnex_fm_init, /* bus_fm_init */ 286 NULL, /* bus_fm_fini */ 287 NULL, /* bus_fm_access_enter */ 288 NULL, /* bus_fm_access_exit */ 289 NULL, /* bus_powr */ 290 rootnex_intr_ops /* bus_intr_op */ 291 }; 292 293 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 294 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 295 static int rootnex_quiesce(dev_info_t *dip); 296 297 static struct dev_ops rootnex_ops = { 298 DEVO_REV, 299 0, 300 ddi_no_info, 301 nulldev, 302 nulldev, 303 rootnex_attach, 304 rootnex_detach, 305 nulldev, 306 &rootnex_cb_ops, 307 &rootnex_bus_ops, 308 NULL, 309 rootnex_quiesce, /* quiesce */ 310 }; 311 312 static struct modldrv rootnex_modldrv = { 313 &mod_driverops, 314 "i86pc root nexus", 315 &rootnex_ops 316 }; 317 318 static struct modlinkage rootnex_modlinkage = { 319 MODREV_1, 320 (void *)&rootnex_modldrv, 321 NULL 322 }; 323 324 #if defined(__amd64) && !defined(__xpv) 325 static iommulib_nexops_t iommulib_nexops = { 326 IOMMU_NEXOPS_VERSION, 327 "Rootnex IOMMU ops Vers 1.1", 328 NULL, 329 rootnex_coredma_allochdl, 330 rootnex_coredma_freehdl, 331 rootnex_coredma_bindhdl, 332 rootnex_coredma_unbindhdl, 333 rootnex_coredma_reset_cookies, 334 rootnex_coredma_get_cookies, 335 rootnex_coredma_set_cookies, 336 rootnex_coredma_clear_cookies, 337 rootnex_coredma_get_sleep_flags, 338 rootnex_coredma_sync, 339 rootnex_coredma_win, 340 rootnex_coredma_hdl_setprivate, 341 rootnex_coredma_hdl_getprivate 342 }; 343 #endif 344 345 /* 346 * extern hacks 347 */ 348 extern struct seg_ops segdev_ops; 349 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 350 #ifdef DDI_MAP_DEBUG 351 extern int ddi_map_debug_flag; 352 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 353 #endif 354 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 355 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 356 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 357 psm_intr_op_t, int *); 358 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 359 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 360 361 /* 362 * Use device arena to use for device control register mappings. 363 * Various kernel memory walkers (debugger, dtrace) need to know 364 * to avoid this address range to prevent undesired device activity. 365 */ 366 extern void *device_arena_alloc(size_t size, int vm_flag); 367 extern void device_arena_free(void * vaddr, size_t size); 368 369 370 /* 371 * Internal functions 372 */ 373 static int rootnex_dma_init(); 374 static void rootnex_add_props(dev_info_t *); 375 static int rootnex_ctl_reportdev(dev_info_t *dip); 376 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 377 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 378 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 379 static int rootnex_map_handle(ddi_map_req_t *mp); 380 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 381 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 382 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 383 ddi_dma_attr_t *attr); 384 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 385 rootnex_sglinfo_t *sglinfo); 386 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 387 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 388 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 389 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 390 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 391 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 392 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 393 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 394 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 395 static void rootnex_teardown_windows(rootnex_dma_t *dma); 396 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 397 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 398 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 399 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 400 size_t *copybuf_used, page_t **cur_pp); 401 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 402 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 403 ddi_dma_attr_t *attr, off_t cur_offset); 404 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 405 rootnex_dma_t *dma, rootnex_window_t **windowp, 406 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 407 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 409 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 410 off_t offset, size_t size, uint_t cache_flags); 411 static int rootnex_verify_buffer(rootnex_dma_t *dma); 412 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 413 const void *comp_addr, const void *not_used); 414 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 415 rootnex_sglinfo_t *sglinfo); 416 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 417 418 /* 419 * _init() 420 * 421 */ 422 int 423 _init(void) 424 { 425 426 rootnex_state = NULL; 427 return (mod_install(&rootnex_modlinkage)); 428 } 429 430 431 /* 432 * _info() 433 * 434 */ 435 int 436 _info(struct modinfo *modinfop) 437 { 438 return (mod_info(&rootnex_modlinkage, modinfop)); 439 } 440 441 442 /* 443 * _fini() 444 * 445 */ 446 int 447 _fini(void) 448 { 449 return (EBUSY); 450 } 451 452 453 /* 454 * rootnex_attach() 455 * 456 */ 457 static int 458 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 459 { 460 int fmcap; 461 int e; 462 463 switch (cmd) { 464 case DDI_ATTACH: 465 break; 466 case DDI_RESUME: 467 #if defined(__amd64) && !defined(__xpv) 468 return (immu_unquiesce()); 469 #else 470 return (DDI_SUCCESS); 471 #endif 472 default: 473 return (DDI_FAILURE); 474 } 475 476 /* 477 * We should only have one instance of rootnex. Save it away since we 478 * don't have an easy way to get it back later. 479 */ 480 ASSERT(rootnex_state == NULL); 481 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 482 483 rootnex_state->r_dip = dip; 484 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 485 rootnex_state->r_reserved_msg_printed = B_FALSE; 486 #ifdef DEBUG 487 rootnex_cnt = &rootnex_state->r_counters[0]; 488 #endif 489 490 /* 491 * Set minimum fm capability level for i86pc platforms and then 492 * initialize error handling. Since we're the rootnex, we don't 493 * care what's returned in the fmcap field. 494 */ 495 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 496 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 497 fmcap = ddi_system_fmcap; 498 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 499 500 /* initialize DMA related state */ 501 e = rootnex_dma_init(); 502 if (e != DDI_SUCCESS) { 503 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 504 return (DDI_FAILURE); 505 } 506 507 /* Add static root node properties */ 508 rootnex_add_props(dip); 509 510 /* since we can't call ddi_report_dev() */ 511 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 512 513 /* Initialize rootnex event handle */ 514 i_ddi_rootnex_init_events(dip); 515 516 #if defined(__amd64) && !defined(__xpv) 517 e = iommulib_nexus_register(dip, &iommulib_nexops, 518 &rootnex_state->r_iommulib_handle); 519 520 ASSERT(e == DDI_SUCCESS); 521 #endif 522 523 return (DDI_SUCCESS); 524 } 525 526 527 /* 528 * rootnex_detach() 529 * 530 */ 531 /*ARGSUSED*/ 532 static int 533 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 534 { 535 switch (cmd) { 536 case DDI_SUSPEND: 537 #if defined(__amd64) && !defined(__xpv) 538 return (immu_quiesce()); 539 #else 540 return (DDI_SUCCESS); 541 #endif 542 default: 543 return (DDI_FAILURE); 544 } 545 /*NOTREACHED*/ 546 547 } 548 549 550 /* 551 * rootnex_dma_init() 552 * 553 */ 554 /*ARGSUSED*/ 555 static int 556 rootnex_dma_init() 557 { 558 size_t bufsize; 559 560 561 /* 562 * size of our cookie/window/copybuf state needed in dma bind that we 563 * pre-alloc in dma_alloc_handle 564 */ 565 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 566 rootnex_state->r_prealloc_size = 567 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 568 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 569 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 570 571 /* 572 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 573 * allocate 16 extra bytes for struct pointer alignment 574 * (p->dmai_private & dma->dp_prealloc_buffer) 575 */ 576 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 577 rootnex_state->r_prealloc_size + 0x10; 578 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 579 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 580 if (rootnex_state->r_dmahdl_cache == NULL) { 581 return (DDI_FAILURE); 582 } 583 584 /* 585 * allocate array to track which major numbers we have printed warnings 586 * for. 587 */ 588 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 589 KM_SLEEP); 590 591 return (DDI_SUCCESS); 592 } 593 594 595 /* 596 * rootnex_add_props() 597 * 598 */ 599 static void 600 rootnex_add_props(dev_info_t *dip) 601 { 602 rootnex_intprop_t *rpp; 603 int i; 604 605 /* Add static integer/boolean properties to the root node */ 606 rpp = rootnex_intprp; 607 for (i = 0; i < NROOT_INTPROPS; i++) { 608 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 609 rpp[i].prop_name, rpp[i].prop_value); 610 } 611 } 612 613 614 615 /* 616 * ************************* 617 * ctlops related routines 618 * ************************* 619 */ 620 621 /* 622 * rootnex_ctlops() 623 * 624 */ 625 /*ARGSUSED*/ 626 static int 627 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 628 void *arg, void *result) 629 { 630 int n, *ptr; 631 struct ddi_parent_private_data *pdp; 632 633 switch (ctlop) { 634 case DDI_CTLOPS_DMAPMAPC: 635 /* 636 * Return 'partial' to indicate that dma mapping 637 * has to be done in the main MMU. 638 */ 639 return (DDI_DMA_PARTIAL); 640 641 case DDI_CTLOPS_BTOP: 642 /* 643 * Convert byte count input to physical page units. 644 * (byte counts that are not a page-size multiple 645 * are rounded down) 646 */ 647 *(ulong_t *)result = btop(*(ulong_t *)arg); 648 return (DDI_SUCCESS); 649 650 case DDI_CTLOPS_PTOB: 651 /* 652 * Convert size in physical pages to bytes 653 */ 654 *(ulong_t *)result = ptob(*(ulong_t *)arg); 655 return (DDI_SUCCESS); 656 657 case DDI_CTLOPS_BTOPR: 658 /* 659 * Convert byte count input to physical page units 660 * (byte counts that are not a page-size multiple 661 * are rounded up) 662 */ 663 *(ulong_t *)result = btopr(*(ulong_t *)arg); 664 return (DDI_SUCCESS); 665 666 case DDI_CTLOPS_INITCHILD: 667 return (impl_ddi_sunbus_initchild(arg)); 668 669 case DDI_CTLOPS_UNINITCHILD: 670 impl_ddi_sunbus_removechild(arg); 671 return (DDI_SUCCESS); 672 673 case DDI_CTLOPS_REPORTDEV: 674 return (rootnex_ctl_reportdev(rdip)); 675 676 case DDI_CTLOPS_IOMIN: 677 /* 678 * Nothing to do here but reflect back.. 679 */ 680 return (DDI_SUCCESS); 681 682 case DDI_CTLOPS_REGSIZE: 683 case DDI_CTLOPS_NREGS: 684 break; 685 686 case DDI_CTLOPS_SIDDEV: 687 if (ndi_dev_is_prom_node(rdip)) 688 return (DDI_SUCCESS); 689 if (ndi_dev_is_persistent_node(rdip)) 690 return (DDI_SUCCESS); 691 return (DDI_FAILURE); 692 693 case DDI_CTLOPS_POWER: 694 return ((*pm_platform_power)((power_req_t *)arg)); 695 696 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 697 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 698 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 699 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 700 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 701 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 702 if (!rootnex_state->r_reserved_msg_printed) { 703 rootnex_state->r_reserved_msg_printed = B_TRUE; 704 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 705 "1 or more reserved/obsolete operations."); 706 } 707 return (DDI_FAILURE); 708 709 default: 710 return (DDI_FAILURE); 711 } 712 /* 713 * The rest are for "hardware" properties 714 */ 715 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 716 return (DDI_FAILURE); 717 718 if (ctlop == DDI_CTLOPS_NREGS) { 719 ptr = (int *)result; 720 *ptr = pdp->par_nreg; 721 } else { 722 off_t *size = (off_t *)result; 723 724 ptr = (int *)arg; 725 n = *ptr; 726 if (n >= pdp->par_nreg) { 727 return (DDI_FAILURE); 728 } 729 *size = (off_t)pdp->par_reg[n].regspec_size; 730 } 731 return (DDI_SUCCESS); 732 } 733 734 735 /* 736 * rootnex_ctl_reportdev() 737 * 738 */ 739 static int 740 rootnex_ctl_reportdev(dev_info_t *dev) 741 { 742 int i, n, len, f_len = 0; 743 char *buf; 744 745 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 746 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 747 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 748 len = strlen(buf); 749 750 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 751 752 struct regspec *rp = sparc_pd_getreg(dev, i); 753 754 if (i == 0) 755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 756 ": "); 757 else 758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 759 " and "); 760 len = strlen(buf); 761 762 switch (rp->regspec_bustype) { 763 764 case BTEISA: 765 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 766 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 767 break; 768 769 case BTISA: 770 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 771 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 772 break; 773 774 default: 775 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 776 "space %x offset %x", 777 rp->regspec_bustype, rp->regspec_addr); 778 break; 779 } 780 len = strlen(buf); 781 } 782 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 783 int pri; 784 785 if (i != 0) { 786 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 787 ","); 788 len = strlen(buf); 789 } 790 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 791 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 792 " sparc ipl %d", pri); 793 len = strlen(buf); 794 } 795 #ifdef DEBUG 796 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 797 cmn_err(CE_NOTE, "next message is truncated: " 798 "printed length 1024, real length %d", f_len); 799 } 800 #endif /* DEBUG */ 801 cmn_err(CE_CONT, "?%s\n", buf); 802 kmem_free(buf, REPORTDEV_BUFSIZE); 803 return (DDI_SUCCESS); 804 } 805 806 807 /* 808 * ****************** 809 * map related code 810 * ****************** 811 */ 812 813 /* 814 * rootnex_map() 815 * 816 */ 817 static int 818 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 819 off_t len, caddr_t *vaddrp) 820 { 821 struct regspec *orp = NULL; 822 struct regspec64 rp = { 0 }; 823 ddi_map_req_t mr = *mp; /* Get private copy of request */ 824 825 mp = &mr; 826 827 switch (mp->map_op) { 828 case DDI_MO_MAP_LOCKED: 829 case DDI_MO_UNMAP: 830 case DDI_MO_MAP_HANDLE: 831 break; 832 default: 833 #ifdef DDI_MAP_DEBUG 834 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 835 mp->map_op); 836 #endif /* DDI_MAP_DEBUG */ 837 return (DDI_ME_UNIMPLEMENTED); 838 } 839 840 if (mp->map_flags & DDI_MF_USER_MAPPING) { 841 #ifdef DDI_MAP_DEBUG 842 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 843 #endif /* DDI_MAP_DEBUG */ 844 return (DDI_ME_UNIMPLEMENTED); 845 } 846 847 /* 848 * First, we need to get the original regspec out before we convert it 849 * to the extended format. If we have a register number, then we need to 850 * convert that to a regspec. 851 */ 852 if (mp->map_type == DDI_MT_RNUMBER) { 853 854 int rnumber = mp->map_obj.rnumber; 855 #ifdef DDI_MAP_DEBUG 856 static char *out_of_range = 857 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 858 #endif /* DDI_MAP_DEBUG */ 859 860 orp = i_ddi_rnumber_to_regspec(rdip, rnumber); 861 if (orp == NULL) { 862 #ifdef DDI_MAP_DEBUG 863 cmn_err(CE_WARN, out_of_range, rnumber, 864 ddi_get_name(rdip)); 865 #endif /* DDI_MAP_DEBUG */ 866 return (DDI_ME_RNUMBER_RANGE); 867 } 868 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) { 869 orp = mp->map_obj.rp; 870 } 871 872 /* 873 * Ensure that we are always using a 64-bit extended regspec regardless 874 * of what was passed into us. If the child driver is using a 64-bit 875 * regspec, then we need to make sure that we copy this to the local 876 * regspec64, rp. 877 */ 878 if (orp != NULL) { 879 rp.regspec_bustype = orp->regspec_bustype; 880 rp.regspec_addr = orp->regspec_addr; 881 rp.regspec_size = orp->regspec_size; 882 } else { 883 struct regspec64 *rp64; 884 rp64 = (struct regspec64 *)mp->map_obj.rp; 885 rp = *rp64; 886 } 887 888 mp->map_type = DDI_MT_REGSPEC; 889 mp->map_flags |= DDI_MF_EXT_REGSPEC; 890 mp->map_obj.rp = (struct regspec *)&rp; 891 892 /* 893 * Adjust offset and length correspnding to called values... 894 * XXX: A non-zero length means override the one in the regspec 895 * XXX: (regardless of what's in the parent's range?) 896 */ 897 898 #ifdef DDI_MAP_DEBUG 899 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 900 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 901 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset, 902 len, mp->map_handlep); 903 #endif /* DDI_MAP_DEBUG */ 904 905 /* 906 * I/O or memory mapping: 907 * 908 * <bustype=0, addr=x, len=x>: memory 909 * <bustype=1, addr=x, len=x>: i/o 910 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 911 */ 912 913 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) { 914 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 915 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">", 916 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 917 rp.regspec_addr, rp.regspec_size); 918 return (DDI_ME_INVAL); 919 } 920 921 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) { 922 /* 923 * compatibility i/o mapping 924 */ 925 rp.regspec_bustype += offset; 926 } else { 927 /* 928 * Normal memory or i/o mapping 929 */ 930 rp.regspec_addr += offset; 931 } 932 933 if (len != 0) 934 rp.regspec_size = len; 935 936 #ifdef DDI_MAP_DEBUG 937 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64 938 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n", 939 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 940 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep); 941 #endif /* DDI_MAP_DEBUG */ 942 943 944 /* 945 * The x86 root nexus does not have any notion of valid ranges of 946 * addresses. Its children have valid ranges, but because there are none 947 * for the nexus, we don't need to call i_ddi_apply_range(). Verify 948 * that is the case. 949 */ 950 ASSERT0(sparc_pd_getnrng(dip)); 951 952 switch (mp->map_op) { 953 case DDI_MO_MAP_LOCKED: 954 955 /* 956 * Set up the locked down kernel mapping to the regspec... 957 */ 958 959 return (rootnex_map_regspec(mp, vaddrp)); 960 961 case DDI_MO_UNMAP: 962 963 /* 964 * Release mapping... 965 */ 966 967 return (rootnex_unmap_regspec(mp, vaddrp)); 968 969 case DDI_MO_MAP_HANDLE: 970 971 return (rootnex_map_handle(mp)); 972 973 default: 974 return (DDI_ME_UNIMPLEMENTED); 975 } 976 } 977 978 979 /* 980 * rootnex_map_fault() 981 * 982 * fault in mappings for requestors 983 */ 984 /*ARGSUSED*/ 985 static int 986 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 987 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 988 uint_t lock) 989 { 990 991 #ifdef DDI_MAP_DEBUG 992 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 993 ddi_map_debug(" Seg <%s>\n", 994 seg->s_ops == &segdev_ops ? "segdev" : 995 seg == &kvseg ? "segkmem" : "NONE!"); 996 #endif /* DDI_MAP_DEBUG */ 997 998 /* 999 * This is all terribly broken, but it is a start 1000 * 1001 * XXX Note that this test means that segdev_ops 1002 * must be exported from seg_dev.c. 1003 * XXX What about devices with their own segment drivers? 1004 */ 1005 if (seg->s_ops == &segdev_ops) { 1006 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1007 1008 if (hat == NULL) { 1009 /* 1010 * This is one plausible interpretation of 1011 * a null hat i.e. use the first hat on the 1012 * address space hat list which by convention is 1013 * the hat of the system MMU. At alternative 1014 * would be to panic .. this might well be better .. 1015 */ 1016 ASSERT(AS_READ_HELD(seg->s_as)); 1017 hat = seg->s_as->a_hat; 1018 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1019 } 1020 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1021 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1022 } else if (seg == &kvseg && dp == NULL) { 1023 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1024 HAT_LOAD_LOCK); 1025 } else 1026 return (DDI_FAILURE); 1027 return (DDI_SUCCESS); 1028 } 1029 1030 1031 static int 1032 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1033 { 1034 rootnex_addr_t rbase; 1035 void *cvaddr; 1036 uint64_t npages, pgoffset; 1037 struct regspec64 *rp; 1038 ddi_acc_hdl_t *hp; 1039 ddi_acc_impl_t *ap; 1040 uint_t hat_acc_flags; 1041 paddr_t pbase; 1042 1043 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1044 rp = (struct regspec64 *)mp->map_obj.rp; 1045 hp = mp->map_handlep; 1046 1047 #ifdef DDI_MAP_DEBUG 1048 ddi_map_debug( 1049 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1050 rp->regspec_bustype, rp->regspec_addr, 1051 rp->regspec_size, mp->map_handlep); 1052 #endif /* DDI_MAP_DEBUG */ 1053 1054 /* 1055 * I/O or memory mapping 1056 * 1057 * <bustype=0, addr=x, len=x>: memory 1058 * <bustype=1, addr=x, len=x>: i/o 1059 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1060 */ 1061 1062 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1063 cmn_err(CE_WARN, "rootnex: invalid register spec" 1064 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">", 1065 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size); 1066 return (DDI_FAILURE); 1067 } 1068 1069 if (rp->regspec_bustype != 0) { 1070 /* 1071 * I/O space - needs a handle. 1072 */ 1073 if (hp == NULL) { 1074 return (DDI_FAILURE); 1075 } 1076 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1077 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1078 impl_acc_hdl_init(hp); 1079 1080 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1081 #ifdef DDI_MAP_DEBUG 1082 ddi_map_debug("rootnex_map_regspec: mmap() " 1083 "to I/O space is not supported.\n"); 1084 #endif /* DDI_MAP_DEBUG */ 1085 return (DDI_ME_INVAL); 1086 } else { 1087 /* 1088 * 1275-compliant vs. compatibility i/o mapping 1089 */ 1090 *vaddrp = 1091 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1092 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1093 ((caddr_t)(uintptr_t)rp->regspec_addr); 1094 #ifdef __xpv 1095 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1096 hp->ah_pfn = xen_assign_pfn( 1097 mmu_btop((ulong_t)rp->regspec_addr & 1098 MMU_PAGEMASK)); 1099 } else { 1100 hp->ah_pfn = mmu_btop( 1101 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1102 } 1103 #else 1104 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1105 MMU_PAGEMASK); 1106 #endif 1107 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1108 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1109 } 1110 1111 #ifdef DDI_MAP_DEBUG 1112 ddi_map_debug( 1113 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1114 rp->regspec_size, *vaddrp); 1115 #endif /* DDI_MAP_DEBUG */ 1116 return (DDI_SUCCESS); 1117 } 1118 1119 /* 1120 * Memory space 1121 */ 1122 1123 if (hp != NULL) { 1124 /* 1125 * hat layer ignores 1126 * hp->ah_acc.devacc_attr_endian_flags. 1127 */ 1128 switch (hp->ah_acc.devacc_attr_dataorder) { 1129 case DDI_STRICTORDER_ACC: 1130 hat_acc_flags = HAT_STRICTORDER; 1131 break; 1132 case DDI_UNORDERED_OK_ACC: 1133 hat_acc_flags = HAT_UNORDERED_OK; 1134 break; 1135 case DDI_MERGING_OK_ACC: 1136 hat_acc_flags = HAT_MERGING_OK; 1137 break; 1138 case DDI_LOADCACHING_OK_ACC: 1139 hat_acc_flags = HAT_LOADCACHING_OK; 1140 break; 1141 case DDI_STORECACHING_OK_ACC: 1142 hat_acc_flags = HAT_STORECACHING_OK; 1143 break; 1144 } 1145 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1146 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1147 impl_acc_hdl_init(hp); 1148 hp->ah_hat_flags = hat_acc_flags; 1149 } else { 1150 hat_acc_flags = HAT_STRICTORDER; 1151 } 1152 1153 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1154 #ifdef __xpv 1155 /* 1156 * If we're dom0, we're using a real device so we need to translate 1157 * the MA to a PA. 1158 */ 1159 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1160 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1161 } else { 1162 pbase = rbase; 1163 } 1164 #else 1165 pbase = rbase; 1166 #endif 1167 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1168 1169 if (rp->regspec_size == 0) { 1170 #ifdef DDI_MAP_DEBUG 1171 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1172 #endif /* DDI_MAP_DEBUG */ 1173 return (DDI_ME_INVAL); 1174 } 1175 1176 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1177 /* extra cast to make gcc happy */ 1178 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1179 } else { 1180 npages = mmu_btopr(rp->regspec_size + pgoffset); 1181 1182 #ifdef DDI_MAP_DEBUG 1183 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1184 "physical %llx", npages, pbase); 1185 #endif /* DDI_MAP_DEBUG */ 1186 1187 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1188 if (cvaddr == NULL) 1189 return (DDI_ME_NORESOURCES); 1190 1191 /* 1192 * Now map in the pages we've allocated... 1193 */ 1194 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1195 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1196 HAT_LOAD_LOCK); 1197 *vaddrp = (caddr_t)cvaddr + pgoffset; 1198 1199 /* save away pfn and npages for FMA */ 1200 hp = mp->map_handlep; 1201 if (hp) { 1202 hp->ah_pfn = mmu_btop(pbase); 1203 hp->ah_pnum = npages; 1204 } 1205 } 1206 1207 #ifdef DDI_MAP_DEBUG 1208 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1209 #endif /* DDI_MAP_DEBUG */ 1210 return (DDI_SUCCESS); 1211 } 1212 1213 1214 static int 1215 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1216 { 1217 caddr_t addr = (caddr_t)*vaddrp; 1218 uint64_t npages, pgoffset; 1219 struct regspec64 *rp; 1220 1221 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1222 return (0); 1223 1224 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1225 rp = (struct regspec64 *)mp->map_obj.rp; 1226 1227 if (rp->regspec_size == 0) { 1228 #ifdef DDI_MAP_DEBUG 1229 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1230 #endif /* DDI_MAP_DEBUG */ 1231 return (DDI_ME_INVAL); 1232 } 1233 1234 /* 1235 * I/O or memory mapping: 1236 * 1237 * <bustype=0, addr=x, len=x>: memory 1238 * <bustype=1, addr=x, len=x>: i/o 1239 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1240 */ 1241 if (rp->regspec_bustype != 0) { 1242 /* 1243 * This is I/O space, which requires no particular 1244 * processing on unmap since it isn't mapped in the 1245 * first place. 1246 */ 1247 return (DDI_SUCCESS); 1248 } 1249 1250 /* 1251 * Memory space 1252 */ 1253 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1254 npages = mmu_btopr(rp->regspec_size + pgoffset); 1255 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1256 device_arena_free(addr - pgoffset, ptob(npages)); 1257 1258 /* 1259 * Destroy the pointer - the mapping has logically gone 1260 */ 1261 *vaddrp = NULL; 1262 1263 return (DDI_SUCCESS); 1264 } 1265 1266 static int 1267 rootnex_map_handle(ddi_map_req_t *mp) 1268 { 1269 rootnex_addr_t rbase; 1270 ddi_acc_hdl_t *hp; 1271 uint64_t pgoffset; 1272 struct regspec64 *rp; 1273 paddr_t pbase; 1274 1275 rp = (struct regspec64 *)mp->map_obj.rp; 1276 1277 #ifdef DDI_MAP_DEBUG 1278 ddi_map_debug( 1279 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1280 rp->regspec_bustype, rp->regspec_addr, 1281 rp->regspec_size, mp->map_handlep); 1282 #endif /* DDI_MAP_DEBUG */ 1283 1284 /* 1285 * I/O or memory mapping: 1286 * 1287 * <bustype=0, addr=x, len=x>: memory 1288 * <bustype=1, addr=x, len=x>: i/o 1289 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1290 */ 1291 if (rp->regspec_bustype != 0) { 1292 /* 1293 * This refers to I/O space, and we don't support "mapping" 1294 * I/O space to a user. 1295 */ 1296 return (DDI_FAILURE); 1297 } 1298 1299 /* 1300 * Set up the hat_flags for the mapping. 1301 */ 1302 hp = mp->map_handlep; 1303 1304 switch (hp->ah_acc.devacc_attr_endian_flags) { 1305 case DDI_NEVERSWAP_ACC: 1306 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1307 break; 1308 case DDI_STRUCTURE_LE_ACC: 1309 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1310 break; 1311 case DDI_STRUCTURE_BE_ACC: 1312 return (DDI_FAILURE); 1313 default: 1314 return (DDI_REGS_ACC_CONFLICT); 1315 } 1316 1317 switch (hp->ah_acc.devacc_attr_dataorder) { 1318 case DDI_STRICTORDER_ACC: 1319 break; 1320 case DDI_UNORDERED_OK_ACC: 1321 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1322 break; 1323 case DDI_MERGING_OK_ACC: 1324 hp->ah_hat_flags |= HAT_MERGING_OK; 1325 break; 1326 case DDI_LOADCACHING_OK_ACC: 1327 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1328 break; 1329 case DDI_STORECACHING_OK_ACC: 1330 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1331 break; 1332 default: 1333 return (DDI_FAILURE); 1334 } 1335 1336 rbase = (rootnex_addr_t)rp->regspec_addr & 1337 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1338 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1339 1340 if (rp->regspec_size == 0) 1341 return (DDI_ME_INVAL); 1342 1343 #ifdef __xpv 1344 /* 1345 * If we're dom0, we're using a real device so we need to translate 1346 * the MA to a PA. 1347 */ 1348 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1349 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1350 (rbase & MMU_PAGEOFFSET); 1351 } else { 1352 pbase = rbase; 1353 } 1354 #else 1355 pbase = rbase; 1356 #endif 1357 1358 hp->ah_pfn = mmu_btop(pbase); 1359 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1360 1361 return (DDI_SUCCESS); 1362 } 1363 1364 1365 1366 /* 1367 * ************************ 1368 * interrupt related code 1369 * ************************ 1370 */ 1371 1372 /* 1373 * rootnex_intr_ops() 1374 * bus_intr_op() function for interrupt support 1375 */ 1376 /* ARGSUSED */ 1377 static int 1378 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1379 ddi_intr_handle_impl_t *hdlp, void *result) 1380 { 1381 struct intrspec *ispec; 1382 1383 DDI_INTR_NEXDBG((CE_CONT, 1384 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1385 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1386 1387 /* Process the interrupt operation */ 1388 switch (intr_op) { 1389 case DDI_INTROP_GETCAP: 1390 /* First check with pcplusmp */ 1391 if (psm_intr_ops == NULL) 1392 return (DDI_FAILURE); 1393 1394 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1395 *(int *)result = 0; 1396 return (DDI_FAILURE); 1397 } 1398 break; 1399 case DDI_INTROP_SETCAP: 1400 if (psm_intr_ops == NULL) 1401 return (DDI_FAILURE); 1402 1403 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1404 return (DDI_FAILURE); 1405 break; 1406 case DDI_INTROP_ALLOC: 1407 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1408 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1409 case DDI_INTROP_FREE: 1410 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1411 return (rootnex_free_intr_fixed(rdip, hdlp)); 1412 case DDI_INTROP_GETPRI: 1413 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1414 return (DDI_FAILURE); 1415 *(int *)result = ispec->intrspec_pri; 1416 break; 1417 case DDI_INTROP_SETPRI: 1418 /* Validate the interrupt priority passed to us */ 1419 if (*(int *)result > LOCK_LEVEL) 1420 return (DDI_FAILURE); 1421 1422 /* Ensure that PSM is all initialized and ispec is ok */ 1423 if ((psm_intr_ops == NULL) || 1424 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1425 return (DDI_FAILURE); 1426 1427 /* Change the priority */ 1428 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1429 PSM_FAILURE) 1430 return (DDI_FAILURE); 1431 1432 /* update the ispec with the new priority */ 1433 ispec->intrspec_pri = *(int *)result; 1434 break; 1435 case DDI_INTROP_ADDISR: 1436 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1437 return (DDI_FAILURE); 1438 ispec->intrspec_func = hdlp->ih_cb_func; 1439 break; 1440 case DDI_INTROP_REMISR: 1441 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1442 return (DDI_FAILURE); 1443 ispec->intrspec_func = (uint_t (*)()) 0; 1444 break; 1445 case DDI_INTROP_ENABLE: 1446 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1447 return (DDI_FAILURE); 1448 1449 /* Call psmi to translate irq with the dip */ 1450 if (psm_intr_ops == NULL) 1451 return (DDI_FAILURE); 1452 1453 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1454 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1455 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1456 return (DDI_FAILURE); 1457 1458 /* Add the interrupt handler */ 1459 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1460 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1461 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1462 return (DDI_FAILURE); 1463 break; 1464 case DDI_INTROP_DISABLE: 1465 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1466 return (DDI_FAILURE); 1467 1468 /* Call psm_ops() to translate irq with the dip */ 1469 if (psm_intr_ops == NULL) 1470 return (DDI_FAILURE); 1471 1472 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1473 (void) (*psm_intr_ops)(rdip, hdlp, 1474 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1475 1476 /* Remove the interrupt handler */ 1477 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1478 hdlp->ih_cb_func, hdlp->ih_vector); 1479 break; 1480 case DDI_INTROP_SETMASK: 1481 if (psm_intr_ops == NULL) 1482 return (DDI_FAILURE); 1483 1484 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1485 return (DDI_FAILURE); 1486 break; 1487 case DDI_INTROP_CLRMASK: 1488 if (psm_intr_ops == NULL) 1489 return (DDI_FAILURE); 1490 1491 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1492 return (DDI_FAILURE); 1493 break; 1494 case DDI_INTROP_GETPENDING: 1495 if (psm_intr_ops == NULL) 1496 return (DDI_FAILURE); 1497 1498 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1499 result)) { 1500 *(int *)result = 0; 1501 return (DDI_FAILURE); 1502 } 1503 break; 1504 case DDI_INTROP_NAVAIL: 1505 case DDI_INTROP_NINTRS: 1506 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1507 if (*(int *)result == 0) { 1508 /* 1509 * Special case for 'pcic' driver' only. This driver 1510 * driver is a child of 'isa' and 'rootnex' drivers. 1511 * 1512 * See detailed comments on this in the function 1513 * rootnex_get_ispec(). 1514 * 1515 * Children of 'pcic' send 'NINITR' request all the 1516 * way to rootnex driver. But, the 'pdp->par_nintr' 1517 * field may not initialized. So, we fake it here 1518 * to return 1 (a la what PCMCIA nexus does). 1519 */ 1520 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1521 *(int *)result = 1; 1522 else 1523 return (DDI_FAILURE); 1524 } 1525 break; 1526 case DDI_INTROP_SUPPORTED_TYPES: 1527 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1528 break; 1529 default: 1530 return (DDI_FAILURE); 1531 } 1532 1533 return (DDI_SUCCESS); 1534 } 1535 1536 1537 /* 1538 * rootnex_get_ispec() 1539 * convert an interrupt number to an interrupt specification. 1540 * The interrupt number determines which interrupt spec will be 1541 * returned if more than one exists. 1542 * 1543 * Look into the parent private data area of the 'rdip' to find out 1544 * the interrupt specification. First check to make sure there is 1545 * one that matchs "inumber" and then return a pointer to it. 1546 * 1547 * Return NULL if one could not be found. 1548 * 1549 * NOTE: This is needed for rootnex_intr_ops() 1550 */ 1551 static struct intrspec * 1552 rootnex_get_ispec(dev_info_t *rdip, int inum) 1553 { 1554 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1555 1556 /* 1557 * Special case handling for drivers that provide their own 1558 * intrspec structures instead of relying on the DDI framework. 1559 * 1560 * A broken hardware driver in ON could potentially provide its 1561 * own intrspec structure, instead of relying on the hardware. 1562 * If these drivers are children of 'rootnex' then we need to 1563 * continue to provide backward compatibility to them here. 1564 * 1565 * Following check is a special case for 'pcic' driver which 1566 * was found to have broken hardwre andby provides its own intrspec. 1567 * 1568 * Verbatim comments from this driver are shown here: 1569 * "Don't use the ddi_add_intr since we don't have a 1570 * default intrspec in all cases." 1571 * 1572 * Since an 'ispec' may not be always created for it, 1573 * check for that and create one if so. 1574 * 1575 * NOTE: Currently 'pcic' is the only driver found to do this. 1576 */ 1577 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1578 pdp->par_nintr = 1; 1579 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1580 pdp->par_nintr, KM_SLEEP); 1581 } 1582 1583 /* Validate the interrupt number */ 1584 if (inum >= pdp->par_nintr) 1585 return (NULL); 1586 1587 /* Get the interrupt structure pointer and return that */ 1588 return ((struct intrspec *)&pdp->par_intr[inum]); 1589 } 1590 1591 /* 1592 * Allocate interrupt vector for FIXED (legacy) type. 1593 */ 1594 static int 1595 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1596 void *result) 1597 { 1598 struct intrspec *ispec; 1599 ddi_intr_handle_impl_t info_hdl; 1600 int ret; 1601 int free_phdl = 0; 1602 apic_get_type_t type_info; 1603 1604 if (psm_intr_ops == NULL) 1605 return (DDI_FAILURE); 1606 1607 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1608 return (DDI_FAILURE); 1609 1610 /* 1611 * If the PSM module is "APIX" then pass the request for it 1612 * to allocate the vector now. 1613 */ 1614 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1615 info_hdl.ih_private = &type_info; 1616 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1617 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1618 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1619 free_phdl = 1; 1620 i_ddi_alloc_intr_phdl(hdlp); 1621 } 1622 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1623 ret = (*psm_intr_ops)(rdip, hdlp, 1624 PSM_INTR_OP_ALLOC_VECTORS, result); 1625 if (free_phdl) { /* free up the phdl structure */ 1626 free_phdl = 0; 1627 i_ddi_free_intr_phdl(hdlp); 1628 hdlp->ih_private = NULL; 1629 } 1630 } else { 1631 /* 1632 * No APIX module; fall back to the old scheme where the 1633 * interrupt vector is allocated during ddi_enable_intr() call. 1634 */ 1635 hdlp->ih_pri = ispec->intrspec_pri; 1636 *(int *)result = hdlp->ih_scratch1; 1637 ret = DDI_SUCCESS; 1638 } 1639 1640 return (ret); 1641 } 1642 1643 /* 1644 * Free up interrupt vector for FIXED (legacy) type. 1645 */ 1646 static int 1647 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1648 { 1649 struct intrspec *ispec; 1650 struct ddi_parent_private_data *pdp; 1651 ddi_intr_handle_impl_t info_hdl; 1652 int ret; 1653 apic_get_type_t type_info; 1654 1655 if (psm_intr_ops == NULL) 1656 return (DDI_FAILURE); 1657 1658 /* 1659 * If the PSM module is "APIX" then pass the request for it 1660 * to free up the vector now. 1661 */ 1662 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1663 info_hdl.ih_private = &type_info; 1664 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1665 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1666 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1667 return (DDI_FAILURE); 1668 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1669 ret = (*psm_intr_ops)(rdip, hdlp, 1670 PSM_INTR_OP_FREE_VECTORS, NULL); 1671 } else { 1672 /* 1673 * No APIX module; fall back to the old scheme where 1674 * the interrupt vector was already freed during 1675 * ddi_disable_intr() call. 1676 */ 1677 ret = DDI_SUCCESS; 1678 } 1679 1680 pdp = ddi_get_parent_data(rdip); 1681 1682 /* 1683 * Special case for 'pcic' driver' only. 1684 * If an intrspec was created for it, clean it up here 1685 * See detailed comments on this in the function 1686 * rootnex_get_ispec(). 1687 */ 1688 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1689 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1690 pdp->par_nintr); 1691 /* 1692 * Set it to zero; so that 1693 * DDI framework doesn't free it again 1694 */ 1695 pdp->par_intr = NULL; 1696 pdp->par_nintr = 0; 1697 } 1698 1699 return (ret); 1700 } 1701 1702 1703 /* 1704 * ****************** 1705 * dma related code 1706 * ****************** 1707 */ 1708 1709 /*ARGSUSED*/ 1710 static int 1711 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1712 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1713 ddi_dma_handle_t *handlep) 1714 { 1715 uint64_t maxsegmentsize_ll; 1716 uint_t maxsegmentsize; 1717 ddi_dma_impl_t *hp; 1718 rootnex_dma_t *dma; 1719 uint64_t count_max; 1720 uint64_t seg; 1721 int kmflag; 1722 int e; 1723 1724 1725 /* convert our sleep flags */ 1726 if (waitfp == DDI_DMA_SLEEP) { 1727 kmflag = KM_SLEEP; 1728 } else { 1729 kmflag = KM_NOSLEEP; 1730 } 1731 1732 /* 1733 * We try to do only one memory allocation here. We'll do a little 1734 * pointer manipulation later. If the bind ends up taking more than 1735 * our prealloc's space, we'll have to allocate more memory in the 1736 * bind operation. Not great, but much better than before and the 1737 * best we can do with the current bind interfaces. 1738 */ 1739 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1740 if (hp == NULL) 1741 return (DDI_DMA_NORESOURCES); 1742 1743 /* Do our pointer manipulation now, align the structures */ 1744 hp->dmai_private = (void *)(((uintptr_t)hp + 1745 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1746 dma = (rootnex_dma_t *)hp->dmai_private; 1747 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1748 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1749 1750 /* setup the handle */ 1751 rootnex_clean_dmahdl(hp); 1752 hp->dmai_error.err_fep = NULL; 1753 hp->dmai_error.err_cf = NULL; 1754 dma->dp_dip = rdip; 1755 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1756 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1757 1758 /* 1759 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1760 * is being used. Set the upper limit to the seg value. 1761 * There will be enough DVMA space to always get addresses 1762 * that will match the constraints. 1763 */ 1764 if (IOMMU_USED(rdip) && 1765 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1766 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1767 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1768 } else 1769 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1770 1771 hp->dmai_minxfer = attr->dma_attr_minxfer; 1772 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1773 hp->dmai_rdip = rdip; 1774 hp->dmai_attr = *attr; 1775 1776 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1777 dma->dp_sglinfo.si_cancross = B_FALSE; 1778 else 1779 dma->dp_sglinfo.si_cancross = B_TRUE; 1780 1781 /* we don't need to worry about the SPL since we do a tryenter */ 1782 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1783 1784 /* 1785 * Figure out our maximum segment size. If the segment size is greater 1786 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1787 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1788 * dma_attr_count_max are size-1 type values. 1789 * 1790 * Maximum segment size is the largest physically contiguous chunk of 1791 * memory that we can return from a bind (i.e. the maximum size of a 1792 * single cookie). 1793 */ 1794 1795 /* handle the rollover cases */ 1796 seg = attr->dma_attr_seg + 1; 1797 if (seg < attr->dma_attr_seg) { 1798 seg = attr->dma_attr_seg; 1799 } 1800 count_max = attr->dma_attr_count_max + 1; 1801 if (count_max < attr->dma_attr_count_max) { 1802 count_max = attr->dma_attr_count_max; 1803 } 1804 1805 /* 1806 * granularity may or may not be a power of two. If it isn't, we can't 1807 * use a simple mask. 1808 */ 1809 if (!ISP2(attr->dma_attr_granular)) { 1810 dma->dp_granularity_power_2 = B_FALSE; 1811 } else { 1812 dma->dp_granularity_power_2 = B_TRUE; 1813 } 1814 1815 /* 1816 * maxxfer should be a whole multiple of granularity. If we're going to 1817 * break up a window because we're greater than maxxfer, we might as 1818 * well make sure it's maxxfer is a whole multiple so we don't have to 1819 * worry about triming the window later on for this case. 1820 */ 1821 if (attr->dma_attr_granular > 1) { 1822 if (dma->dp_granularity_power_2) { 1823 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1824 (attr->dma_attr_maxxfer & 1825 (attr->dma_attr_granular - 1)); 1826 } else { 1827 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1828 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1829 } 1830 } else { 1831 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1832 } 1833 1834 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1835 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1836 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1837 maxsegmentsize = 0xFFFFFFFF; 1838 } else { 1839 maxsegmentsize = maxsegmentsize_ll; 1840 } 1841 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1842 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1843 1844 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1845 if (rootnex_alloc_check_parms) { 1846 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1847 if (e != DDI_SUCCESS) { 1848 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1849 (void) rootnex_dma_freehdl(dip, rdip, 1850 (ddi_dma_handle_t)hp); 1851 return (e); 1852 } 1853 } 1854 1855 *handlep = (ddi_dma_handle_t)hp; 1856 1857 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1858 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1859 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1860 1861 return (DDI_SUCCESS); 1862 } 1863 1864 1865 /* 1866 * rootnex_dma_allochdl() 1867 * called from ddi_dma_alloc_handle(). 1868 */ 1869 static int 1870 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1871 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1872 { 1873 int retval = DDI_SUCCESS; 1874 #if defined(__amd64) && !defined(__xpv) 1875 1876 if (IOMMU_UNITIALIZED(rdip)) { 1877 retval = iommulib_nex_open(dip, rdip); 1878 1879 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1880 return (retval); 1881 } 1882 1883 if (IOMMU_UNUSED(rdip)) { 1884 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1885 handlep); 1886 } else { 1887 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1888 waitfp, arg, handlep); 1889 } 1890 #else 1891 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1892 handlep); 1893 #endif 1894 switch (retval) { 1895 case DDI_DMA_NORESOURCES: 1896 if (waitfp != DDI_DMA_DONTWAIT) { 1897 ddi_set_callback(waitfp, arg, 1898 &rootnex_state->r_dvma_call_list_id); 1899 } 1900 break; 1901 case DDI_SUCCESS: 1902 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1903 break; 1904 default: 1905 break; 1906 } 1907 return (retval); 1908 } 1909 1910 /*ARGSUSED*/ 1911 static int 1912 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1913 ddi_dma_handle_t handle) 1914 { 1915 ddi_dma_impl_t *hp; 1916 rootnex_dma_t *dma; 1917 1918 1919 hp = (ddi_dma_impl_t *)handle; 1920 dma = (rootnex_dma_t *)hp->dmai_private; 1921 1922 /* unbind should have been called first */ 1923 ASSERT(!dma->dp_inuse); 1924 1925 mutex_destroy(&dma->dp_mutex); 1926 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1927 1928 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1929 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1930 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1931 1932 return (DDI_SUCCESS); 1933 } 1934 1935 /* 1936 * rootnex_dma_freehdl() 1937 * called from ddi_dma_free_handle(). 1938 */ 1939 static int 1940 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1941 { 1942 int ret; 1943 1944 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1945 #if defined(__amd64) && !defined(__xpv) 1946 if (IOMMU_USED(rdip)) 1947 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1948 else 1949 #endif 1950 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1951 1952 if (rootnex_state->r_dvma_call_list_id) 1953 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1954 1955 return (ret); 1956 } 1957 1958 /*ARGSUSED*/ 1959 static int 1960 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1961 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1962 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1963 { 1964 rootnex_sglinfo_t *sinfo; 1965 ddi_dma_obj_t *dmao; 1966 #if defined(__amd64) && !defined(__xpv) 1967 struct dvmaseg *dvs; 1968 ddi_dma_cookie_t *cookie; 1969 #endif 1970 ddi_dma_attr_t *attr; 1971 ddi_dma_impl_t *hp; 1972 rootnex_dma_t *dma; 1973 int kmflag; 1974 int e; 1975 uint_t ncookies; 1976 1977 hp = (ddi_dma_impl_t *)handle; 1978 dma = (rootnex_dma_t *)hp->dmai_private; 1979 dmao = &dma->dp_dma; 1980 sinfo = &dma->dp_sglinfo; 1981 attr = &hp->dmai_attr; 1982 1983 /* convert the sleep flags */ 1984 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1985 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1986 } else { 1987 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1988 } 1989 1990 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1991 1992 /* 1993 * This is useful for debugging a driver. Not as useful in a production 1994 * system. The only time this will fail is if you have a driver bug. 1995 */ 1996 if (rootnex_bind_check_inuse) { 1997 /* 1998 * No one else should ever have this lock unless someone else 1999 * is trying to use this handle. So contention on the lock 2000 * is the same as inuse being set. 2001 */ 2002 e = mutex_tryenter(&dma->dp_mutex); 2003 if (e == 0) { 2004 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2005 return (DDI_DMA_INUSE); 2006 } 2007 if (dma->dp_inuse) { 2008 mutex_exit(&dma->dp_mutex); 2009 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2010 return (DDI_DMA_INUSE); 2011 } 2012 dma->dp_inuse = B_TRUE; 2013 mutex_exit(&dma->dp_mutex); 2014 } 2015 2016 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2017 if (rootnex_bind_check_parms) { 2018 e = rootnex_valid_bind_parms(dmareq, attr); 2019 if (e != DDI_SUCCESS) { 2020 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2021 rootnex_clean_dmahdl(hp); 2022 return (e); 2023 } 2024 } 2025 2026 /* save away the original bind info */ 2027 dma->dp_dma = dmareq->dmar_object; 2028 2029 #if defined(__amd64) && !defined(__xpv) 2030 if (IOMMU_USED(rdip)) { 2031 dmao = &dma->dp_dvma; 2032 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2033 switch (e) { 2034 case DDI_SUCCESS: 2035 if (sinfo->si_cancross || 2036 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2037 dmao->dmao_size > sinfo->si_max_cookie_size) { 2038 dma->dp_dvma_used = B_TRUE; 2039 break; 2040 } 2041 sinfo->si_sgl_size = 1; 2042 hp->dmai_rflags |= DMP_NOSYNC; 2043 2044 dma->dp_dvma_used = B_TRUE; 2045 dma->dp_need_to_free_cookie = B_FALSE; 2046 2047 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2048 cookie = hp->dmai_cookie = dma->dp_cookies = 2049 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2050 cookie->dmac_laddress = dvs->dvs_start + 2051 dmao->dmao_obj.dvma_obj.dv_off; 2052 cookie->dmac_size = dvs->dvs_len; 2053 cookie->dmac_type = 0; 2054 2055 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2056 rdip); 2057 goto fast; 2058 case DDI_ENOTSUP: 2059 break; 2060 default: 2061 rootnex_clean_dmahdl(hp); 2062 return (e); 2063 } 2064 } 2065 #endif 2066 2067 /* 2068 * Figure out a rough estimate of what maximum number of pages 2069 * this buffer could use (a high estimate of course). 2070 */ 2071 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2072 2073 if (dma->dp_dvma_used) { 2074 /* 2075 * The number of physical pages is the worst case. 2076 * 2077 * For DVMA, the worst case is the length divided 2078 * by the maximum cookie length, plus 1. Add to that 2079 * the number of segment boundaries potentially crossed, and 2080 * the additional number of DVMA segments that was returned. 2081 * 2082 * In the normal case, for modern devices, si_cancross will 2083 * be false, and dv_nseg will be 1, and the fast path will 2084 * have been taken above. 2085 */ 2086 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2087 + 1; 2088 if (sinfo->si_cancross) 2089 ncookies += 2090 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2091 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2092 2093 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2094 } 2095 2096 /* 2097 * We'll use the pre-allocated cookies for any bind that will *always* 2098 * fit (more important to be consistent, we don't want to create 2099 * additional degenerate cases). 2100 */ 2101 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2102 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2103 dma->dp_need_to_free_cookie = B_FALSE; 2104 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2105 uint_t, sinfo->si_max_pages); 2106 2107 /* 2108 * For anything larger than that, we'll go ahead and allocate the 2109 * maximum number of pages we expect to see. Hopefuly, we won't be 2110 * seeing this path in the fast path for high performance devices very 2111 * frequently. 2112 * 2113 * a ddi bind interface that allowed the driver to provide storage to 2114 * the bind interface would speed this case up. 2115 */ 2116 } else { 2117 /* 2118 * Save away how much memory we allocated. If we're doing a 2119 * nosleep, the alloc could fail... 2120 */ 2121 dma->dp_cookie_size = sinfo->si_max_pages * 2122 sizeof (ddi_dma_cookie_t); 2123 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2124 if (dma->dp_cookies == NULL) { 2125 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2126 rootnex_clean_dmahdl(hp); 2127 return (DDI_DMA_NORESOURCES); 2128 } 2129 dma->dp_need_to_free_cookie = B_TRUE; 2130 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2131 uint_t, sinfo->si_max_pages); 2132 } 2133 hp->dmai_cookie = dma->dp_cookies; 2134 2135 /* 2136 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2137 * looking at the constraints in the dma structure. It will then put 2138 * some additional state about the sgl in the dma struct (i.e. is 2139 * the sgl clean, or do we need to do some munging; how many pages 2140 * need to be copied, etc.) 2141 */ 2142 if (dma->dp_dvma_used) 2143 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2144 else 2145 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2146 2147 out: 2148 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2149 /* if we don't need a copy buffer, we don't need to sync */ 2150 if (sinfo->si_copybuf_req == 0) { 2151 hp->dmai_rflags |= DMP_NOSYNC; 2152 } 2153 2154 /* 2155 * if we don't need the copybuf and we don't need to do a partial, we 2156 * hit the fast path. All the high performance devices should be trying 2157 * to hit this path. To hit this path, a device should be able to reach 2158 * all of memory, shouldn't try to bind more than it can transfer, and 2159 * the buffer shouldn't require more cookies than the driver/device can 2160 * handle [sgllen]). 2161 * 2162 * Note that negative values of dma_attr_sgllen are supposed 2163 * to mean unlimited, but we just cast them to mean a 2164 * "ridiculous large limit". This saves some extra checks on 2165 * hot paths. 2166 */ 2167 if ((sinfo->si_copybuf_req == 0) && 2168 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2169 (dmao->dmao_size < dma->dp_maxxfer)) { 2170 fast: 2171 /* 2172 * If the driver supports FMA, insert the handle in the FMA DMA 2173 * handle cache. 2174 */ 2175 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2176 hp->dmai_error.err_cf = rootnex_dma_check; 2177 2178 /* 2179 * copy out the first cookie and ccountp, set the cookie 2180 * pointer to the second cookie. The first cookie is passed 2181 * back on the stack. Additional cookies are accessed via 2182 * ddi_dma_nextcookie() 2183 */ 2184 *cookiep = dma->dp_cookies[0]; 2185 *ccountp = sinfo->si_sgl_size; 2186 hp->dmai_cookie++; 2187 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2188 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2189 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2190 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2191 uint_t, dmao->dmao_size, uint_t, *ccountp); 2192 2193 2194 return (DDI_DMA_MAPPED); 2195 } 2196 2197 /* 2198 * go to the slow path, we may need to alloc more memory, create 2199 * multiple windows, and munge up a sgl to make the device happy. 2200 */ 2201 2202 /* 2203 * With the IOMMU mapobject method used, we should never hit 2204 * the slow path. If we do, something is seriously wrong. 2205 * Clean up and return an error. 2206 */ 2207 2208 #if defined(__amd64) && !defined(__xpv) 2209 2210 if (dma->dp_dvma_used) { 2211 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2212 &dma->dp_dvma); 2213 e = DDI_DMA_NOMAPPING; 2214 } else { 2215 #endif 2216 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2217 kmflag); 2218 #if defined(__amd64) && !defined(__xpv) 2219 } 2220 #endif 2221 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2222 if (dma->dp_need_to_free_cookie) { 2223 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2224 } 2225 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2226 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2227 return (e); 2228 } 2229 2230 /* 2231 * If the driver supports FMA, insert the handle in the FMA DMA handle 2232 * cache. 2233 */ 2234 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2235 hp->dmai_error.err_cf = rootnex_dma_check; 2236 2237 /* if the first window uses the copy buffer, sync it for the device */ 2238 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2239 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2240 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2241 DDI_DMA_SYNC_FORDEV); 2242 } 2243 2244 /* 2245 * copy out the first cookie and ccountp, set the cookie pointer to the 2246 * second cookie. Make sure the partial flag is set/cleared correctly. 2247 * If we have a partial map (i.e. multiple windows), the number of 2248 * cookies we return is the number of cookies in the first window. 2249 */ 2250 if (e == DDI_DMA_MAPPED) { 2251 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2252 *ccountp = sinfo->si_sgl_size; 2253 hp->dmai_nwin = 1; 2254 } else { 2255 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2256 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2257 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2258 } 2259 *cookiep = dma->dp_cookies[0]; 2260 hp->dmai_cookie++; 2261 2262 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2263 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2264 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2265 dmao->dmao_size, uint_t, *ccountp); 2266 return (e); 2267 } 2268 2269 /* 2270 * rootnex_dma_bindhdl() 2271 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2272 */ 2273 static int 2274 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2275 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2276 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2277 { 2278 int ret; 2279 #if defined(__amd64) && !defined(__xpv) 2280 if (IOMMU_USED(rdip)) 2281 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2282 cookiep, ccountp); 2283 else 2284 #endif 2285 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2286 cookiep, ccountp); 2287 2288 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2289 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2290 &rootnex_state->r_dvma_call_list_id); 2291 } 2292 2293 return (ret); 2294 } 2295 2296 2297 2298 /*ARGSUSED*/ 2299 static int 2300 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2301 ddi_dma_handle_t handle) 2302 { 2303 ddi_dma_impl_t *hp; 2304 rootnex_dma_t *dma; 2305 int e; 2306 2307 hp = (ddi_dma_impl_t *)handle; 2308 dma = (rootnex_dma_t *)hp->dmai_private; 2309 2310 /* make sure the buffer wasn't free'd before calling unbind */ 2311 if (rootnex_unbind_verify_buffer) { 2312 e = rootnex_verify_buffer(dma); 2313 if (e != DDI_SUCCESS) { 2314 ASSERT(0); 2315 return (DDI_FAILURE); 2316 } 2317 } 2318 2319 /* sync the current window before unbinding the buffer */ 2320 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2321 (hp->dmai_rflags & DDI_DMA_READ)) { 2322 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2323 DDI_DMA_SYNC_FORCPU); 2324 } 2325 2326 /* 2327 * cleanup and copy buffer or window state. if we didn't use the copy 2328 * buffer or windows, there won't be much to do :-) 2329 */ 2330 rootnex_teardown_copybuf(dma); 2331 rootnex_teardown_windows(dma); 2332 2333 #if defined(__amd64) && !defined(__xpv) 2334 if (IOMMU_USED(rdip)) 2335 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2336 &dma->dp_dvma); 2337 #endif 2338 2339 /* 2340 * If we had to allocate space to for the worse case sgl (it didn't 2341 * fit into our pre-allocate buffer), free that up now 2342 */ 2343 if (dma->dp_need_to_free_cookie) { 2344 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2345 } 2346 2347 /* 2348 * clean up the handle so it's ready for the next bind (i.e. if the 2349 * handle is reused). 2350 */ 2351 rootnex_clean_dmahdl(hp); 2352 hp->dmai_error.err_cf = NULL; 2353 2354 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2355 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2356 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2357 2358 return (DDI_SUCCESS); 2359 } 2360 2361 /* 2362 * rootnex_dma_unbindhdl() 2363 * called from ddi_dma_unbind_handle() 2364 */ 2365 /*ARGSUSED*/ 2366 static int 2367 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2368 ddi_dma_handle_t handle) 2369 { 2370 int ret; 2371 2372 #if defined(__amd64) && !defined(__xpv) 2373 if (IOMMU_USED(rdip)) 2374 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2375 else 2376 #endif 2377 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2378 2379 if (rootnex_state->r_dvma_call_list_id) 2380 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2381 2382 return (ret); 2383 } 2384 2385 #if defined(__amd64) && !defined(__xpv) 2386 2387 static int 2388 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2389 { 2390 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2391 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2392 2393 if (dma->dp_sleep_flags != KM_SLEEP && 2394 dma->dp_sleep_flags != KM_NOSLEEP) 2395 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2396 return (dma->dp_sleep_flags); 2397 } 2398 /*ARGSUSED*/ 2399 static void 2400 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2401 { 2402 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2403 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2404 rootnex_window_t *window; 2405 2406 if (dma->dp_window) { 2407 window = &dma->dp_window[dma->dp_current_win]; 2408 hp->dmai_cookie = window->wd_first_cookie; 2409 } else { 2410 hp->dmai_cookie = dma->dp_cookies; 2411 } 2412 hp->dmai_cookie++; 2413 } 2414 2415 /*ARGSUSED*/ 2416 static int 2417 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2418 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2419 { 2420 int i; 2421 int km_flags; 2422 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2423 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2424 rootnex_window_t *window; 2425 ddi_dma_cookie_t *cp; 2426 ddi_dma_cookie_t *cookie; 2427 2428 ASSERT(*cookiepp == NULL); 2429 ASSERT(*ccountp == 0); 2430 2431 if (dma->dp_window) { 2432 window = &dma->dp_window[dma->dp_current_win]; 2433 cp = window->wd_first_cookie; 2434 *ccountp = window->wd_cookie_cnt; 2435 } else { 2436 cp = dma->dp_cookies; 2437 *ccountp = dma->dp_sglinfo.si_sgl_size; 2438 } 2439 2440 km_flags = rootnex_coredma_get_sleep_flags(handle); 2441 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2442 if (cookie == NULL) { 2443 return (DDI_DMA_NORESOURCES); 2444 } 2445 2446 for (i = 0; i < *ccountp; i++) { 2447 cookie[i].dmac_notused = cp[i].dmac_notused; 2448 cookie[i].dmac_type = cp[i].dmac_type; 2449 cookie[i].dmac_address = cp[i].dmac_address; 2450 cookie[i].dmac_size = cp[i].dmac_size; 2451 } 2452 2453 *cookiepp = cookie; 2454 2455 return (DDI_SUCCESS); 2456 } 2457 2458 /*ARGSUSED*/ 2459 static int 2460 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2461 ddi_dma_cookie_t *cookiep, uint_t ccount) 2462 { 2463 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2464 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2465 rootnex_window_t *window; 2466 ddi_dma_cookie_t *cur_cookiep; 2467 2468 ASSERT(cookiep); 2469 ASSERT(ccount != 0); 2470 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2471 2472 if (dma->dp_window) { 2473 window = &dma->dp_window[dma->dp_current_win]; 2474 dma->dp_saved_cookies = window->wd_first_cookie; 2475 window->wd_first_cookie = cookiep; 2476 ASSERT(ccount == window->wd_cookie_cnt); 2477 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2478 + window->wd_first_cookie; 2479 } else { 2480 dma->dp_saved_cookies = dma->dp_cookies; 2481 dma->dp_cookies = cookiep; 2482 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2483 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2484 + dma->dp_cookies; 2485 } 2486 2487 dma->dp_need_to_switch_cookies = B_TRUE; 2488 hp->dmai_cookie = cur_cookiep; 2489 2490 return (DDI_SUCCESS); 2491 } 2492 2493 /*ARGSUSED*/ 2494 static int 2495 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2496 { 2497 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2498 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2499 rootnex_window_t *window; 2500 ddi_dma_cookie_t *cur_cookiep; 2501 ddi_dma_cookie_t *cookie_array; 2502 uint_t ccount; 2503 2504 /* check if cookies have not been switched */ 2505 if (dma->dp_need_to_switch_cookies == B_FALSE) 2506 return (DDI_SUCCESS); 2507 2508 ASSERT(dma->dp_saved_cookies); 2509 2510 if (dma->dp_window) { 2511 window = &dma->dp_window[dma->dp_current_win]; 2512 cookie_array = window->wd_first_cookie; 2513 window->wd_first_cookie = dma->dp_saved_cookies; 2514 dma->dp_saved_cookies = NULL; 2515 ccount = window->wd_cookie_cnt; 2516 cur_cookiep = (hp->dmai_cookie - cookie_array) 2517 + window->wd_first_cookie; 2518 } else { 2519 cookie_array = dma->dp_cookies; 2520 dma->dp_cookies = dma->dp_saved_cookies; 2521 dma->dp_saved_cookies = NULL; 2522 ccount = dma->dp_sglinfo.si_sgl_size; 2523 cur_cookiep = (hp->dmai_cookie - cookie_array) 2524 + dma->dp_cookies; 2525 } 2526 2527 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2528 2529 hp->dmai_cookie = cur_cookiep; 2530 2531 dma->dp_need_to_switch_cookies = B_FALSE; 2532 2533 return (DDI_SUCCESS); 2534 } 2535 2536 #endif 2537 2538 static struct as * 2539 rootnex_get_as(ddi_dma_obj_t *dmao) 2540 { 2541 struct as *asp; 2542 2543 switch (dmao->dmao_type) { 2544 case DMA_OTYP_VADDR: 2545 case DMA_OTYP_BUFVADDR: 2546 asp = dmao->dmao_obj.virt_obj.v_as; 2547 if (asp == NULL) 2548 asp = &kas; 2549 break; 2550 default: 2551 asp = NULL; 2552 break; 2553 } 2554 return (asp); 2555 } 2556 2557 /* 2558 * rootnex_verify_buffer() 2559 * verify buffer wasn't free'd 2560 */ 2561 static int 2562 rootnex_verify_buffer(rootnex_dma_t *dma) 2563 { 2564 page_t **pplist; 2565 caddr_t vaddr; 2566 uint_t pcnt; 2567 uint_t poff; 2568 page_t *pp; 2569 char b; 2570 int i; 2571 2572 /* Figure out how many pages this buffer occupies */ 2573 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2574 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2575 } else { 2576 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2577 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2578 } 2579 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2580 2581 switch (dma->dp_dma.dmao_type) { 2582 case DMA_OTYP_PAGES: 2583 /* 2584 * for a linked list of pp's walk through them to make sure 2585 * they're locked and not free. 2586 */ 2587 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2588 for (i = 0; i < pcnt; i++) { 2589 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2590 return (DDI_FAILURE); 2591 } 2592 pp = pp->p_next; 2593 } 2594 break; 2595 2596 case DMA_OTYP_VADDR: 2597 case DMA_OTYP_BUFVADDR: 2598 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2599 /* 2600 * for an array of pp's walk through them to make sure they're 2601 * not free. It's possible that they may not be locked. 2602 */ 2603 if (pplist) { 2604 for (i = 0; i < pcnt; i++) { 2605 if (PP_ISFREE(pplist[i])) { 2606 return (DDI_FAILURE); 2607 } 2608 } 2609 2610 /* For a virtual address, try to peek at each page */ 2611 } else { 2612 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2613 for (i = 0; i < pcnt; i++) { 2614 if (ddi_peek8(NULL, vaddr, &b) == 2615 DDI_FAILURE) 2616 return (DDI_FAILURE); 2617 vaddr += MMU_PAGESIZE; 2618 } 2619 } 2620 } 2621 break; 2622 2623 default: 2624 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2625 break; 2626 } 2627 2628 return (DDI_SUCCESS); 2629 } 2630 2631 2632 /* 2633 * rootnex_clean_dmahdl() 2634 * Clean the dma handle. This should be called on a handle alloc and an 2635 * unbind handle. Set the handle state to the default settings. 2636 */ 2637 static void 2638 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2639 { 2640 rootnex_dma_t *dma; 2641 2642 2643 dma = (rootnex_dma_t *)hp->dmai_private; 2644 2645 hp->dmai_nwin = 0; 2646 dma->dp_current_cookie = 0; 2647 dma->dp_copybuf_size = 0; 2648 dma->dp_window = NULL; 2649 dma->dp_cbaddr = NULL; 2650 dma->dp_inuse = B_FALSE; 2651 dma->dp_dvma_used = B_FALSE; 2652 dma->dp_need_to_free_cookie = B_FALSE; 2653 dma->dp_need_to_switch_cookies = B_FALSE; 2654 dma->dp_saved_cookies = NULL; 2655 dma->dp_sleep_flags = KM_PANIC; 2656 dma->dp_need_to_free_window = B_FALSE; 2657 dma->dp_partial_required = B_FALSE; 2658 dma->dp_trim_required = B_FALSE; 2659 dma->dp_sglinfo.si_copybuf_req = 0; 2660 #if !defined(__amd64) 2661 dma->dp_cb_remaping = B_FALSE; 2662 dma->dp_kva = NULL; 2663 #endif 2664 2665 /* FMA related initialization */ 2666 hp->dmai_fault = 0; 2667 hp->dmai_fault_check = NULL; 2668 hp->dmai_fault_notify = NULL; 2669 hp->dmai_error.err_ena = 0; 2670 hp->dmai_error.err_status = DDI_FM_OK; 2671 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2672 hp->dmai_error.err_ontrap = NULL; 2673 } 2674 2675 2676 /* 2677 * rootnex_valid_alloc_parms() 2678 * Called in ddi_dma_alloc_handle path to validate its parameters. 2679 */ 2680 static int 2681 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2682 { 2683 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2684 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2685 (attr->dma_attr_granular > MMU_PAGESIZE) || 2686 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2687 return (DDI_DMA_BADATTR); 2688 } 2689 2690 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2691 return (DDI_DMA_BADATTR); 2692 } 2693 2694 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2695 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2696 attr->dma_attr_sgllen == 0) { 2697 return (DDI_DMA_BADATTR); 2698 } 2699 2700 /* We should be able to DMA into every byte offset in a page */ 2701 if (maxsegmentsize < MMU_PAGESIZE) { 2702 return (DDI_DMA_BADATTR); 2703 } 2704 2705 /* if we're bouncing on seg, seg must be <= addr_hi */ 2706 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2707 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2708 return (DDI_DMA_BADATTR); 2709 } 2710 return (DDI_SUCCESS); 2711 } 2712 2713 /* 2714 * rootnex_valid_bind_parms() 2715 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2716 */ 2717 /* ARGSUSED */ 2718 static int 2719 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2720 { 2721 #if !defined(__amd64) 2722 /* 2723 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2724 * we can track the offset for the obsoleted interfaces. 2725 */ 2726 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2727 return (DDI_DMA_TOOBIG); 2728 } 2729 #endif 2730 2731 return (DDI_SUCCESS); 2732 } 2733 2734 2735 /* 2736 * rootnex_need_bounce_seg() 2737 * check to see if the buffer lives on both side of the seg. 2738 */ 2739 static boolean_t 2740 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2741 { 2742 ddi_dma_atyp_t buftype; 2743 rootnex_addr_t raddr; 2744 boolean_t lower_addr; 2745 boolean_t upper_addr; 2746 uint64_t offset; 2747 page_t **pplist; 2748 uint64_t paddr; 2749 uint32_t psize; 2750 uint32_t size; 2751 caddr_t vaddr; 2752 uint_t pcnt; 2753 page_t *pp; 2754 2755 2756 /* shortcuts */ 2757 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2758 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2759 buftype = dmar_object->dmao_type; 2760 size = dmar_object->dmao_size; 2761 2762 lower_addr = B_FALSE; 2763 upper_addr = B_FALSE; 2764 pcnt = 0; 2765 2766 /* 2767 * Process the first page to handle the initial offset of the buffer. 2768 * We'll use the base address we get later when we loop through all 2769 * the pages. 2770 */ 2771 if (buftype == DMA_OTYP_PAGES) { 2772 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2773 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2774 MMU_PAGEOFFSET; 2775 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2776 psize = MIN(size, (MMU_PAGESIZE - offset)); 2777 pp = pp->p_next; 2778 sglinfo->si_asp = NULL; 2779 } else if (pplist != NULL) { 2780 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2781 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2782 if (sglinfo->si_asp == NULL) { 2783 sglinfo->si_asp = &kas; 2784 } 2785 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2786 paddr += offset; 2787 psize = MIN(size, (MMU_PAGESIZE - offset)); 2788 pcnt++; 2789 } else { 2790 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2791 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2792 if (sglinfo->si_asp == NULL) { 2793 sglinfo->si_asp = &kas; 2794 } 2795 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2796 paddr += offset; 2797 psize = MIN(size, (MMU_PAGESIZE - offset)); 2798 vaddr += psize; 2799 } 2800 2801 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2802 2803 if ((raddr + psize) > sglinfo->si_segmask) { 2804 upper_addr = B_TRUE; 2805 } else { 2806 lower_addr = B_TRUE; 2807 } 2808 size -= psize; 2809 2810 /* 2811 * Walk through the rest of the pages in the buffer. Track to see 2812 * if we have pages on both sides of the segment boundary. 2813 */ 2814 while (size > 0) { 2815 /* partial or full page */ 2816 psize = MIN(size, MMU_PAGESIZE); 2817 2818 if (buftype == DMA_OTYP_PAGES) { 2819 /* get the paddr from the page_t */ 2820 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2821 paddr = pfn_to_pa(pp->p_pagenum); 2822 pp = pp->p_next; 2823 } else if (pplist != NULL) { 2824 /* index into the array of page_t's to get the paddr */ 2825 ASSERT(!PP_ISFREE(pplist[pcnt])); 2826 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2827 pcnt++; 2828 } else { 2829 /* call into the VM to get the paddr */ 2830 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2831 vaddr)); 2832 vaddr += psize; 2833 } 2834 2835 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2836 2837 if ((raddr + psize) > sglinfo->si_segmask) { 2838 upper_addr = B_TRUE; 2839 } else { 2840 lower_addr = B_TRUE; 2841 } 2842 /* 2843 * if the buffer lives both above and below the segment 2844 * boundary, or the current page is the page immediately 2845 * after the segment, we will use a copy/bounce buffer for 2846 * all pages > seg. 2847 */ 2848 if ((lower_addr && upper_addr) || 2849 (raddr == (sglinfo->si_segmask + 1))) { 2850 return (B_TRUE); 2851 } 2852 2853 size -= psize; 2854 } 2855 2856 return (B_FALSE); 2857 } 2858 2859 /* 2860 * rootnex_get_sgl() 2861 * Called in bind fastpath to get the sgl. Most of this will be replaced 2862 * with a call to the vm layer when vm2.0 comes around... 2863 */ 2864 static void 2865 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2866 rootnex_sglinfo_t *sglinfo) 2867 { 2868 ddi_dma_atyp_t buftype; 2869 rootnex_addr_t raddr; 2870 uint64_t last_page; 2871 uint64_t offset; 2872 uint64_t addrhi; 2873 uint64_t addrlo; 2874 uint64_t maxseg; 2875 page_t **pplist; 2876 uint64_t paddr; 2877 uint32_t psize; 2878 uint32_t size; 2879 caddr_t vaddr; 2880 uint_t pcnt; 2881 page_t *pp; 2882 uint_t cnt; 2883 2884 2885 /* shortcuts */ 2886 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2887 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2888 maxseg = sglinfo->si_max_cookie_size; 2889 buftype = dmar_object->dmao_type; 2890 addrhi = sglinfo->si_max_addr; 2891 addrlo = sglinfo->si_min_addr; 2892 size = dmar_object->dmao_size; 2893 2894 pcnt = 0; 2895 cnt = 0; 2896 2897 2898 /* 2899 * check to see if we need to use the copy buffer for pages over 2900 * the segment attr. 2901 */ 2902 sglinfo->si_bounce_on_seg = B_FALSE; 2903 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2904 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2905 dmar_object, sglinfo); 2906 } 2907 2908 /* 2909 * if we were passed down a linked list of pages, i.e. pointer to 2910 * page_t, use this to get our physical address and buf offset. 2911 */ 2912 if (buftype == DMA_OTYP_PAGES) { 2913 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2914 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2915 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2916 MMU_PAGEOFFSET; 2917 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2918 psize = MIN(size, (MMU_PAGESIZE - offset)); 2919 pp = pp->p_next; 2920 sglinfo->si_asp = NULL; 2921 2922 /* 2923 * We weren't passed down a linked list of pages, but if we were passed 2924 * down an array of pages, use this to get our physical address and buf 2925 * offset. 2926 */ 2927 } else if (pplist != NULL) { 2928 ASSERT((buftype == DMA_OTYP_VADDR) || 2929 (buftype == DMA_OTYP_BUFVADDR)); 2930 2931 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2932 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2933 if (sglinfo->si_asp == NULL) { 2934 sglinfo->si_asp = &kas; 2935 } 2936 2937 ASSERT(!PP_ISFREE(pplist[pcnt])); 2938 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2939 paddr += offset; 2940 psize = MIN(size, (MMU_PAGESIZE - offset)); 2941 pcnt++; 2942 2943 /* 2944 * All we have is a virtual address, we'll need to call into the VM 2945 * to get the physical address. 2946 */ 2947 } else { 2948 ASSERT((buftype == DMA_OTYP_VADDR) || 2949 (buftype == DMA_OTYP_BUFVADDR)); 2950 2951 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2952 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2953 if (sglinfo->si_asp == NULL) { 2954 sglinfo->si_asp = &kas; 2955 } 2956 2957 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2958 paddr += offset; 2959 psize = MIN(size, (MMU_PAGESIZE - offset)); 2960 vaddr += psize; 2961 } 2962 2963 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2964 2965 /* 2966 * Setup the first cookie with the physical address of the page and the 2967 * size of the page (which takes into account the initial offset into 2968 * the page. 2969 */ 2970 sgl[cnt].dmac_laddress = raddr; 2971 sgl[cnt].dmac_size = psize; 2972 sgl[cnt].dmac_type = 0; 2973 2974 /* 2975 * Save away the buffer offset into the page. We'll need this later in 2976 * the copy buffer code to help figure out the page index within the 2977 * buffer and the offset into the current page. 2978 */ 2979 sglinfo->si_buf_offset = offset; 2980 2981 /* 2982 * If we are using the copy buffer for anything over the segment 2983 * boundary, and this page is over the segment boundary. 2984 * OR 2985 * if the DMA engine can't reach the physical address. 2986 */ 2987 if (((sglinfo->si_bounce_on_seg) && 2988 ((raddr + psize) > sglinfo->si_segmask)) || 2989 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2990 /* 2991 * Increase how much copy buffer we use. We always increase by 2992 * pagesize so we don't have to worry about converting offsets. 2993 * Set a flag in the cookies dmac_type to indicate that it uses 2994 * the copy buffer. If this isn't the last cookie, go to the 2995 * next cookie (since we separate each page which uses the copy 2996 * buffer in case the copy buffer is not physically contiguous. 2997 */ 2998 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2999 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3000 if ((cnt + 1) < sglinfo->si_max_pages) { 3001 cnt++; 3002 sgl[cnt].dmac_laddress = 0; 3003 sgl[cnt].dmac_size = 0; 3004 sgl[cnt].dmac_type = 0; 3005 } 3006 } 3007 3008 /* 3009 * save this page's physical address so we can figure out if the next 3010 * page is physically contiguous. Keep decrementing size until we are 3011 * done with the buffer. 3012 */ 3013 last_page = raddr & MMU_PAGEMASK; 3014 size -= psize; 3015 3016 while (size > 0) { 3017 /* Get the size for this page (i.e. partial or full page) */ 3018 psize = MIN(size, MMU_PAGESIZE); 3019 3020 if (buftype == DMA_OTYP_PAGES) { 3021 /* get the paddr from the page_t */ 3022 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3023 paddr = pfn_to_pa(pp->p_pagenum); 3024 pp = pp->p_next; 3025 } else if (pplist != NULL) { 3026 /* index into the array of page_t's to get the paddr */ 3027 ASSERT(!PP_ISFREE(pplist[pcnt])); 3028 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3029 pcnt++; 3030 } else { 3031 /* call into the VM to get the paddr */ 3032 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3033 vaddr)); 3034 vaddr += psize; 3035 } 3036 3037 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3038 3039 /* 3040 * If we are using the copy buffer for anything over the 3041 * segment boundary, and this page is over the segment 3042 * boundary. 3043 * OR 3044 * if the DMA engine can't reach the physical address. 3045 */ 3046 if (((sglinfo->si_bounce_on_seg) && 3047 ((raddr + psize) > sglinfo->si_segmask)) || 3048 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3049 3050 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3051 3052 /* 3053 * if there is something in the current cookie, go to 3054 * the next one. We only want one page in a cookie which 3055 * uses the copybuf since the copybuf doesn't have to 3056 * be physically contiguous. 3057 */ 3058 if (sgl[cnt].dmac_size != 0) { 3059 cnt++; 3060 } 3061 sgl[cnt].dmac_laddress = raddr; 3062 sgl[cnt].dmac_size = psize; 3063 #if defined(__amd64) 3064 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3065 #else 3066 /* 3067 * save the buf offset for 32-bit kernel. used in the 3068 * obsoleted interfaces. 3069 */ 3070 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3071 (dmar_object->dmao_size - size); 3072 #endif 3073 /* if this isn't the last cookie, go to the next one */ 3074 if ((cnt + 1) < sglinfo->si_max_pages) { 3075 cnt++; 3076 sgl[cnt].dmac_laddress = 0; 3077 sgl[cnt].dmac_size = 0; 3078 sgl[cnt].dmac_type = 0; 3079 } 3080 3081 /* 3082 * this page didn't need the copy buffer, if it's not physically 3083 * contiguous, or it would put us over a segment boundary, or it 3084 * puts us over the max cookie size, or the current sgl doesn't 3085 * have anything in it. 3086 */ 3087 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3088 !(raddr & sglinfo->si_segmask) || 3089 ((sgl[cnt].dmac_size + psize) > maxseg) || 3090 (sgl[cnt].dmac_size == 0)) { 3091 /* 3092 * if we're not already in a new cookie, go to the next 3093 * cookie. 3094 */ 3095 if (sgl[cnt].dmac_size != 0) { 3096 cnt++; 3097 } 3098 3099 /* save the cookie information */ 3100 sgl[cnt].dmac_laddress = raddr; 3101 sgl[cnt].dmac_size = psize; 3102 #if defined(__amd64) 3103 sgl[cnt].dmac_type = 0; 3104 #else 3105 /* 3106 * save the buf offset for 32-bit kernel. used in the 3107 * obsoleted interfaces. 3108 */ 3109 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3110 #endif 3111 3112 /* 3113 * this page didn't need the copy buffer, it is physically 3114 * contiguous with the last page, and it's <= the max cookie 3115 * size. 3116 */ 3117 } else { 3118 sgl[cnt].dmac_size += psize; 3119 3120 /* 3121 * if this exactly == the maximum cookie size, and 3122 * it isn't the last cookie, go to the next cookie. 3123 */ 3124 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3125 ((cnt + 1) < sglinfo->si_max_pages)) { 3126 cnt++; 3127 sgl[cnt].dmac_laddress = 0; 3128 sgl[cnt].dmac_size = 0; 3129 sgl[cnt].dmac_type = 0; 3130 } 3131 } 3132 3133 /* 3134 * save this page's physical address so we can figure out if the 3135 * next page is physically contiguous. Keep decrementing size 3136 * until we are done with the buffer. 3137 */ 3138 last_page = raddr; 3139 size -= psize; 3140 } 3141 3142 /* we're done, save away how many cookies the sgl has */ 3143 if (sgl[cnt].dmac_size == 0) { 3144 ASSERT(cnt < sglinfo->si_max_pages); 3145 sglinfo->si_sgl_size = cnt; 3146 } else { 3147 sglinfo->si_sgl_size = cnt + 1; 3148 } 3149 } 3150 3151 static void 3152 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3153 rootnex_sglinfo_t *sglinfo) 3154 { 3155 uint64_t offset; 3156 uint64_t maxseg; 3157 uint64_t dvaddr; 3158 struct dvmaseg *dvs; 3159 uint64_t paddr; 3160 uint32_t psize, ssize; 3161 uint32_t size; 3162 uint_t cnt; 3163 int physcontig; 3164 3165 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3166 3167 /* shortcuts */ 3168 maxseg = sglinfo->si_max_cookie_size; 3169 size = dmar_object->dmao_size; 3170 3171 cnt = 0; 3172 sglinfo->si_bounce_on_seg = B_FALSE; 3173 3174 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3175 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3176 ssize = dvs->dvs_len; 3177 paddr = dvs->dvs_start; 3178 paddr += offset; 3179 psize = MIN(ssize, (maxseg - offset)); 3180 dvaddr = paddr + psize; 3181 ssize -= psize; 3182 3183 sgl[cnt].dmac_laddress = paddr; 3184 sgl[cnt].dmac_size = psize; 3185 sgl[cnt].dmac_type = 0; 3186 3187 size -= psize; 3188 while (size > 0) { 3189 if (ssize == 0) { 3190 dvs++; 3191 ssize = dvs->dvs_len; 3192 dvaddr = dvs->dvs_start; 3193 physcontig = 0; 3194 } else 3195 physcontig = 1; 3196 3197 paddr = dvaddr; 3198 psize = MIN(ssize, maxseg); 3199 dvaddr += psize; 3200 ssize -= psize; 3201 3202 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3203 ((sgl[cnt].dmac_size + psize) > maxseg) || 3204 (sgl[cnt].dmac_size == 0)) { 3205 /* 3206 * if we're not already in a new cookie, go to the next 3207 * cookie. 3208 */ 3209 if (sgl[cnt].dmac_size != 0) { 3210 cnt++; 3211 } 3212 3213 /* save the cookie information */ 3214 sgl[cnt].dmac_laddress = paddr; 3215 sgl[cnt].dmac_size = psize; 3216 sgl[cnt].dmac_type = 0; 3217 } else { 3218 sgl[cnt].dmac_size += psize; 3219 3220 /* 3221 * if this exactly == the maximum cookie size, and 3222 * it isn't the last cookie, go to the next cookie. 3223 */ 3224 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3225 ((cnt + 1) < sglinfo->si_max_pages)) { 3226 cnt++; 3227 sgl[cnt].dmac_laddress = 0; 3228 sgl[cnt].dmac_size = 0; 3229 sgl[cnt].dmac_type = 0; 3230 } 3231 } 3232 size -= psize; 3233 } 3234 3235 /* we're done, save away how many cookies the sgl has */ 3236 if (sgl[cnt].dmac_size == 0) { 3237 sglinfo->si_sgl_size = cnt; 3238 } else { 3239 sglinfo->si_sgl_size = cnt + 1; 3240 } 3241 } 3242 3243 /* 3244 * rootnex_bind_slowpath() 3245 * Call in the bind path if the calling driver can't use the sgl without 3246 * modifying it. We either need to use the copy buffer and/or we will end up 3247 * with a partial bind. 3248 */ 3249 static int 3250 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3251 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3252 { 3253 rootnex_sglinfo_t *sinfo; 3254 rootnex_window_t *window; 3255 ddi_dma_cookie_t *cookie; 3256 size_t copybuf_used; 3257 size_t dmac_size; 3258 boolean_t partial; 3259 off_t cur_offset; 3260 page_t *cur_pp; 3261 major_t mnum; 3262 int e; 3263 int i; 3264 3265 3266 sinfo = &dma->dp_sglinfo; 3267 copybuf_used = 0; 3268 partial = B_FALSE; 3269 3270 /* 3271 * If we're using the copybuf, set the copybuf state in dma struct. 3272 * Needs to be first since it sets the copy buffer size. 3273 */ 3274 if (sinfo->si_copybuf_req != 0) { 3275 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3276 if (e != DDI_SUCCESS) { 3277 return (e); 3278 } 3279 } else { 3280 dma->dp_copybuf_size = 0; 3281 } 3282 3283 /* 3284 * Figure out if we need to do a partial mapping. If so, figure out 3285 * if we need to trim the buffers when we munge the sgl. 3286 */ 3287 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3288 (dmao->dmao_size > dma->dp_maxxfer) || 3289 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3290 dma->dp_partial_required = B_TRUE; 3291 if (attr->dma_attr_granular != 1) { 3292 dma->dp_trim_required = B_TRUE; 3293 } 3294 } else { 3295 dma->dp_partial_required = B_FALSE; 3296 dma->dp_trim_required = B_FALSE; 3297 } 3298 3299 /* If we need to do a partial bind, make sure the driver supports it */ 3300 if (dma->dp_partial_required && 3301 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3302 3303 mnum = ddi_driver_major(dma->dp_dip); 3304 /* 3305 * patchable which allows us to print one warning per major 3306 * number. 3307 */ 3308 if ((rootnex_bind_warn) && 3309 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3310 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3311 cmn_err(CE_WARN, "!%s: coding error detected, the " 3312 "driver is using ddi_dma_attr(9S) incorrectly. " 3313 "There is a small risk of data corruption in " 3314 "particular with large I/Os. The driver should be " 3315 "replaced with a corrected version for proper " 3316 "system operation. To disable this warning, add " 3317 "'set rootnex:rootnex_bind_warn=0' to " 3318 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3319 } 3320 return (DDI_DMA_TOOBIG); 3321 } 3322 3323 /* 3324 * we might need multiple windows, setup state to handle them. In this 3325 * code path, we will have at least one window. 3326 */ 3327 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3328 if (e != DDI_SUCCESS) { 3329 rootnex_teardown_copybuf(dma); 3330 return (e); 3331 } 3332 3333 window = &dma->dp_window[0]; 3334 cookie = &dma->dp_cookies[0]; 3335 cur_offset = 0; 3336 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3337 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3338 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3339 } 3340 3341 /* loop though all the cookies we got back from get_sgl() */ 3342 for (i = 0; i < sinfo->si_sgl_size; i++) { 3343 /* 3344 * If we're using the copy buffer, check this cookie and setup 3345 * its associated copy buffer state. If this cookie uses the 3346 * copy buffer, make sure we sync this window during dma_sync. 3347 */ 3348 if (dma->dp_copybuf_size > 0) { 3349 rootnex_setup_cookie(dmao, dma, cookie, 3350 cur_offset, ©buf_used, &cur_pp); 3351 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3352 window->wd_dosync = B_TRUE; 3353 } 3354 } 3355 3356 /* 3357 * save away the cookie size, since it could be modified in 3358 * the windowing code. 3359 */ 3360 dmac_size = cookie->dmac_size; 3361 3362 /* if we went over max copybuf size */ 3363 if (dma->dp_copybuf_size && 3364 (copybuf_used > dma->dp_copybuf_size)) { 3365 partial = B_TRUE; 3366 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3367 cookie, cur_offset, ©buf_used); 3368 if (e != DDI_SUCCESS) { 3369 rootnex_teardown_copybuf(dma); 3370 rootnex_teardown_windows(dma); 3371 return (e); 3372 } 3373 3374 /* 3375 * if the coookie uses the copy buffer, make sure the 3376 * new window we just moved to is set to sync. 3377 */ 3378 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3379 window->wd_dosync = B_TRUE; 3380 } 3381 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3382 dma->dp_dip); 3383 3384 /* if the cookie cnt == max sgllen, move to the next window */ 3385 } else if (window->wd_cookie_cnt >= 3386 (unsigned)attr->dma_attr_sgllen) { 3387 partial = B_TRUE; 3388 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3389 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3390 cookie, attr, cur_offset); 3391 if (e != DDI_SUCCESS) { 3392 rootnex_teardown_copybuf(dma); 3393 rootnex_teardown_windows(dma); 3394 return (e); 3395 } 3396 3397 /* 3398 * if the coookie uses the copy buffer, make sure the 3399 * new window we just moved to is set to sync. 3400 */ 3401 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3402 window->wd_dosync = B_TRUE; 3403 } 3404 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3405 dma->dp_dip); 3406 3407 /* else if we will be over maxxfer */ 3408 } else if ((window->wd_size + dmac_size) > 3409 dma->dp_maxxfer) { 3410 partial = B_TRUE; 3411 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3412 cookie); 3413 if (e != DDI_SUCCESS) { 3414 rootnex_teardown_copybuf(dma); 3415 rootnex_teardown_windows(dma); 3416 return (e); 3417 } 3418 3419 /* 3420 * if the coookie uses the copy buffer, make sure the 3421 * new window we just moved to is set to sync. 3422 */ 3423 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3424 window->wd_dosync = B_TRUE; 3425 } 3426 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3427 dma->dp_dip); 3428 3429 /* else this cookie fits in the current window */ 3430 } else { 3431 window->wd_cookie_cnt++; 3432 window->wd_size += dmac_size; 3433 } 3434 3435 /* track our offset into the buffer, go to the next cookie */ 3436 ASSERT(dmac_size <= dmao->dmao_size); 3437 ASSERT(cookie->dmac_size <= dmac_size); 3438 cur_offset += dmac_size; 3439 cookie++; 3440 } 3441 3442 /* if we ended up with a zero sized window in the end, clean it up */ 3443 if (window->wd_size == 0) { 3444 hp->dmai_nwin--; 3445 window--; 3446 } 3447 3448 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3449 3450 if (!partial) { 3451 return (DDI_DMA_MAPPED); 3452 } 3453 3454 ASSERT(dma->dp_partial_required); 3455 return (DDI_DMA_PARTIAL_MAP); 3456 } 3457 3458 /* 3459 * rootnex_setup_copybuf() 3460 * Called in bind slowpath. Figures out if we're going to use the copy 3461 * buffer, and if we do, sets up the basic state to handle it. 3462 */ 3463 static int 3464 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3465 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3466 { 3467 rootnex_sglinfo_t *sinfo; 3468 ddi_dma_attr_t lattr; 3469 size_t max_copybuf; 3470 int cansleep; 3471 int e; 3472 #if !defined(__amd64) 3473 int vmflag; 3474 #endif 3475 3476 ASSERT(!dma->dp_dvma_used); 3477 3478 sinfo = &dma->dp_sglinfo; 3479 3480 /* read this first so it's consistent through the routine */ 3481 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3482 3483 /* We need to call into the rootnex on ddi_dma_sync() */ 3484 hp->dmai_rflags &= ~DMP_NOSYNC; 3485 3486 /* make sure the copybuf size <= the max size */ 3487 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3488 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3489 3490 #if !defined(__amd64) 3491 /* 3492 * if we don't have kva space to copy to/from, allocate the KVA space 3493 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3494 * the 64-bit kernel. 3495 */ 3496 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3497 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3498 3499 /* convert the sleep flags */ 3500 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3501 vmflag = VM_SLEEP; 3502 } else { 3503 vmflag = VM_NOSLEEP; 3504 } 3505 3506 /* allocate Kernel VA space that we can bcopy to/from */ 3507 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3508 vmflag); 3509 if (dma->dp_kva == NULL) { 3510 return (DDI_DMA_NORESOURCES); 3511 } 3512 } 3513 #endif 3514 3515 /* convert the sleep flags */ 3516 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3517 cansleep = 1; 3518 } else { 3519 cansleep = 0; 3520 } 3521 3522 /* 3523 * Allocate the actual copy buffer. This needs to fit within the DMA 3524 * engine limits, so we can't use kmem_alloc... We don't need 3525 * contiguous memory (sgllen) since we will be forcing windows on 3526 * sgllen anyway. 3527 */ 3528 lattr = *attr; 3529 lattr.dma_attr_align = MMU_PAGESIZE; 3530 lattr.dma_attr_sgllen = -1; /* no limit */ 3531 /* 3532 * if we're using the copy buffer because of seg, use that for our 3533 * upper address limit. 3534 */ 3535 if (sinfo->si_bounce_on_seg) { 3536 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3537 } 3538 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3539 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3540 if (e != DDI_SUCCESS) { 3541 #if !defined(__amd64) 3542 if (dma->dp_kva != NULL) { 3543 vmem_free(heap_arena, dma->dp_kva, 3544 dma->dp_copybuf_size); 3545 } 3546 #endif 3547 return (DDI_DMA_NORESOURCES); 3548 } 3549 3550 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3551 size_t, dma->dp_copybuf_size); 3552 3553 return (DDI_SUCCESS); 3554 } 3555 3556 3557 /* 3558 * rootnex_setup_windows() 3559 * Called in bind slowpath to setup the window state. We always have windows 3560 * in the slowpath. Even if the window count = 1. 3561 */ 3562 static int 3563 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3564 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3565 { 3566 rootnex_window_t *windowp; 3567 rootnex_sglinfo_t *sinfo; 3568 size_t copy_state_size; 3569 size_t win_state_size; 3570 size_t state_available; 3571 size_t space_needed; 3572 uint_t copybuf_win; 3573 uint_t maxxfer_win; 3574 size_t space_used; 3575 uint_t sglwin; 3576 3577 3578 sinfo = &dma->dp_sglinfo; 3579 3580 dma->dp_current_win = 0; 3581 hp->dmai_nwin = 0; 3582 3583 /* If we don't need to do a partial, we only have one window */ 3584 if (!dma->dp_partial_required) { 3585 dma->dp_max_win = 1; 3586 3587 /* 3588 * we need multiple windows, need to figure out the worse case number 3589 * of windows. 3590 */ 3591 } else { 3592 /* 3593 * if we need windows because we need more copy buffer that 3594 * we allow, the worse case number of windows we could need 3595 * here would be (copybuf space required / copybuf space that 3596 * we have) plus one for remainder, and plus 2 to handle the 3597 * extra pages on the trim for the first and last pages of the 3598 * buffer (a page is the minimum window size so under the right 3599 * attr settings, you could have a window for each page). 3600 * The last page will only be hit here if the size is not a 3601 * multiple of the granularity (which theoretically shouldn't 3602 * be the case but never has been enforced, so we could have 3603 * broken things without it). 3604 */ 3605 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3606 ASSERT(dma->dp_copybuf_size > 0); 3607 copybuf_win = (sinfo->si_copybuf_req / 3608 dma->dp_copybuf_size) + 1 + 2; 3609 } else { 3610 copybuf_win = 0; 3611 } 3612 3613 /* 3614 * if we need windows because we have more cookies than the H/W 3615 * can handle, the number of windows we would need here would 3616 * be (cookie count / cookies count H/W supports minus 1[for 3617 * trim]) plus one for remainder. 3618 */ 3619 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3620 sglwin = (sinfo->si_sgl_size / 3621 (attr->dma_attr_sgllen - 1)) + 1; 3622 } else { 3623 sglwin = 0; 3624 } 3625 3626 /* 3627 * if we need windows because we're binding more memory than the 3628 * H/W can transfer at once, the number of windows we would need 3629 * here would be (xfer count / max xfer H/W supports) plus one 3630 * for remainder, and plus 2 to handle the extra pages on the 3631 * trim (see above comment about trim) 3632 */ 3633 if (dmao->dmao_size > dma->dp_maxxfer) { 3634 maxxfer_win = (dmao->dmao_size / 3635 dma->dp_maxxfer) + 1 + 2; 3636 } else { 3637 maxxfer_win = 0; 3638 } 3639 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3640 ASSERT(dma->dp_max_win > 0); 3641 } 3642 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3643 3644 /* 3645 * Get space for window and potential copy buffer state. Before we 3646 * go and allocate memory, see if we can get away with using what's 3647 * left in the pre-allocted state or the dynamically allocated sgl. 3648 */ 3649 space_used = (uintptr_t)(sinfo->si_sgl_size * 3650 sizeof (ddi_dma_cookie_t)); 3651 3652 /* if we dynamically allocated space for the cookies */ 3653 if (dma->dp_need_to_free_cookie) { 3654 /* if we have more space in the pre-allocted buffer, use it */ 3655 ASSERT(space_used <= dma->dp_cookie_size); 3656 if ((dma->dp_cookie_size - space_used) <= 3657 rootnex_state->r_prealloc_size) { 3658 state_available = rootnex_state->r_prealloc_size; 3659 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3660 3661 /* 3662 * else, we have more free space in the dynamically allocated 3663 * buffer, i.e. the buffer wasn't worse case fragmented so we 3664 * didn't need a lot of cookies. 3665 */ 3666 } else { 3667 state_available = dma->dp_cookie_size - space_used; 3668 windowp = (rootnex_window_t *) 3669 &dma->dp_cookies[sinfo->si_sgl_size]; 3670 } 3671 3672 /* we used the pre-alloced buffer */ 3673 } else { 3674 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3675 state_available = rootnex_state->r_prealloc_size - space_used; 3676 windowp = (rootnex_window_t *) 3677 &dma->dp_cookies[sinfo->si_sgl_size]; 3678 } 3679 3680 /* 3681 * figure out how much state we need to track the copy buffer. Add an 3682 * addition 8 bytes for pointer alignemnt later. 3683 */ 3684 if (dma->dp_copybuf_size > 0) { 3685 copy_state_size = sinfo->si_max_pages * 3686 sizeof (rootnex_pgmap_t); 3687 } else { 3688 copy_state_size = 0; 3689 } 3690 /* add an additional 8 bytes for pointer alignment */ 3691 space_needed = win_state_size + copy_state_size + 0x8; 3692 3693 /* if we have enough space already, use it */ 3694 if (state_available >= space_needed) { 3695 dma->dp_window = windowp; 3696 dma->dp_need_to_free_window = B_FALSE; 3697 3698 /* not enough space, need to allocate more. */ 3699 } else { 3700 dma->dp_window = kmem_alloc(space_needed, kmflag); 3701 if (dma->dp_window == NULL) { 3702 return (DDI_DMA_NORESOURCES); 3703 } 3704 dma->dp_need_to_free_window = B_TRUE; 3705 dma->dp_window_size = space_needed; 3706 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3707 dma->dp_dip, size_t, space_needed); 3708 } 3709 3710 /* 3711 * we allocate copy buffer state and window state at the same time. 3712 * setup our copy buffer state pointers. Make sure it's aligned. 3713 */ 3714 if (dma->dp_copybuf_size > 0) { 3715 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3716 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3717 3718 #if !defined(__amd64) 3719 /* 3720 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3721 * false/NULL. Should be quicker to bzero vs loop and set. 3722 */ 3723 bzero(dma->dp_pgmap, copy_state_size); 3724 #endif 3725 } else { 3726 dma->dp_pgmap = NULL; 3727 } 3728 3729 return (DDI_SUCCESS); 3730 } 3731 3732 3733 /* 3734 * rootnex_teardown_copybuf() 3735 * cleans up after rootnex_setup_copybuf() 3736 */ 3737 static void 3738 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3739 { 3740 #if !defined(__amd64) 3741 int i; 3742 3743 /* 3744 * if we allocated kernel heap VMEM space, go through all the pages and 3745 * map out any of the ones that we're mapped into the kernel heap VMEM 3746 * arena. Then free the VMEM space. 3747 */ 3748 if (dma->dp_kva != NULL) { 3749 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3750 if (dma->dp_pgmap[i].pm_mapped) { 3751 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3752 MMU_PAGESIZE, HAT_UNLOAD); 3753 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3754 } 3755 } 3756 3757 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3758 } 3759 3760 #endif 3761 3762 /* if we allocated a copy buffer, free it */ 3763 if (dma->dp_cbaddr != NULL) { 3764 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3765 } 3766 } 3767 3768 3769 /* 3770 * rootnex_teardown_windows() 3771 * cleans up after rootnex_setup_windows() 3772 */ 3773 static void 3774 rootnex_teardown_windows(rootnex_dma_t *dma) 3775 { 3776 /* 3777 * if we had to allocate window state on the last bind (because we 3778 * didn't have enough pre-allocated space in the handle), free it. 3779 */ 3780 if (dma->dp_need_to_free_window) { 3781 kmem_free(dma->dp_window, dma->dp_window_size); 3782 } 3783 } 3784 3785 3786 /* 3787 * rootnex_init_win() 3788 * Called in bind slow path during creation of a new window. Initializes 3789 * window state to default values. 3790 */ 3791 /*ARGSUSED*/ 3792 static void 3793 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3794 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3795 { 3796 hp->dmai_nwin++; 3797 window->wd_dosync = B_FALSE; 3798 window->wd_offset = cur_offset; 3799 window->wd_size = 0; 3800 window->wd_first_cookie = cookie; 3801 window->wd_cookie_cnt = 0; 3802 window->wd_trim.tr_trim_first = B_FALSE; 3803 window->wd_trim.tr_trim_last = B_FALSE; 3804 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3805 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3806 #if !defined(__amd64) 3807 window->wd_remap_copybuf = dma->dp_cb_remaping; 3808 #endif 3809 } 3810 3811 3812 /* 3813 * rootnex_setup_cookie() 3814 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3815 * the sgl uses the copy buffer, we need to go through each cookie, figure 3816 * out if it uses the copy buffer, and if it does, save away everything we'll 3817 * need during sync. 3818 */ 3819 static void 3820 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3821 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3822 page_t **cur_pp) 3823 { 3824 boolean_t copybuf_sz_power_2; 3825 rootnex_sglinfo_t *sinfo; 3826 paddr_t paddr; 3827 uint_t pidx; 3828 uint_t pcnt; 3829 off_t poff; 3830 #if defined(__amd64) 3831 pfn_t pfn; 3832 #else 3833 page_t **pplist; 3834 #endif 3835 3836 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3837 3838 sinfo = &dma->dp_sglinfo; 3839 3840 /* 3841 * Calculate the page index relative to the start of the buffer. The 3842 * index to the current page for our buffer is the offset into the 3843 * first page of the buffer plus our current offset into the buffer 3844 * itself, shifted of course... 3845 */ 3846 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3847 ASSERT(pidx < sinfo->si_max_pages); 3848 3849 /* if this cookie uses the copy buffer */ 3850 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3851 /* 3852 * NOTE: we know that since this cookie uses the copy buffer, it 3853 * is <= MMU_PAGESIZE. 3854 */ 3855 3856 /* 3857 * get the offset into the page. For the 64-bit kernel, get the 3858 * pfn which we'll use with seg kpm. 3859 */ 3860 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3861 #if defined(__amd64) 3862 /* mfn_to_pfn() is a NOP on i86pc */ 3863 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3864 #endif /* __amd64 */ 3865 3866 /* figure out if the copybuf size is a power of 2 */ 3867 if (!ISP2(dma->dp_copybuf_size)) { 3868 copybuf_sz_power_2 = B_FALSE; 3869 } else { 3870 copybuf_sz_power_2 = B_TRUE; 3871 } 3872 3873 /* This page uses the copy buffer */ 3874 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3875 3876 /* 3877 * save the copy buffer KVA that we'll use with this page. 3878 * if we still fit within the copybuf, it's a simple add. 3879 * otherwise, we need to wrap over using & or % accordingly. 3880 */ 3881 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3882 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3883 *copybuf_used; 3884 } else { 3885 if (copybuf_sz_power_2) { 3886 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3887 (uintptr_t)dma->dp_cbaddr + 3888 (*copybuf_used & 3889 (dma->dp_copybuf_size - 1))); 3890 } else { 3891 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3892 (uintptr_t)dma->dp_cbaddr + 3893 (*copybuf_used % dma->dp_copybuf_size)); 3894 } 3895 } 3896 3897 /* 3898 * over write the cookie physical address with the address of 3899 * the physical address of the copy buffer page that we will 3900 * use. 3901 */ 3902 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3903 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3904 3905 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3906 3907 /* if we have a kernel VA, it's easy, just save that address */ 3908 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3909 (sinfo->si_asp == &kas)) { 3910 /* 3911 * save away the page aligned virtual address of the 3912 * driver buffer. Offsets are handled in the sync code. 3913 */ 3914 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3915 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3916 & MMU_PAGEMASK); 3917 #if !defined(__amd64) 3918 /* 3919 * we didn't need to, and will never need to map this 3920 * page. 3921 */ 3922 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3923 #endif 3924 3925 /* we don't have a kernel VA. We need one for the bcopy. */ 3926 } else { 3927 #if defined(__amd64) 3928 /* 3929 * for the 64-bit kernel, it's easy. We use seg kpm to 3930 * get a Kernel VA for the corresponding pfn. 3931 */ 3932 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3933 #else 3934 /* 3935 * for the 32-bit kernel, this is a pain. First we'll 3936 * save away the page_t or user VA for this page. This 3937 * is needed in rootnex_dma_win() when we switch to a 3938 * new window which requires us to re-map the copy 3939 * buffer. 3940 */ 3941 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3942 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3943 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3944 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3945 } else if (pplist != NULL) { 3946 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3947 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3948 } else { 3949 dma->dp_pgmap[pidx].pm_pp = NULL; 3950 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3951 (((uintptr_t) 3952 dmar_object->dmao_obj.virt_obj.v_addr + 3953 cur_offset) & MMU_PAGEMASK); 3954 } 3955 3956 /* 3957 * save away the page aligned virtual address which was 3958 * allocated from the kernel heap arena (taking into 3959 * account if we need more copy buffer than we alloced 3960 * and use multiple windows to handle this, i.e. &,%). 3961 * NOTE: there isn't and physical memory backing up this 3962 * virtual address space currently. 3963 */ 3964 if ((*copybuf_used + MMU_PAGESIZE) <= 3965 dma->dp_copybuf_size) { 3966 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3967 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3968 MMU_PAGEMASK); 3969 } else { 3970 if (copybuf_sz_power_2) { 3971 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3972 (((uintptr_t)dma->dp_kva + 3973 (*copybuf_used & 3974 (dma->dp_copybuf_size - 1))) & 3975 MMU_PAGEMASK); 3976 } else { 3977 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3978 (((uintptr_t)dma->dp_kva + 3979 (*copybuf_used % 3980 dma->dp_copybuf_size)) & 3981 MMU_PAGEMASK); 3982 } 3983 } 3984 3985 /* 3986 * if we haven't used up the available copy buffer yet, 3987 * map the kva to the physical page. 3988 */ 3989 if (!dma->dp_cb_remaping && ((*copybuf_used + 3990 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3991 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3992 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3993 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3994 dma->dp_pgmap[pidx].pm_kaddr); 3995 } else { 3996 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3997 sinfo->si_asp, 3998 dma->dp_pgmap[pidx].pm_kaddr); 3999 } 4000 4001 /* 4002 * we've used up the available copy buffer, this page 4003 * will have to be mapped during rootnex_dma_win() when 4004 * we switch to a new window which requires a re-map 4005 * the copy buffer. (32-bit kernel only) 4006 */ 4007 } else { 4008 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4009 } 4010 #endif 4011 /* go to the next page_t */ 4012 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4013 *cur_pp = (*cur_pp)->p_next; 4014 } 4015 } 4016 4017 /* add to the copy buffer count */ 4018 *copybuf_used += MMU_PAGESIZE; 4019 4020 /* 4021 * This cookie doesn't use the copy buffer. Walk through the pages this 4022 * cookie occupies to reflect this. 4023 */ 4024 } else { 4025 /* 4026 * figure out how many pages the cookie occupies. We need to 4027 * use the original page offset of the buffer and the cookies 4028 * offset in the buffer to do this. 4029 */ 4030 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4031 pcnt = mmu_btopr(cookie->dmac_size + poff); 4032 4033 while (pcnt > 0) { 4034 #if !defined(__amd64) 4035 /* 4036 * the 32-bit kernel doesn't have seg kpm, so we need 4037 * to map in the driver buffer (if it didn't come down 4038 * with a kernel VA) on the fly. Since this page doesn't 4039 * use the copy buffer, it's not, or will it ever, have 4040 * to be mapped in. 4041 */ 4042 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4043 #endif 4044 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4045 4046 /* 4047 * we need to update pidx and cur_pp or we'll loose 4048 * track of where we are. 4049 */ 4050 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4051 *cur_pp = (*cur_pp)->p_next; 4052 } 4053 pidx++; 4054 pcnt--; 4055 } 4056 } 4057 } 4058 4059 4060 /* 4061 * rootnex_sgllen_window_boundary() 4062 * Called in the bind slow path when the next cookie causes us to exceed (in 4063 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4064 * length supported by the DMA H/W. 4065 */ 4066 static int 4067 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4068 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4069 off_t cur_offset) 4070 { 4071 off_t new_offset; 4072 size_t trim_sz; 4073 off_t coffset; 4074 4075 4076 /* 4077 * if we know we'll never have to trim, it's pretty easy. Just move to 4078 * the next window and init it. We're done. 4079 */ 4080 if (!dma->dp_trim_required) { 4081 (*windowp)++; 4082 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4083 (*windowp)->wd_cookie_cnt++; 4084 (*windowp)->wd_size = cookie->dmac_size; 4085 return (DDI_SUCCESS); 4086 } 4087 4088 /* figure out how much we need to trim from the window */ 4089 ASSERT(attr->dma_attr_granular != 0); 4090 if (dma->dp_granularity_power_2) { 4091 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4092 } else { 4093 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4094 } 4095 4096 /* The window's a whole multiple of granularity. We're done */ 4097 if (trim_sz == 0) { 4098 (*windowp)++; 4099 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4100 (*windowp)->wd_cookie_cnt++; 4101 (*windowp)->wd_size = cookie->dmac_size; 4102 return (DDI_SUCCESS); 4103 } 4104 4105 /* 4106 * The window's not a whole multiple of granularity, since we know this 4107 * is due to the sgllen, we need to go back to the last cookie and trim 4108 * that one, add the left over part of the old cookie into the new 4109 * window, and then add in the new cookie into the new window. 4110 */ 4111 4112 /* 4113 * make sure the driver isn't making us do something bad... Trimming and 4114 * sgllen == 1 don't go together. 4115 */ 4116 if (attr->dma_attr_sgllen == 1) { 4117 return (DDI_DMA_NOMAPPING); 4118 } 4119 4120 /* 4121 * first, setup the current window to account for the trim. Need to go 4122 * back to the last cookie for this. 4123 */ 4124 cookie--; 4125 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4126 (*windowp)->wd_trim.tr_last_cookie = cookie; 4127 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4128 ASSERT(cookie->dmac_size > trim_sz); 4129 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4130 (*windowp)->wd_size -= trim_sz; 4131 4132 /* save the buffer offsets for the next window */ 4133 coffset = cookie->dmac_size - trim_sz; 4134 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4135 4136 /* 4137 * set this now in case this is the first window. all other cases are 4138 * set in dma_win() 4139 */ 4140 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4141 4142 /* 4143 * initialize the next window using what's left over in the previous 4144 * cookie. 4145 */ 4146 (*windowp)++; 4147 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4148 (*windowp)->wd_cookie_cnt++; 4149 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4150 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4151 (*windowp)->wd_trim.tr_first_size = trim_sz; 4152 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4153 (*windowp)->wd_dosync = B_TRUE; 4154 } 4155 4156 /* 4157 * now go back to the current cookie and add it to the new window. set 4158 * the new window size to the what was left over from the previous 4159 * cookie and what's in the current cookie. 4160 */ 4161 cookie++; 4162 (*windowp)->wd_cookie_cnt++; 4163 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4164 4165 /* 4166 * trim plus the next cookie could put us over maxxfer (a cookie can be 4167 * a max size of maxxfer). Handle that case. 4168 */ 4169 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4170 /* 4171 * maxxfer is already a whole multiple of granularity, and this 4172 * trim will be <= the previous trim (since a cookie can't be 4173 * larger than maxxfer). Make things simple here. 4174 */ 4175 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4176 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4177 (*windowp)->wd_trim.tr_last_cookie = cookie; 4178 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4179 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4180 (*windowp)->wd_size -= trim_sz; 4181 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4182 4183 /* save the buffer offsets for the next window */ 4184 coffset = cookie->dmac_size - trim_sz; 4185 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4186 4187 /* setup the next window */ 4188 (*windowp)++; 4189 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4190 (*windowp)->wd_cookie_cnt++; 4191 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4192 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4193 coffset; 4194 (*windowp)->wd_trim.tr_first_size = trim_sz; 4195 } 4196 4197 return (DDI_SUCCESS); 4198 } 4199 4200 4201 /* 4202 * rootnex_copybuf_window_boundary() 4203 * Called in bind slowpath when we get to a window boundary because we used 4204 * up all the copy buffer that we have. 4205 */ 4206 static int 4207 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4208 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4209 size_t *copybuf_used) 4210 { 4211 rootnex_sglinfo_t *sinfo; 4212 off_t new_offset; 4213 size_t trim_sz; 4214 paddr_t paddr; 4215 off_t coffset; 4216 uint_t pidx; 4217 off_t poff; 4218 4219 4220 sinfo = &dma->dp_sglinfo; 4221 4222 /* 4223 * the copy buffer should be a whole multiple of page size. We know that 4224 * this cookie is <= MMU_PAGESIZE. 4225 */ 4226 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4227 4228 /* 4229 * from now on, all new windows in this bind need to be re-mapped during 4230 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4231 * space... 4232 */ 4233 #if !defined(__amd64) 4234 dma->dp_cb_remaping = B_TRUE; 4235 #endif 4236 4237 /* reset copybuf used */ 4238 *copybuf_used = 0; 4239 4240 /* 4241 * if we don't have to trim (since granularity is set to 1), go to the 4242 * next window and add the current cookie to it. We know the current 4243 * cookie uses the copy buffer since we're in this code path. 4244 */ 4245 if (!dma->dp_trim_required) { 4246 (*windowp)++; 4247 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4248 4249 /* Add this cookie to the new window */ 4250 (*windowp)->wd_cookie_cnt++; 4251 (*windowp)->wd_size += cookie->dmac_size; 4252 *copybuf_used += MMU_PAGESIZE; 4253 return (DDI_SUCCESS); 4254 } 4255 4256 /* 4257 * *** may need to trim, figure it out. 4258 */ 4259 4260 /* figure out how much we need to trim from the window */ 4261 if (dma->dp_granularity_power_2) { 4262 trim_sz = (*windowp)->wd_size & 4263 (hp->dmai_attr.dma_attr_granular - 1); 4264 } else { 4265 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4266 } 4267 4268 /* 4269 * if the window's a whole multiple of granularity, go to the next 4270 * window, init it, then add in the current cookie. We know the current 4271 * cookie uses the copy buffer since we're in this code path. 4272 */ 4273 if (trim_sz == 0) { 4274 (*windowp)++; 4275 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4276 4277 /* Add this cookie to the new window */ 4278 (*windowp)->wd_cookie_cnt++; 4279 (*windowp)->wd_size += cookie->dmac_size; 4280 *copybuf_used += MMU_PAGESIZE; 4281 return (DDI_SUCCESS); 4282 } 4283 4284 /* 4285 * *** We figured it out, we definitly need to trim 4286 */ 4287 4288 /* 4289 * make sure the driver isn't making us do something bad... 4290 * Trimming and sgllen == 1 don't go together. 4291 */ 4292 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4293 return (DDI_DMA_NOMAPPING); 4294 } 4295 4296 /* 4297 * first, setup the current window to account for the trim. Need to go 4298 * back to the last cookie for this. Some of the last cookie will be in 4299 * the current window, and some of the last cookie will be in the new 4300 * window. All of the current cookie will be in the new window. 4301 */ 4302 cookie--; 4303 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4304 (*windowp)->wd_trim.tr_last_cookie = cookie; 4305 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4306 ASSERT(cookie->dmac_size > trim_sz); 4307 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4308 (*windowp)->wd_size -= trim_sz; 4309 4310 /* 4311 * we're trimming the last cookie (not the current cookie). So that 4312 * last cookie may have or may not have been using the copy buffer ( 4313 * we know the cookie passed in uses the copy buffer since we're in 4314 * this code path). 4315 * 4316 * If the last cookie doesn't use the copy buffer, nothing special to 4317 * do. However, if it does uses the copy buffer, it will be both the 4318 * last page in the current window and the first page in the next 4319 * window. Since we are reusing the copy buffer (and KVA space on the 4320 * 32-bit kernel), this page will use the end of the copy buffer in the 4321 * current window, and the start of the copy buffer in the next window. 4322 * Track that info... The cookie physical address was already set to 4323 * the copy buffer physical address in setup_cookie.. 4324 */ 4325 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4326 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4327 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4328 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4329 (*windowp)->wd_trim.tr_last_pidx = pidx; 4330 (*windowp)->wd_trim.tr_last_cbaddr = 4331 dma->dp_pgmap[pidx].pm_cbaddr; 4332 #if !defined(__amd64) 4333 (*windowp)->wd_trim.tr_last_kaddr = 4334 dma->dp_pgmap[pidx].pm_kaddr; 4335 #endif 4336 } 4337 4338 /* save the buffer offsets for the next window */ 4339 coffset = cookie->dmac_size - trim_sz; 4340 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4341 4342 /* 4343 * set this now in case this is the first window. all other cases are 4344 * set in dma_win() 4345 */ 4346 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4347 4348 /* 4349 * initialize the next window using what's left over in the previous 4350 * cookie. 4351 */ 4352 (*windowp)++; 4353 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4354 (*windowp)->wd_cookie_cnt++; 4355 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4356 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4357 (*windowp)->wd_trim.tr_first_size = trim_sz; 4358 4359 /* 4360 * again, we're tracking if the last cookie uses the copy buffer. 4361 * read the comment above for more info on why we need to track 4362 * additional state. 4363 * 4364 * For the first cookie in the new window, we need reset the physical 4365 * address to DMA into to the start of the copy buffer plus any 4366 * initial page offset which may be present. 4367 */ 4368 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4369 (*windowp)->wd_dosync = B_TRUE; 4370 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4371 (*windowp)->wd_trim.tr_first_pidx = pidx; 4372 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4373 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4374 4375 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4376 poff; 4377 (*windowp)->wd_trim.tr_first_paddr = 4378 ROOTNEX_PADDR_TO_RBASE(paddr); 4379 4380 #if !defined(__amd64) 4381 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4382 #endif 4383 /* account for the cookie copybuf usage in the new window */ 4384 *copybuf_used += MMU_PAGESIZE; 4385 4386 /* 4387 * every piece of code has to have a hack, and here is this 4388 * ones :-) 4389 * 4390 * There is a complex interaction between setup_cookie and the 4391 * copybuf window boundary. The complexity had to be in either 4392 * the maxxfer window, or the copybuf window, and I chose the 4393 * copybuf code. 4394 * 4395 * So in this code path, we have taken the last cookie, 4396 * virtually broken it in half due to the trim, and it happens 4397 * to use the copybuf which further complicates life. At the 4398 * same time, we have already setup the current cookie, which 4399 * is now wrong. More background info: the current cookie uses 4400 * the copybuf, so it is only a page long max. So we need to 4401 * fix the current cookies copy buffer address, physical 4402 * address, and kva for the 32-bit kernel. We due this by 4403 * bumping them by page size (of course, we can't due this on 4404 * the physical address since the copy buffer may not be 4405 * physically contiguous). 4406 */ 4407 cookie++; 4408 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4409 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4410 4411 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4412 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4413 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4414 4415 #if !defined(__amd64) 4416 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4417 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4418 #endif 4419 } else { 4420 /* go back to the current cookie */ 4421 cookie++; 4422 } 4423 4424 /* 4425 * add the current cookie to the new window. set the new window size to 4426 * the what was left over from the previous cookie and what's in the 4427 * current cookie. 4428 */ 4429 (*windowp)->wd_cookie_cnt++; 4430 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4431 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4432 4433 /* 4434 * we know that the cookie passed in always uses the copy buffer. We 4435 * wouldn't be here if it didn't. 4436 */ 4437 *copybuf_used += MMU_PAGESIZE; 4438 4439 return (DDI_SUCCESS); 4440 } 4441 4442 4443 /* 4444 * rootnex_maxxfer_window_boundary() 4445 * Called in bind slowpath when we get to a window boundary because we will 4446 * go over maxxfer. 4447 */ 4448 static int 4449 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4450 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4451 { 4452 size_t dmac_size; 4453 off_t new_offset; 4454 size_t trim_sz; 4455 off_t coffset; 4456 4457 4458 /* 4459 * calculate how much we have to trim off of the current cookie to equal 4460 * maxxfer. We don't have to account for granularity here since our 4461 * maxxfer already takes that into account. 4462 */ 4463 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4464 ASSERT(trim_sz <= cookie->dmac_size); 4465 ASSERT(trim_sz <= dma->dp_maxxfer); 4466 4467 /* save cookie size since we need it later and we might change it */ 4468 dmac_size = cookie->dmac_size; 4469 4470 /* 4471 * if we're not trimming the entire cookie, setup the current window to 4472 * account for the trim. 4473 */ 4474 if (trim_sz < cookie->dmac_size) { 4475 (*windowp)->wd_cookie_cnt++; 4476 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4477 (*windowp)->wd_trim.tr_last_cookie = cookie; 4478 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4479 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4480 (*windowp)->wd_size = dma->dp_maxxfer; 4481 4482 /* 4483 * set the adjusted cookie size now in case this is the first 4484 * window. All other windows are taken care of in get win 4485 */ 4486 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4487 } 4488 4489 /* 4490 * coffset is the current offset within the cookie, new_offset is the 4491 * current offset with the entire buffer. 4492 */ 4493 coffset = dmac_size - trim_sz; 4494 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4495 4496 /* initialize the next window */ 4497 (*windowp)++; 4498 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4499 (*windowp)->wd_cookie_cnt++; 4500 (*windowp)->wd_size = trim_sz; 4501 if (trim_sz < dmac_size) { 4502 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4503 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4504 coffset; 4505 (*windowp)->wd_trim.tr_first_size = trim_sz; 4506 } 4507 4508 return (DDI_SUCCESS); 4509 } 4510 4511 4512 /*ARGSUSED*/ 4513 static int 4514 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4515 off_t off, size_t len, uint_t cache_flags) 4516 { 4517 rootnex_sglinfo_t *sinfo; 4518 rootnex_pgmap_t *cbpage; 4519 rootnex_window_t *win; 4520 ddi_dma_impl_t *hp; 4521 rootnex_dma_t *dma; 4522 caddr_t fromaddr; 4523 caddr_t toaddr; 4524 uint_t psize; 4525 off_t offset; 4526 uint_t pidx; 4527 size_t size; 4528 off_t poff; 4529 int e; 4530 4531 4532 hp = (ddi_dma_impl_t *)handle; 4533 dma = (rootnex_dma_t *)hp->dmai_private; 4534 sinfo = &dma->dp_sglinfo; 4535 4536 /* 4537 * if we don't have any windows, we don't need to sync. A copybuf 4538 * will cause us to have at least one window. 4539 */ 4540 if (dma->dp_window == NULL) { 4541 return (DDI_SUCCESS); 4542 } 4543 4544 /* This window may not need to be sync'd */ 4545 win = &dma->dp_window[dma->dp_current_win]; 4546 if (!win->wd_dosync) { 4547 return (DDI_SUCCESS); 4548 } 4549 4550 /* handle off and len special cases */ 4551 if ((off == 0) || (rootnex_sync_ignore_params)) { 4552 offset = win->wd_offset; 4553 } else { 4554 offset = off; 4555 } 4556 if ((len == 0) || (rootnex_sync_ignore_params)) { 4557 size = win->wd_size; 4558 } else { 4559 size = len; 4560 } 4561 4562 /* check the sync args to make sure they make a little sense */ 4563 if (rootnex_sync_check_parms) { 4564 e = rootnex_valid_sync_parms(hp, win, offset, size, 4565 cache_flags); 4566 if (e != DDI_SUCCESS) { 4567 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4568 return (DDI_FAILURE); 4569 } 4570 } 4571 4572 /* 4573 * special case the first page to handle the offset into the page. The 4574 * offset to the current page for our buffer is the offset into the 4575 * first page of the buffer plus our current offset into the buffer 4576 * itself, masked of course. 4577 */ 4578 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4579 psize = MIN((MMU_PAGESIZE - poff), size); 4580 4581 /* go through all the pages that we want to sync */ 4582 while (size > 0) { 4583 /* 4584 * Calculate the page index relative to the start of the buffer. 4585 * The index to the current page for our buffer is the offset 4586 * into the first page of the buffer plus our current offset 4587 * into the buffer itself, shifted of course... 4588 */ 4589 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4590 ASSERT(pidx < sinfo->si_max_pages); 4591 4592 /* 4593 * if this page uses the copy buffer, we need to sync it, 4594 * otherwise, go on to the next page. 4595 */ 4596 cbpage = &dma->dp_pgmap[pidx]; 4597 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4598 (cbpage->pm_uses_copybuf == B_FALSE)); 4599 if (cbpage->pm_uses_copybuf) { 4600 /* cbaddr and kaddr should be page aligned */ 4601 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4602 MMU_PAGEOFFSET) == 0); 4603 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4604 MMU_PAGEOFFSET) == 0); 4605 4606 /* 4607 * if we're copying for the device, we are going to 4608 * copy from the drivers buffer and to the rootnex 4609 * allocated copy buffer. 4610 */ 4611 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4612 fromaddr = cbpage->pm_kaddr + poff; 4613 toaddr = cbpage->pm_cbaddr + poff; 4614 ROOTNEX_DPROBE2(rootnex__sync__dev, 4615 dev_info_t *, dma->dp_dip, size_t, psize); 4616 4617 /* 4618 * if we're copying for the cpu/kernel, we are going to 4619 * copy from the rootnex allocated copy buffer to the 4620 * drivers buffer. 4621 */ 4622 } else { 4623 fromaddr = cbpage->pm_cbaddr + poff; 4624 toaddr = cbpage->pm_kaddr + poff; 4625 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4626 dev_info_t *, dma->dp_dip, size_t, psize); 4627 } 4628 4629 bcopy(fromaddr, toaddr, psize); 4630 } 4631 4632 /* 4633 * decrement size until we're done, update our offset into the 4634 * buffer, and get the next page size. 4635 */ 4636 size -= psize; 4637 offset += psize; 4638 psize = MIN(MMU_PAGESIZE, size); 4639 4640 /* page offset is zero for the rest of this loop */ 4641 poff = 0; 4642 } 4643 4644 return (DDI_SUCCESS); 4645 } 4646 4647 /* 4648 * rootnex_dma_sync() 4649 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4650 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4651 * is set, ddi_dma_sync() returns immediately passing back success. 4652 */ 4653 /*ARGSUSED*/ 4654 static int 4655 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4656 off_t off, size_t len, uint_t cache_flags) 4657 { 4658 #if defined(__amd64) && !defined(__xpv) 4659 if (IOMMU_USED(rdip)) { 4660 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4661 cache_flags)); 4662 } 4663 #endif 4664 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4665 cache_flags)); 4666 } 4667 4668 /* 4669 * rootnex_valid_sync_parms() 4670 * checks the parameters passed to sync to verify they are correct. 4671 */ 4672 static int 4673 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4674 off_t offset, size_t size, uint_t cache_flags) 4675 { 4676 off_t woffset; 4677 4678 4679 /* 4680 * the first part of the test to make sure the offset passed in is 4681 * within the window. 4682 */ 4683 if (offset < win->wd_offset) { 4684 return (DDI_FAILURE); 4685 } 4686 4687 /* 4688 * second and last part of the test to make sure the offset and length 4689 * passed in is within the window. 4690 */ 4691 woffset = offset - win->wd_offset; 4692 if ((woffset + size) > win->wd_size) { 4693 return (DDI_FAILURE); 4694 } 4695 4696 /* 4697 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4698 * be set too. 4699 */ 4700 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4701 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4702 return (DDI_SUCCESS); 4703 } 4704 4705 /* 4706 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4707 * should be set. Also DDI_DMA_READ should be set in the flags. 4708 */ 4709 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4710 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4711 (hp->dmai_rflags & DDI_DMA_READ)) { 4712 return (DDI_SUCCESS); 4713 } 4714 4715 return (DDI_FAILURE); 4716 } 4717 4718 4719 /*ARGSUSED*/ 4720 static int 4721 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4722 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4723 uint_t *ccountp) 4724 { 4725 rootnex_window_t *window; 4726 rootnex_trim_t *trim; 4727 ddi_dma_impl_t *hp; 4728 rootnex_dma_t *dma; 4729 ddi_dma_obj_t *dmao; 4730 #if !defined(__amd64) 4731 rootnex_sglinfo_t *sinfo; 4732 rootnex_pgmap_t *pmap; 4733 uint_t pidx; 4734 uint_t pcnt; 4735 off_t poff; 4736 int i; 4737 #endif 4738 4739 4740 hp = (ddi_dma_impl_t *)handle; 4741 dma = (rootnex_dma_t *)hp->dmai_private; 4742 #if !defined(__amd64) 4743 sinfo = &dma->dp_sglinfo; 4744 #endif 4745 4746 /* If we try and get a window which doesn't exist, return failure */ 4747 if (win >= hp->dmai_nwin) { 4748 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4749 return (DDI_FAILURE); 4750 } 4751 4752 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4753 4754 /* 4755 * if we don't have any windows, and they're asking for the first 4756 * window, setup the cookie pointer to the first cookie in the bind. 4757 * setup our return values, then increment the cookie since we return 4758 * the first cookie on the stack. 4759 */ 4760 if (dma->dp_window == NULL) { 4761 if (win != 0) { 4762 ROOTNEX_DPROF_INC( 4763 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4764 return (DDI_FAILURE); 4765 } 4766 hp->dmai_cookie = dma->dp_cookies; 4767 *offp = 0; 4768 *lenp = dmao->dmao_size; 4769 *ccountp = dma->dp_sglinfo.si_sgl_size; 4770 *cookiep = hp->dmai_cookie[0]; 4771 hp->dmai_cookie++; 4772 return (DDI_SUCCESS); 4773 } 4774 4775 /* sync the old window before moving on to the new one */ 4776 window = &dma->dp_window[dma->dp_current_win]; 4777 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4778 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4779 DDI_DMA_SYNC_FORCPU); 4780 } 4781 4782 #if !defined(__amd64) 4783 /* 4784 * before we move to the next window, if we need to re-map, unmap all 4785 * the pages in this window. 4786 */ 4787 if (dma->dp_cb_remaping) { 4788 /* 4789 * If we switch to this window again, we'll need to map in 4790 * on the fly next time. 4791 */ 4792 window->wd_remap_copybuf = B_TRUE; 4793 4794 /* 4795 * calculate the page index into the buffer where this window 4796 * starts, and the number of pages this window takes up. 4797 */ 4798 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4799 MMU_PAGESHIFT; 4800 poff = (sinfo->si_buf_offset + window->wd_offset) & 4801 MMU_PAGEOFFSET; 4802 pcnt = mmu_btopr(window->wd_size + poff); 4803 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4804 4805 /* unmap pages which are currently mapped in this window */ 4806 for (i = 0; i < pcnt; i++) { 4807 if (dma->dp_pgmap[pidx].pm_mapped) { 4808 hat_unload(kas.a_hat, 4809 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4810 HAT_UNLOAD); 4811 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4812 } 4813 pidx++; 4814 } 4815 } 4816 #endif 4817 4818 /* 4819 * Move to the new window. 4820 * NOTE: current_win must be set for sync to work right 4821 */ 4822 dma->dp_current_win = win; 4823 window = &dma->dp_window[win]; 4824 4825 /* if needed, adjust the first and/or last cookies for trim */ 4826 trim = &window->wd_trim; 4827 if (trim->tr_trim_first) { 4828 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4829 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4830 #if !defined(__amd64) 4831 window->wd_first_cookie->dmac_type = 4832 (window->wd_first_cookie->dmac_type & 4833 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4834 #endif 4835 if (trim->tr_first_copybuf_win) { 4836 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4837 trim->tr_first_cbaddr; 4838 #if !defined(__amd64) 4839 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4840 trim->tr_first_kaddr; 4841 #endif 4842 } 4843 } 4844 if (trim->tr_trim_last) { 4845 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4846 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4847 if (trim->tr_last_copybuf_win) { 4848 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4849 trim->tr_last_cbaddr; 4850 #if !defined(__amd64) 4851 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4852 trim->tr_last_kaddr; 4853 #endif 4854 } 4855 } 4856 4857 /* 4858 * setup the cookie pointer to the first cookie in the window. setup 4859 * our return values, then increment the cookie since we return the 4860 * first cookie on the stack. 4861 */ 4862 hp->dmai_cookie = window->wd_first_cookie; 4863 *offp = window->wd_offset; 4864 *lenp = window->wd_size; 4865 *ccountp = window->wd_cookie_cnt; 4866 *cookiep = hp->dmai_cookie[0]; 4867 hp->dmai_cookie++; 4868 4869 #if !defined(__amd64) 4870 /* re-map copybuf if required for this window */ 4871 if (dma->dp_cb_remaping) { 4872 /* 4873 * calculate the page index into the buffer where this 4874 * window starts. 4875 */ 4876 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4877 MMU_PAGESHIFT; 4878 ASSERT(pidx < sinfo->si_max_pages); 4879 4880 /* 4881 * the first page can get unmapped if it's shared with the 4882 * previous window. Even if the rest of this window is already 4883 * mapped in, we need to still check this one. 4884 */ 4885 pmap = &dma->dp_pgmap[pidx]; 4886 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4887 if (pmap->pm_pp != NULL) { 4888 pmap->pm_mapped = B_TRUE; 4889 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4890 } else if (pmap->pm_vaddr != NULL) { 4891 pmap->pm_mapped = B_TRUE; 4892 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4893 pmap->pm_kaddr); 4894 } 4895 } 4896 pidx++; 4897 4898 /* map in the rest of the pages if required */ 4899 if (window->wd_remap_copybuf) { 4900 window->wd_remap_copybuf = B_FALSE; 4901 4902 /* figure out many pages this window takes up */ 4903 poff = (sinfo->si_buf_offset + window->wd_offset) & 4904 MMU_PAGEOFFSET; 4905 pcnt = mmu_btopr(window->wd_size + poff); 4906 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4907 4908 /* map pages which require it */ 4909 for (i = 1; i < pcnt; i++) { 4910 pmap = &dma->dp_pgmap[pidx]; 4911 if (pmap->pm_uses_copybuf) { 4912 ASSERT(pmap->pm_mapped == B_FALSE); 4913 if (pmap->pm_pp != NULL) { 4914 pmap->pm_mapped = B_TRUE; 4915 i86_pp_map(pmap->pm_pp, 4916 pmap->pm_kaddr); 4917 } else if (pmap->pm_vaddr != NULL) { 4918 pmap->pm_mapped = B_TRUE; 4919 i86_va_map(pmap->pm_vaddr, 4920 sinfo->si_asp, 4921 pmap->pm_kaddr); 4922 } 4923 } 4924 pidx++; 4925 } 4926 } 4927 } 4928 #endif 4929 4930 /* if the new window uses the copy buffer, sync it for the device */ 4931 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4932 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4933 DDI_DMA_SYNC_FORDEV); 4934 } 4935 4936 return (DDI_SUCCESS); 4937 } 4938 4939 /* 4940 * rootnex_dma_win() 4941 * called from ddi_dma_getwin() 4942 */ 4943 /*ARGSUSED*/ 4944 static int 4945 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4946 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4947 uint_t *ccountp) 4948 { 4949 #if defined(__amd64) && !defined(__xpv) 4950 if (IOMMU_USED(rdip)) { 4951 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4952 cookiep, ccountp)); 4953 } 4954 #endif 4955 4956 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4957 cookiep, ccountp)); 4958 } 4959 4960 #if defined(__amd64) && !defined(__xpv) 4961 /*ARGSUSED*/ 4962 static int 4963 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4964 ddi_dma_handle_t handle, void *v) 4965 { 4966 ddi_dma_impl_t *hp; 4967 rootnex_dma_t *dma; 4968 4969 hp = (ddi_dma_impl_t *)handle; 4970 dma = (rootnex_dma_t *)hp->dmai_private; 4971 dma->dp_iommu_private = v; 4972 4973 return (DDI_SUCCESS); 4974 } 4975 4976 /*ARGSUSED*/ 4977 static void * 4978 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4979 ddi_dma_handle_t handle) 4980 { 4981 ddi_dma_impl_t *hp; 4982 rootnex_dma_t *dma; 4983 4984 hp = (ddi_dma_impl_t *)handle; 4985 dma = (rootnex_dma_t *)hp->dmai_private; 4986 4987 return (dma->dp_iommu_private); 4988 } 4989 #endif 4990 4991 /* 4992 * ************************ 4993 * obsoleted dma routines 4994 * ************************ 4995 */ 4996 4997 /* 4998 * rootnex_dma_mctl() 4999 * 5000 * We don't support this legacy interface any more on x86. 5001 */ 5002 /* ARGSUSED */ 5003 static int 5004 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5005 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5006 uint_t cache_flags) 5007 { 5008 /* 5009 * The only thing dma_mctl is usef for anymore is legacy SPARC 5010 * dvma and sbus-specific routines. 5011 */ 5012 return (DDI_FAILURE); 5013 } 5014 5015 /* 5016 * ********* 5017 * FMA Code 5018 * ********* 5019 */ 5020 5021 /* 5022 * rootnex_fm_init() 5023 * FMA init busop 5024 */ 5025 /* ARGSUSED */ 5026 static int 5027 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5028 ddi_iblock_cookie_t *ibc) 5029 { 5030 *ibc = rootnex_state->r_err_ibc; 5031 5032 return (ddi_system_fmcap); 5033 } 5034 5035 /* 5036 * rootnex_dma_check() 5037 * Function called after a dma fault occurred to find out whether the 5038 * fault address is associated with a driver that is able to handle faults 5039 * and recover from faults. 5040 */ 5041 /* ARGSUSED */ 5042 static int 5043 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5044 const void *not_used) 5045 { 5046 rootnex_window_t *window; 5047 uint64_t start_addr; 5048 uint64_t fault_addr; 5049 ddi_dma_impl_t *hp; 5050 rootnex_dma_t *dma; 5051 uint64_t end_addr; 5052 size_t csize; 5053 int i; 5054 int j; 5055 5056 5057 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5058 hp = (ddi_dma_impl_t *)handle; 5059 ASSERT(hp); 5060 5061 dma = (rootnex_dma_t *)hp->dmai_private; 5062 5063 /* Get the address that we need to search for */ 5064 fault_addr = *(uint64_t *)addr; 5065 5066 /* 5067 * if we don't have any windows, we can just walk through all the 5068 * cookies. 5069 */ 5070 if (dma->dp_window == NULL) { 5071 /* for each cookie */ 5072 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5073 /* 5074 * if the faulted address is within the physical address 5075 * range of the cookie, return DDI_FM_NONFATAL. 5076 */ 5077 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5078 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5079 dma->dp_cookies[i].dmac_size))) { 5080 return (DDI_FM_NONFATAL); 5081 } 5082 } 5083 5084 /* fault_addr not within this DMA handle */ 5085 return (DDI_FM_UNKNOWN); 5086 } 5087 5088 /* we have mutiple windows, walk through each window */ 5089 for (i = 0; i < hp->dmai_nwin; i++) { 5090 window = &dma->dp_window[i]; 5091 5092 /* Go through all the cookies in the window */ 5093 for (j = 0; j < window->wd_cookie_cnt; j++) { 5094 5095 start_addr = window->wd_first_cookie[j].dmac_laddress; 5096 csize = window->wd_first_cookie[j].dmac_size; 5097 5098 /* 5099 * if we are trimming the first cookie in the window, 5100 * and this is the first cookie, adjust the start 5101 * address and size of the cookie to account for the 5102 * trim. 5103 */ 5104 if (window->wd_trim.tr_trim_first && (j == 0)) { 5105 start_addr = window->wd_trim.tr_first_paddr; 5106 csize = window->wd_trim.tr_first_size; 5107 } 5108 5109 /* 5110 * if we are trimming the last cookie in the window, 5111 * and this is the last cookie, adjust the start 5112 * address and size of the cookie to account for the 5113 * trim. 5114 */ 5115 if (window->wd_trim.tr_trim_last && 5116 (j == (window->wd_cookie_cnt - 1))) { 5117 start_addr = window->wd_trim.tr_last_paddr; 5118 csize = window->wd_trim.tr_last_size; 5119 } 5120 5121 end_addr = start_addr + csize; 5122 5123 /* 5124 * if the faulted address is within the physical 5125 * address of the cookie, return DDI_FM_NONFATAL. 5126 */ 5127 if ((fault_addr >= start_addr) && 5128 (fault_addr <= end_addr)) { 5129 return (DDI_FM_NONFATAL); 5130 } 5131 } 5132 } 5133 5134 /* fault_addr not within this DMA handle */ 5135 return (DDI_FM_UNKNOWN); 5136 } 5137 5138 /*ARGSUSED*/ 5139 static int 5140 rootnex_quiesce(dev_info_t *dip) 5141 { 5142 #if defined(__amd64) && !defined(__xpv) 5143 return (immu_quiesce()); 5144 #else 5145 return (DDI_SUCCESS); 5146 #endif 5147 } 5148 5149 #if defined(__xpv) 5150 void 5151 immu_init(void) 5152 { 5153 ; 5154 } 5155 5156 void 5157 immu_startup(void) 5158 { 5159 ; 5160 } 5161 /*ARGSUSED*/ 5162 void 5163 immu_physmem_update(uint64_t addr, uint64_t size) 5164 { 5165 ; 5166 } 5167 #endif 5168