1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 * Copyright 2017 Joyent, Inc. 29 * Copyright 2020 Ryan Zezeski 30 */ 31 32 /* 33 * x86 root nexus driver 34 */ 35 36 #include <sys/sysmacros.h> 37 #include <sys/conf.h> 38 #include <sys/autoconf.h> 39 #include <sys/sysmacros.h> 40 #include <sys/debug.h> 41 #include <sys/psw.h> 42 #include <sys/ddidmareq.h> 43 #include <sys/promif.h> 44 #include <sys/devops.h> 45 #include <sys/kmem.h> 46 #include <sys/cmn_err.h> 47 #include <vm/seg.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_dev.h> 50 #include <sys/vmem.h> 51 #include <sys/mman.h> 52 #include <vm/hat.h> 53 #include <vm/as.h> 54 #include <vm/page.h> 55 #include <sys/avintr.h> 56 #include <sys/errno.h> 57 #include <sys/modctl.h> 58 #include <sys/ddi_impldefs.h> 59 #include <sys/sunddi.h> 60 #include <sys/sunndi.h> 61 #include <sys/mach_intr.h> 62 #include <sys/psm.h> 63 #include <sys/ontrap.h> 64 #include <sys/atomic.h> 65 #include <sys/sdt.h> 66 #include <sys/rootnex.h> 67 #include <vm/hat_i86.h> 68 #include <sys/ddifm.h> 69 #include <sys/ddi_isa.h> 70 #include <sys/apic.h> 71 72 #ifdef __xpv 73 #include <sys/bootinfo.h> 74 #include <sys/hypervisor.h> 75 #include <sys/bootconf.h> 76 #include <vm/kboot_mmu.h> 77 #endif 78 79 #if defined(__amd64) && !defined(__xpv) 80 #include <sys/immu.h> 81 #endif 82 83 84 /* 85 * enable/disable extra checking of function parameters. Useful for debugging 86 * drivers. 87 */ 88 #ifdef DEBUG 89 int rootnex_alloc_check_parms = 1; 90 int rootnex_bind_check_parms = 1; 91 int rootnex_bind_check_inuse = 1; 92 int rootnex_unbind_verify_buffer = 0; 93 int rootnex_sync_check_parms = 1; 94 #else 95 int rootnex_alloc_check_parms = 0; 96 int rootnex_bind_check_parms = 0; 97 int rootnex_bind_check_inuse = 0; 98 int rootnex_unbind_verify_buffer = 0; 99 int rootnex_sync_check_parms = 0; 100 #endif 101 102 boolean_t rootnex_dmar_not_setup; 103 104 /* Master Abort and Target Abort panic flag */ 105 int rootnex_fm_ma_ta_panic_flag = 0; 106 107 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 108 int rootnex_bind_fail = 1; 109 int rootnex_bind_warn = 1; 110 uint8_t *rootnex_warn_list; 111 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 112 #define ROOTNEX_BIND_WARNING (0x1 << 0) 113 114 /* 115 * revert back to old broken behavior of always sync'ing entire copy buffer. 116 * This is useful if be have a buggy driver which doesn't correctly pass in 117 * the offset and size into ddi_dma_sync(). 118 */ 119 int rootnex_sync_ignore_params = 0; 120 121 /* 122 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 123 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 124 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 125 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 126 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 127 * (< 8K). We will still need to allocate the copy buffer during bind though 128 * (if we need one). These can only be modified in /etc/system before rootnex 129 * attach. 130 */ 131 #if defined(__amd64) 132 int rootnex_prealloc_cookies = 65; 133 int rootnex_prealloc_windows = 4; 134 int rootnex_prealloc_copybuf = 2; 135 #else 136 int rootnex_prealloc_cookies = 33; 137 int rootnex_prealloc_windows = 4; 138 int rootnex_prealloc_copybuf = 2; 139 #endif 140 141 /* driver global state */ 142 static rootnex_state_t *rootnex_state; 143 144 #ifdef DEBUG 145 /* shortcut to rootnex counters */ 146 static uint64_t *rootnex_cnt; 147 #endif 148 149 /* 150 * XXX - does x86 even need these or are they left over from the SPARC days? 151 */ 152 /* statically defined integer/boolean properties for the root node */ 153 static rootnex_intprop_t rootnex_intprp[] = { 154 { "PAGESIZE", PAGESIZE }, 155 { "MMU_PAGESIZE", MMU_PAGESIZE }, 156 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 157 { DDI_RELATIVE_ADDRESSING, 1 }, 158 }; 159 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 160 161 /* 162 * If we're dom0, we're using a real device so we need to load 163 * the cookies with MFNs instead of PFNs. 164 */ 165 #ifdef __xpv 166 typedef maddr_t rootnex_addr_t; 167 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 168 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 169 #else 170 typedef paddr_t rootnex_addr_t; 171 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 172 #endif 173 174 static struct cb_ops rootnex_cb_ops = { 175 nodev, /* open */ 176 nodev, /* close */ 177 nodev, /* strategy */ 178 nodev, /* print */ 179 nodev, /* dump */ 180 nodev, /* read */ 181 nodev, /* write */ 182 nodev, /* ioctl */ 183 nodev, /* devmap */ 184 nodev, /* mmap */ 185 nodev, /* segmap */ 186 nochpoll, /* chpoll */ 187 ddi_prop_op, /* cb_prop_op */ 188 NULL, /* struct streamtab */ 189 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 190 CB_REV, /* Rev */ 191 nodev, /* cb_aread */ 192 nodev /* cb_awrite */ 193 }; 194 195 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 196 off_t offset, off_t len, caddr_t *vaddrp); 197 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 198 struct hat *hat, struct seg *seg, caddr_t addr, 199 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 200 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 202 ddi_dma_handle_t *handlep); 203 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle); 205 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 206 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 207 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 208 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 209 ddi_dma_handle_t handle); 210 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 211 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 212 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 213 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 214 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 215 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 216 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 217 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 218 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 219 ddi_ctl_enum_t ctlop, void *arg, void *result); 220 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 221 ddi_iblock_cookie_t *ibc); 222 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 223 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 224 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 225 void *); 226 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 227 228 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 229 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 230 ddi_dma_handle_t *handlep); 231 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 232 ddi_dma_handle_t handle); 233 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 234 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 235 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 236 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 237 ddi_dma_handle_t handle); 238 #if defined(__amd64) && !defined(__xpv) 239 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 240 ddi_dma_handle_t handle); 241 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 242 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 243 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 244 ddi_dma_cookie_t *cookiep, uint_t ccount); 245 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 246 ddi_dma_handle_t handle); 247 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 248 #endif 249 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 250 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 251 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 252 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 253 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 254 255 #if defined(__amd64) && !defined(__xpv) 256 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 257 ddi_dma_handle_t handle, void *v); 258 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 259 ddi_dma_handle_t handle); 260 #endif 261 262 263 static struct bus_ops rootnex_bus_ops = { 264 BUSO_REV, 265 rootnex_map, 266 NULL, 267 NULL, 268 NULL, 269 rootnex_map_fault, 270 0, 271 rootnex_dma_allochdl, 272 rootnex_dma_freehdl, 273 rootnex_dma_bindhdl, 274 rootnex_dma_unbindhdl, 275 rootnex_dma_sync, 276 rootnex_dma_win, 277 rootnex_dma_mctl, 278 rootnex_ctlops, 279 ddi_bus_prop_op, 280 i_ddi_rootnex_get_eventcookie, 281 i_ddi_rootnex_add_eventcall, 282 i_ddi_rootnex_remove_eventcall, 283 i_ddi_rootnex_post_event, 284 0, /* bus_intr_ctl */ 285 0, /* bus_config */ 286 0, /* bus_unconfig */ 287 rootnex_fm_init, /* bus_fm_init */ 288 NULL, /* bus_fm_fini */ 289 NULL, /* bus_fm_access_enter */ 290 NULL, /* bus_fm_access_exit */ 291 NULL, /* bus_powr */ 292 rootnex_intr_ops /* bus_intr_op */ 293 }; 294 295 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 296 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 297 static int rootnex_quiesce(dev_info_t *dip); 298 299 static struct dev_ops rootnex_ops = { 300 DEVO_REV, 301 0, 302 ddi_no_info, 303 nulldev, 304 nulldev, 305 rootnex_attach, 306 rootnex_detach, 307 nulldev, 308 &rootnex_cb_ops, 309 &rootnex_bus_ops, 310 NULL, 311 rootnex_quiesce, /* quiesce */ 312 }; 313 314 static struct modldrv rootnex_modldrv = { 315 &mod_driverops, 316 "i86pc root nexus", 317 &rootnex_ops 318 }; 319 320 static struct modlinkage rootnex_modlinkage = { 321 MODREV_1, 322 (void *)&rootnex_modldrv, 323 NULL 324 }; 325 326 #if defined(__amd64) && !defined(__xpv) 327 static iommulib_nexops_t iommulib_nexops = { 328 IOMMU_NEXOPS_VERSION, 329 "Rootnex IOMMU ops Vers 1.1", 330 NULL, 331 rootnex_coredma_allochdl, 332 rootnex_coredma_freehdl, 333 rootnex_coredma_bindhdl, 334 rootnex_coredma_unbindhdl, 335 rootnex_coredma_reset_cookies, 336 rootnex_coredma_get_cookies, 337 rootnex_coredma_set_cookies, 338 rootnex_coredma_clear_cookies, 339 rootnex_coredma_get_sleep_flags, 340 rootnex_coredma_sync, 341 rootnex_coredma_win, 342 rootnex_coredma_hdl_setprivate, 343 rootnex_coredma_hdl_getprivate 344 }; 345 #endif 346 347 /* 348 * extern hacks 349 */ 350 extern struct seg_ops segdev_ops; 351 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 352 #ifdef DDI_MAP_DEBUG 353 extern int ddi_map_debug_flag; 354 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 355 #endif 356 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 357 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 358 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 359 psm_intr_op_t, int *); 360 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 361 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 362 363 /* 364 * Use device arena to use for device control register mappings. 365 * Various kernel memory walkers (debugger, dtrace) need to know 366 * to avoid this address range to prevent undesired device activity. 367 */ 368 extern void *device_arena_alloc(size_t size, int vm_flag); 369 extern void device_arena_free(void * vaddr, size_t size); 370 371 372 /* 373 * Internal functions 374 */ 375 static int rootnex_dma_init(); 376 static void rootnex_add_props(dev_info_t *); 377 static int rootnex_ctl_reportdev(dev_info_t *dip); 378 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 379 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 380 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 381 static int rootnex_map_handle(ddi_map_req_t *mp); 382 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 383 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 384 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 385 ddi_dma_attr_t *attr); 386 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 387 rootnex_sglinfo_t *sglinfo); 388 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 389 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 390 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 391 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 392 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 393 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 394 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 395 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 396 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 397 static void rootnex_teardown_windows(rootnex_dma_t *dma); 398 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 399 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 400 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 401 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 402 size_t *copybuf_used, page_t **cur_pp); 403 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 404 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 405 ddi_dma_attr_t *attr, off_t cur_offset); 406 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 407 rootnex_dma_t *dma, rootnex_window_t **windowp, 408 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 409 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 410 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 411 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 412 off_t offset, size_t size, uint_t cache_flags); 413 static int rootnex_verify_buffer(rootnex_dma_t *dma); 414 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 415 const void *comp_addr, const void *not_used); 416 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 417 rootnex_sglinfo_t *sglinfo); 418 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 419 420 /* 421 * _init() 422 * 423 */ 424 int 425 _init(void) 426 { 427 428 rootnex_state = NULL; 429 return (mod_install(&rootnex_modlinkage)); 430 } 431 432 433 /* 434 * _info() 435 * 436 */ 437 int 438 _info(struct modinfo *modinfop) 439 { 440 return (mod_info(&rootnex_modlinkage, modinfop)); 441 } 442 443 444 /* 445 * _fini() 446 * 447 */ 448 int 449 _fini(void) 450 { 451 return (EBUSY); 452 } 453 454 455 /* 456 * rootnex_attach() 457 * 458 */ 459 static int 460 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 461 { 462 int fmcap; 463 int e; 464 465 switch (cmd) { 466 case DDI_ATTACH: 467 break; 468 case DDI_RESUME: 469 #if defined(__amd64) && !defined(__xpv) 470 return (immu_unquiesce()); 471 #else 472 return (DDI_SUCCESS); 473 #endif 474 default: 475 return (DDI_FAILURE); 476 } 477 478 /* 479 * We should only have one instance of rootnex. Save it away since we 480 * don't have an easy way to get it back later. 481 */ 482 ASSERT(rootnex_state == NULL); 483 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 484 485 rootnex_state->r_dip = dip; 486 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 487 rootnex_state->r_reserved_msg_printed = B_FALSE; 488 #ifdef DEBUG 489 rootnex_cnt = &rootnex_state->r_counters[0]; 490 #endif 491 492 /* 493 * Set minimum fm capability level for i86pc platforms and then 494 * initialize error handling. Since we're the rootnex, we don't 495 * care what's returned in the fmcap field. 496 */ 497 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 498 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 499 fmcap = ddi_system_fmcap; 500 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 501 502 /* initialize DMA related state */ 503 e = rootnex_dma_init(); 504 if (e != DDI_SUCCESS) { 505 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 506 return (DDI_FAILURE); 507 } 508 509 /* Add static root node properties */ 510 rootnex_add_props(dip); 511 512 /* since we can't call ddi_report_dev() */ 513 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 514 515 /* Initialize rootnex event handle */ 516 i_ddi_rootnex_init_events(dip); 517 518 #if defined(__amd64) && !defined(__xpv) 519 e = iommulib_nexus_register(dip, &iommulib_nexops, 520 &rootnex_state->r_iommulib_handle); 521 522 ASSERT(e == DDI_SUCCESS); 523 #endif 524 525 return (DDI_SUCCESS); 526 } 527 528 529 /* 530 * rootnex_detach() 531 * 532 */ 533 /*ARGSUSED*/ 534 static int 535 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 536 { 537 switch (cmd) { 538 case DDI_SUSPEND: 539 #if defined(__amd64) && !defined(__xpv) 540 return (immu_quiesce()); 541 #else 542 return (DDI_SUCCESS); 543 #endif 544 default: 545 return (DDI_FAILURE); 546 } 547 /*NOTREACHED*/ 548 549 } 550 551 552 /* 553 * rootnex_dma_init() 554 * 555 */ 556 /*ARGSUSED*/ 557 static int 558 rootnex_dma_init() 559 { 560 size_t bufsize; 561 562 563 /* 564 * size of our cookie/window/copybuf state needed in dma bind that we 565 * pre-alloc in dma_alloc_handle 566 */ 567 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 568 rootnex_state->r_prealloc_size = 569 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 570 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 571 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 572 573 /* 574 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 575 * allocate 16 extra bytes for struct pointer alignment 576 * (p->dmai_private & dma->dp_prealloc_buffer) 577 */ 578 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 579 rootnex_state->r_prealloc_size + 0x10; 580 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 581 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 582 if (rootnex_state->r_dmahdl_cache == NULL) { 583 return (DDI_FAILURE); 584 } 585 586 /* 587 * allocate array to track which major numbers we have printed warnings 588 * for. 589 */ 590 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 591 KM_SLEEP); 592 593 return (DDI_SUCCESS); 594 } 595 596 597 /* 598 * rootnex_add_props() 599 * 600 */ 601 static void 602 rootnex_add_props(dev_info_t *dip) 603 { 604 rootnex_intprop_t *rpp; 605 int i; 606 607 /* Add static integer/boolean properties to the root node */ 608 rpp = rootnex_intprp; 609 for (i = 0; i < NROOT_INTPROPS; i++) { 610 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 611 rpp[i].prop_name, rpp[i].prop_value); 612 } 613 } 614 615 616 617 /* 618 * ************************* 619 * ctlops related routines 620 * ************************* 621 */ 622 623 /* 624 * rootnex_ctlops() 625 * 626 */ 627 /*ARGSUSED*/ 628 static int 629 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 630 void *arg, void *result) 631 { 632 int n, *ptr; 633 struct ddi_parent_private_data *pdp; 634 635 switch (ctlop) { 636 case DDI_CTLOPS_DMAPMAPC: 637 /* 638 * Return 'partial' to indicate that dma mapping 639 * has to be done in the main MMU. 640 */ 641 return (DDI_DMA_PARTIAL); 642 643 case DDI_CTLOPS_BTOP: 644 /* 645 * Convert byte count input to physical page units. 646 * (byte counts that are not a page-size multiple 647 * are rounded down) 648 */ 649 *(ulong_t *)result = btop(*(ulong_t *)arg); 650 return (DDI_SUCCESS); 651 652 case DDI_CTLOPS_PTOB: 653 /* 654 * Convert size in physical pages to bytes 655 */ 656 *(ulong_t *)result = ptob(*(ulong_t *)arg); 657 return (DDI_SUCCESS); 658 659 case DDI_CTLOPS_BTOPR: 660 /* 661 * Convert byte count input to physical page units 662 * (byte counts that are not a page-size multiple 663 * are rounded up) 664 */ 665 *(ulong_t *)result = btopr(*(ulong_t *)arg); 666 return (DDI_SUCCESS); 667 668 case DDI_CTLOPS_INITCHILD: 669 return (impl_ddi_sunbus_initchild(arg)); 670 671 case DDI_CTLOPS_UNINITCHILD: 672 impl_ddi_sunbus_removechild(arg); 673 return (DDI_SUCCESS); 674 675 case DDI_CTLOPS_REPORTDEV: 676 return (rootnex_ctl_reportdev(rdip)); 677 678 case DDI_CTLOPS_IOMIN: 679 /* 680 * Nothing to do here but reflect back.. 681 */ 682 return (DDI_SUCCESS); 683 684 case DDI_CTLOPS_REGSIZE: 685 case DDI_CTLOPS_NREGS: 686 break; 687 688 case DDI_CTLOPS_SIDDEV: 689 if (ndi_dev_is_prom_node(rdip)) 690 return (DDI_SUCCESS); 691 if (ndi_dev_is_persistent_node(rdip)) 692 return (DDI_SUCCESS); 693 return (DDI_FAILURE); 694 695 case DDI_CTLOPS_POWER: 696 return ((*pm_platform_power)((power_req_t *)arg)); 697 698 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 699 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 700 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 701 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 702 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 703 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 704 if (!rootnex_state->r_reserved_msg_printed) { 705 rootnex_state->r_reserved_msg_printed = B_TRUE; 706 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 707 "1 or more reserved/obsolete operations."); 708 } 709 return (DDI_FAILURE); 710 711 default: 712 return (DDI_FAILURE); 713 } 714 /* 715 * The rest are for "hardware" properties 716 */ 717 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 718 return (DDI_FAILURE); 719 720 if (ctlop == DDI_CTLOPS_NREGS) { 721 ptr = (int *)result; 722 *ptr = pdp->par_nreg; 723 } else { 724 off_t *size = (off_t *)result; 725 726 ptr = (int *)arg; 727 n = *ptr; 728 if (n >= pdp->par_nreg) { 729 return (DDI_FAILURE); 730 } 731 *size = (off_t)pdp->par_reg[n].regspec_size; 732 } 733 return (DDI_SUCCESS); 734 } 735 736 737 /* 738 * rootnex_ctl_reportdev() 739 * 740 */ 741 static int 742 rootnex_ctl_reportdev(dev_info_t *dev) 743 { 744 int i, n, len, f_len = 0; 745 char *buf; 746 747 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 748 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 749 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 750 len = strlen(buf); 751 752 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 753 754 struct regspec *rp = sparc_pd_getreg(dev, i); 755 756 if (i == 0) 757 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 758 ": "); 759 else 760 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 761 " and "); 762 len = strlen(buf); 763 764 switch (rp->regspec_bustype) { 765 766 case BTEISA: 767 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 768 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 769 break; 770 771 case BTISA: 772 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 773 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 774 break; 775 776 default: 777 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 778 "space %x offset %x", 779 rp->regspec_bustype, rp->regspec_addr); 780 break; 781 } 782 len = strlen(buf); 783 } 784 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 785 int pri; 786 787 if (i != 0) { 788 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 789 ","); 790 len = strlen(buf); 791 } 792 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 793 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 794 " sparc ipl %d", pri); 795 len = strlen(buf); 796 } 797 #ifdef DEBUG 798 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 799 cmn_err(CE_NOTE, "next message is truncated: " 800 "printed length 1024, real length %d", f_len); 801 } 802 #endif /* DEBUG */ 803 cmn_err(CE_CONT, "?%s\n", buf); 804 kmem_free(buf, REPORTDEV_BUFSIZE); 805 return (DDI_SUCCESS); 806 } 807 808 809 /* 810 * ****************** 811 * map related code 812 * ****************** 813 */ 814 815 /* 816 * rootnex_map() 817 * 818 */ 819 static int 820 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 821 off_t len, caddr_t *vaddrp) 822 { 823 struct regspec *orp = NULL; 824 struct regspec64 rp = { 0 }; 825 ddi_map_req_t mr = *mp; /* Get private copy of request */ 826 827 mp = &mr; 828 829 switch (mp->map_op) { 830 case DDI_MO_MAP_LOCKED: 831 case DDI_MO_UNMAP: 832 case DDI_MO_MAP_HANDLE: 833 break; 834 default: 835 #ifdef DDI_MAP_DEBUG 836 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 837 mp->map_op); 838 #endif /* DDI_MAP_DEBUG */ 839 return (DDI_ME_UNIMPLEMENTED); 840 } 841 842 if (mp->map_flags & DDI_MF_USER_MAPPING) { 843 #ifdef DDI_MAP_DEBUG 844 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 845 #endif /* DDI_MAP_DEBUG */ 846 return (DDI_ME_UNIMPLEMENTED); 847 } 848 849 /* 850 * First, we need to get the original regspec out before we convert it 851 * to the extended format. If we have a register number, then we need to 852 * convert that to a regspec. 853 */ 854 if (mp->map_type == DDI_MT_RNUMBER) { 855 856 int rnumber = mp->map_obj.rnumber; 857 #ifdef DDI_MAP_DEBUG 858 static char *out_of_range = 859 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 860 #endif /* DDI_MAP_DEBUG */ 861 862 orp = i_ddi_rnumber_to_regspec(rdip, rnumber); 863 if (orp == NULL) { 864 #ifdef DDI_MAP_DEBUG 865 cmn_err(CE_WARN, out_of_range, rnumber, 866 ddi_get_name(rdip)); 867 #endif /* DDI_MAP_DEBUG */ 868 return (DDI_ME_RNUMBER_RANGE); 869 } 870 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) { 871 orp = mp->map_obj.rp; 872 } 873 874 /* 875 * Ensure that we are always using a 64-bit extended regspec regardless 876 * of what was passed into us. If the child driver is using a 64-bit 877 * regspec, then we need to make sure that we copy this to the local 878 * regspec64, rp. 879 */ 880 if (orp != NULL) { 881 rp.regspec_bustype = orp->regspec_bustype; 882 rp.regspec_addr = orp->regspec_addr; 883 rp.regspec_size = orp->regspec_size; 884 } else { 885 struct regspec64 *rp64; 886 rp64 = (struct regspec64 *)mp->map_obj.rp; 887 rp = *rp64; 888 } 889 890 mp->map_type = DDI_MT_REGSPEC; 891 mp->map_flags |= DDI_MF_EXT_REGSPEC; 892 mp->map_obj.rp = (struct regspec *)&rp; 893 894 /* 895 * Adjust offset and length correspnding to called values... 896 * XXX: A non-zero length means override the one in the regspec 897 * XXX: (regardless of what's in the parent's range?) 898 */ 899 900 #ifdef DDI_MAP_DEBUG 901 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 902 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 903 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset, 904 len, mp->map_handlep); 905 #endif /* DDI_MAP_DEBUG */ 906 907 /* 908 * I/O or memory mapping: 909 * 910 * <bustype=0, addr=x, len=x>: memory 911 * <bustype=1, addr=x, len=x>: i/o 912 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 913 */ 914 915 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) { 916 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 917 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">", 918 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 919 rp.regspec_addr, rp.regspec_size); 920 return (DDI_ME_INVAL); 921 } 922 923 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) { 924 /* 925 * compatibility i/o mapping 926 */ 927 rp.regspec_bustype += offset; 928 } else { 929 /* 930 * Normal memory or i/o mapping 931 */ 932 rp.regspec_addr += offset; 933 } 934 935 if (len != 0) 936 rp.regspec_size = len; 937 938 #ifdef DDI_MAP_DEBUG 939 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64 940 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n", 941 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 942 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep); 943 #endif /* DDI_MAP_DEBUG */ 944 945 946 /* 947 * The x86 root nexus does not have any notion of valid ranges of 948 * addresses. Its children have valid ranges, but because there are none 949 * for the nexus, we don't need to call i_ddi_apply_range(). Verify 950 * that is the case. 951 */ 952 ASSERT0(sparc_pd_getnrng(dip)); 953 954 switch (mp->map_op) { 955 case DDI_MO_MAP_LOCKED: 956 957 /* 958 * Set up the locked down kernel mapping to the regspec... 959 */ 960 961 return (rootnex_map_regspec(mp, vaddrp)); 962 963 case DDI_MO_UNMAP: 964 965 /* 966 * Release mapping... 967 */ 968 969 return (rootnex_unmap_regspec(mp, vaddrp)); 970 971 case DDI_MO_MAP_HANDLE: 972 973 return (rootnex_map_handle(mp)); 974 975 default: 976 return (DDI_ME_UNIMPLEMENTED); 977 } 978 } 979 980 981 /* 982 * rootnex_map_fault() 983 * 984 * fault in mappings for requestors 985 */ 986 /*ARGSUSED*/ 987 static int 988 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 989 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 990 uint_t lock) 991 { 992 993 #ifdef DDI_MAP_DEBUG 994 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 995 ddi_map_debug(" Seg <%s>\n", 996 seg->s_ops == &segdev_ops ? "segdev" : 997 seg == &kvseg ? "segkmem" : "NONE!"); 998 #endif /* DDI_MAP_DEBUG */ 999 1000 /* 1001 * This is all terribly broken, but it is a start 1002 * 1003 * XXX Note that this test means that segdev_ops 1004 * must be exported from seg_dev.c. 1005 * XXX What about devices with their own segment drivers? 1006 */ 1007 if (seg->s_ops == &segdev_ops) { 1008 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1009 1010 if (hat == NULL) { 1011 /* 1012 * This is one plausible interpretation of 1013 * a null hat i.e. use the first hat on the 1014 * address space hat list which by convention is 1015 * the hat of the system MMU. At alternative 1016 * would be to panic .. this might well be better .. 1017 */ 1018 ASSERT(AS_READ_HELD(seg->s_as)); 1019 hat = seg->s_as->a_hat; 1020 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1021 } 1022 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1023 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1024 } else if (seg == &kvseg && dp == NULL) { 1025 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1026 HAT_LOAD_LOCK); 1027 } else 1028 return (DDI_FAILURE); 1029 return (DDI_SUCCESS); 1030 } 1031 1032 1033 static int 1034 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1035 { 1036 rootnex_addr_t rbase; 1037 void *cvaddr; 1038 uint64_t npages, pgoffset; 1039 struct regspec64 *rp; 1040 ddi_acc_hdl_t *hp; 1041 ddi_acc_impl_t *ap; 1042 uint_t hat_acc_flags; 1043 paddr_t pbase; 1044 1045 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1046 rp = (struct regspec64 *)mp->map_obj.rp; 1047 hp = mp->map_handlep; 1048 1049 #ifdef DDI_MAP_DEBUG 1050 ddi_map_debug( 1051 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1052 rp->regspec_bustype, rp->regspec_addr, 1053 rp->regspec_size, mp->map_handlep); 1054 #endif /* DDI_MAP_DEBUG */ 1055 1056 /* 1057 * I/O or memory mapping 1058 * 1059 * <bustype=0, addr=x, len=x>: memory 1060 * <bustype=1, addr=x, len=x>: i/o 1061 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1062 */ 1063 1064 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1065 cmn_err(CE_WARN, "rootnex: invalid register spec" 1066 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">", 1067 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size); 1068 return (DDI_FAILURE); 1069 } 1070 1071 if (rp->regspec_bustype != 0) { 1072 /* 1073 * I/O space - needs a handle. 1074 */ 1075 if (hp == NULL) { 1076 return (DDI_FAILURE); 1077 } 1078 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1079 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1080 impl_acc_hdl_init(hp); 1081 1082 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1083 #ifdef DDI_MAP_DEBUG 1084 ddi_map_debug("rootnex_map_regspec: mmap() " 1085 "to I/O space is not supported.\n"); 1086 #endif /* DDI_MAP_DEBUG */ 1087 return (DDI_ME_INVAL); 1088 } else { 1089 /* 1090 * 1275-compliant vs. compatibility i/o mapping 1091 */ 1092 *vaddrp = 1093 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1094 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1095 ((caddr_t)(uintptr_t)rp->regspec_addr); 1096 #ifdef __xpv 1097 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1098 hp->ah_pfn = xen_assign_pfn( 1099 mmu_btop((ulong_t)rp->regspec_addr & 1100 MMU_PAGEMASK)); 1101 } else { 1102 hp->ah_pfn = mmu_btop( 1103 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1104 } 1105 #else 1106 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1107 MMU_PAGEMASK); 1108 #endif 1109 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1110 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1111 } 1112 1113 #ifdef DDI_MAP_DEBUG 1114 ddi_map_debug( 1115 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1116 rp->regspec_size, *vaddrp); 1117 #endif /* DDI_MAP_DEBUG */ 1118 return (DDI_SUCCESS); 1119 } 1120 1121 /* 1122 * Memory space 1123 */ 1124 1125 if (hp != NULL) { 1126 /* 1127 * hat layer ignores 1128 * hp->ah_acc.devacc_attr_endian_flags. 1129 */ 1130 switch (hp->ah_acc.devacc_attr_dataorder) { 1131 case DDI_STRICTORDER_ACC: 1132 hat_acc_flags = HAT_STRICTORDER; 1133 break; 1134 case DDI_UNORDERED_OK_ACC: 1135 hat_acc_flags = HAT_UNORDERED_OK; 1136 break; 1137 case DDI_MERGING_OK_ACC: 1138 hat_acc_flags = HAT_MERGING_OK; 1139 break; 1140 case DDI_LOADCACHING_OK_ACC: 1141 hat_acc_flags = HAT_LOADCACHING_OK; 1142 break; 1143 case DDI_STORECACHING_OK_ACC: 1144 hat_acc_flags = HAT_STORECACHING_OK; 1145 break; 1146 default: 1147 return (DDI_ME_INVAL); 1148 } 1149 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1150 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1151 impl_acc_hdl_init(hp); 1152 hp->ah_hat_flags = hat_acc_flags; 1153 } else { 1154 hat_acc_flags = HAT_STRICTORDER; 1155 } 1156 1157 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1158 #ifdef __xpv 1159 /* 1160 * If we're dom0, we're using a real device so we need to translate 1161 * the MA to a PA. 1162 */ 1163 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1164 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1165 } else { 1166 pbase = rbase; 1167 } 1168 #else 1169 pbase = rbase; 1170 #endif 1171 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1172 1173 if (rp->regspec_size == 0) { 1174 #ifdef DDI_MAP_DEBUG 1175 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1176 #endif /* DDI_MAP_DEBUG */ 1177 return (DDI_ME_INVAL); 1178 } 1179 1180 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1181 /* extra cast to make gcc happy */ 1182 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1183 } else { 1184 npages = mmu_btopr(rp->regspec_size + pgoffset); 1185 1186 #ifdef DDI_MAP_DEBUG 1187 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1188 "physical %llx", npages, pbase); 1189 #endif /* DDI_MAP_DEBUG */ 1190 1191 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1192 if (cvaddr == NULL) 1193 return (DDI_ME_NORESOURCES); 1194 1195 /* 1196 * Now map in the pages we've allocated... 1197 */ 1198 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1199 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1200 HAT_LOAD_LOCK); 1201 *vaddrp = (caddr_t)cvaddr + pgoffset; 1202 1203 /* save away pfn and npages for FMA */ 1204 hp = mp->map_handlep; 1205 if (hp) { 1206 hp->ah_pfn = mmu_btop(pbase); 1207 hp->ah_pnum = npages; 1208 } 1209 } 1210 1211 #ifdef DDI_MAP_DEBUG 1212 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1213 #endif /* DDI_MAP_DEBUG */ 1214 return (DDI_SUCCESS); 1215 } 1216 1217 1218 static int 1219 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1220 { 1221 caddr_t addr = (caddr_t)*vaddrp; 1222 uint64_t npages, pgoffset; 1223 struct regspec64 *rp; 1224 1225 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1226 return (0); 1227 1228 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1229 rp = (struct regspec64 *)mp->map_obj.rp; 1230 1231 if (rp->regspec_size == 0) { 1232 #ifdef DDI_MAP_DEBUG 1233 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1234 #endif /* DDI_MAP_DEBUG */ 1235 return (DDI_ME_INVAL); 1236 } 1237 1238 /* 1239 * I/O or memory mapping: 1240 * 1241 * <bustype=0, addr=x, len=x>: memory 1242 * <bustype=1, addr=x, len=x>: i/o 1243 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1244 */ 1245 if (rp->regspec_bustype != 0) { 1246 /* 1247 * This is I/O space, which requires no particular 1248 * processing on unmap since it isn't mapped in the 1249 * first place. 1250 */ 1251 return (DDI_SUCCESS); 1252 } 1253 1254 /* 1255 * Memory space 1256 */ 1257 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1258 npages = mmu_btopr(rp->regspec_size + pgoffset); 1259 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1260 device_arena_free(addr - pgoffset, ptob(npages)); 1261 1262 /* 1263 * Destroy the pointer - the mapping has logically gone 1264 */ 1265 *vaddrp = NULL; 1266 1267 return (DDI_SUCCESS); 1268 } 1269 1270 static int 1271 rootnex_map_handle(ddi_map_req_t *mp) 1272 { 1273 rootnex_addr_t rbase; 1274 ddi_acc_hdl_t *hp; 1275 uint64_t pgoffset; 1276 struct regspec64 *rp; 1277 paddr_t pbase; 1278 1279 rp = (struct regspec64 *)mp->map_obj.rp; 1280 1281 #ifdef DDI_MAP_DEBUG 1282 ddi_map_debug( 1283 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1284 rp->regspec_bustype, rp->regspec_addr, 1285 rp->regspec_size, mp->map_handlep); 1286 #endif /* DDI_MAP_DEBUG */ 1287 1288 /* 1289 * I/O or memory mapping: 1290 * 1291 * <bustype=0, addr=x, len=x>: memory 1292 * <bustype=1, addr=x, len=x>: i/o 1293 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1294 */ 1295 if (rp->regspec_bustype != 0) { 1296 /* 1297 * This refers to I/O space, and we don't support "mapping" 1298 * I/O space to a user. 1299 */ 1300 return (DDI_FAILURE); 1301 } 1302 1303 /* 1304 * Set up the hat_flags for the mapping. 1305 */ 1306 hp = mp->map_handlep; 1307 1308 switch (hp->ah_acc.devacc_attr_endian_flags) { 1309 case DDI_NEVERSWAP_ACC: 1310 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1311 break; 1312 case DDI_STRUCTURE_LE_ACC: 1313 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1314 break; 1315 case DDI_STRUCTURE_BE_ACC: 1316 return (DDI_FAILURE); 1317 default: 1318 return (DDI_REGS_ACC_CONFLICT); 1319 } 1320 1321 switch (hp->ah_acc.devacc_attr_dataorder) { 1322 case DDI_STRICTORDER_ACC: 1323 break; 1324 case DDI_UNORDERED_OK_ACC: 1325 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1326 break; 1327 case DDI_MERGING_OK_ACC: 1328 hp->ah_hat_flags |= HAT_MERGING_OK; 1329 break; 1330 case DDI_LOADCACHING_OK_ACC: 1331 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1332 break; 1333 case DDI_STORECACHING_OK_ACC: 1334 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1335 break; 1336 default: 1337 return (DDI_FAILURE); 1338 } 1339 1340 rbase = (rootnex_addr_t)rp->regspec_addr & 1341 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1342 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1343 1344 if (rp->regspec_size == 0) 1345 return (DDI_ME_INVAL); 1346 1347 #ifdef __xpv 1348 /* 1349 * If we're dom0, we're using a real device so we need to translate 1350 * the MA to a PA. 1351 */ 1352 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1353 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1354 (rbase & MMU_PAGEOFFSET); 1355 } else { 1356 pbase = rbase; 1357 } 1358 #else 1359 pbase = rbase; 1360 #endif 1361 1362 hp->ah_pfn = mmu_btop(pbase); 1363 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1364 1365 return (DDI_SUCCESS); 1366 } 1367 1368 1369 1370 /* 1371 * ************************ 1372 * interrupt related code 1373 * ************************ 1374 */ 1375 1376 /* 1377 * rootnex_intr_ops() 1378 * bus_intr_op() function for interrupt support 1379 */ 1380 /* ARGSUSED */ 1381 static int 1382 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1383 ddi_intr_handle_impl_t *hdlp, void *result) 1384 { 1385 struct intrspec *ispec; 1386 1387 DDI_INTR_NEXDBG((CE_CONT, 1388 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1389 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1390 1391 /* Process the interrupt operation */ 1392 switch (intr_op) { 1393 case DDI_INTROP_GETCAP: 1394 /* First check with pcplusmp */ 1395 if (psm_intr_ops == NULL) 1396 return (DDI_FAILURE); 1397 1398 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1399 *(int *)result = 0; 1400 return (DDI_FAILURE); 1401 } 1402 break; 1403 case DDI_INTROP_SETCAP: 1404 if (psm_intr_ops == NULL) 1405 return (DDI_FAILURE); 1406 1407 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1408 return (DDI_FAILURE); 1409 break; 1410 case DDI_INTROP_ALLOC: 1411 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1412 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1413 case DDI_INTROP_FREE: 1414 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1415 return (rootnex_free_intr_fixed(rdip, hdlp)); 1416 case DDI_INTROP_GETPRI: 1417 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1418 return (DDI_FAILURE); 1419 *(int *)result = ispec->intrspec_pri; 1420 break; 1421 case DDI_INTROP_SETPRI: 1422 /* Validate the interrupt priority passed to us */ 1423 if (*(int *)result > LOCK_LEVEL) 1424 return (DDI_FAILURE); 1425 1426 /* Ensure that PSM is all initialized and ispec is ok */ 1427 if ((psm_intr_ops == NULL) || 1428 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1429 return (DDI_FAILURE); 1430 1431 /* Change the priority */ 1432 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1433 PSM_FAILURE) 1434 return (DDI_FAILURE); 1435 1436 /* update the ispec with the new priority */ 1437 ispec->intrspec_pri = *(int *)result; 1438 break; 1439 case DDI_INTROP_ADDISR: 1440 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1441 return (DDI_FAILURE); 1442 ispec->intrspec_func = hdlp->ih_cb_func; 1443 break; 1444 case DDI_INTROP_REMISR: 1445 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1446 return (DDI_FAILURE); 1447 ispec->intrspec_func = (uint_t (*)()) 0; 1448 break; 1449 case DDI_INTROP_ENABLE: 1450 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1451 return (DDI_FAILURE); 1452 1453 /* Call psmi to translate irq with the dip */ 1454 if (psm_intr_ops == NULL) 1455 return (DDI_FAILURE); 1456 1457 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1458 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1459 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1460 return (DDI_FAILURE); 1461 1462 /* Add the interrupt handler */ 1463 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1464 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1465 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1466 return (DDI_FAILURE); 1467 break; 1468 case DDI_INTROP_DISABLE: 1469 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1470 return (DDI_FAILURE); 1471 1472 /* Call psm_ops() to translate irq with the dip */ 1473 if (psm_intr_ops == NULL) 1474 return (DDI_FAILURE); 1475 1476 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1477 (void) (*psm_intr_ops)(rdip, hdlp, 1478 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1479 1480 /* Remove the interrupt handler */ 1481 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1482 hdlp->ih_cb_func, hdlp->ih_vector); 1483 break; 1484 case DDI_INTROP_SETMASK: 1485 if (psm_intr_ops == NULL) 1486 return (DDI_FAILURE); 1487 1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1489 return (DDI_FAILURE); 1490 break; 1491 case DDI_INTROP_CLRMASK: 1492 if (psm_intr_ops == NULL) 1493 return (DDI_FAILURE); 1494 1495 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1496 return (DDI_FAILURE); 1497 break; 1498 case DDI_INTROP_GETPENDING: 1499 if (psm_intr_ops == NULL) 1500 return (DDI_FAILURE); 1501 1502 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1503 result)) { 1504 *(int *)result = 0; 1505 return (DDI_FAILURE); 1506 } 1507 break; 1508 case DDI_INTROP_NAVAIL: 1509 case DDI_INTROP_NINTRS: 1510 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1511 if (*(int *)result == 0) { 1512 /* 1513 * Special case for 'pcic' driver' only. This driver 1514 * driver is a child of 'isa' and 'rootnex' drivers. 1515 * 1516 * See detailed comments on this in the function 1517 * rootnex_get_ispec(). 1518 * 1519 * Children of 'pcic' send 'NINITR' request all the 1520 * way to rootnex driver. But, the 'pdp->par_nintr' 1521 * field may not initialized. So, we fake it here 1522 * to return 1 (a la what PCMCIA nexus does). 1523 */ 1524 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1525 *(int *)result = 1; 1526 else 1527 return (DDI_FAILURE); 1528 } 1529 break; 1530 case DDI_INTROP_SUPPORTED_TYPES: 1531 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1532 break; 1533 default: 1534 return (DDI_FAILURE); 1535 } 1536 1537 return (DDI_SUCCESS); 1538 } 1539 1540 1541 /* 1542 * rootnex_get_ispec() 1543 * convert an interrupt number to an interrupt specification. 1544 * The interrupt number determines which interrupt spec will be 1545 * returned if more than one exists. 1546 * 1547 * Look into the parent private data area of the 'rdip' to find out 1548 * the interrupt specification. First check to make sure there is 1549 * one that matchs "inumber" and then return a pointer to it. 1550 * 1551 * Return NULL if one could not be found. 1552 * 1553 * NOTE: This is needed for rootnex_intr_ops() 1554 */ 1555 static struct intrspec * 1556 rootnex_get_ispec(dev_info_t *rdip, int inum) 1557 { 1558 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1559 1560 /* 1561 * Special case handling for drivers that provide their own 1562 * intrspec structures instead of relying on the DDI framework. 1563 * 1564 * A broken hardware driver in ON could potentially provide its 1565 * own intrspec structure, instead of relying on the hardware. 1566 * If these drivers are children of 'rootnex' then we need to 1567 * continue to provide backward compatibility to them here. 1568 * 1569 * Following check is a special case for 'pcic' driver which 1570 * was found to have broken hardwre andby provides its own intrspec. 1571 * 1572 * Verbatim comments from this driver are shown here: 1573 * "Don't use the ddi_add_intr since we don't have a 1574 * default intrspec in all cases." 1575 * 1576 * Since an 'ispec' may not be always created for it, 1577 * check for that and create one if so. 1578 * 1579 * NOTE: Currently 'pcic' is the only driver found to do this. 1580 */ 1581 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1582 pdp->par_nintr = 1; 1583 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1584 pdp->par_nintr, KM_SLEEP); 1585 } 1586 1587 /* Validate the interrupt number */ 1588 if (inum >= pdp->par_nintr) 1589 return (NULL); 1590 1591 /* Get the interrupt structure pointer and return that */ 1592 return ((struct intrspec *)&pdp->par_intr[inum]); 1593 } 1594 1595 /* 1596 * Allocate interrupt vector for FIXED (legacy) type. 1597 */ 1598 static int 1599 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1600 void *result) 1601 { 1602 struct intrspec *ispec; 1603 ddi_intr_handle_impl_t info_hdl; 1604 int ret; 1605 int free_phdl = 0; 1606 apic_get_type_t type_info; 1607 1608 if (psm_intr_ops == NULL) 1609 return (DDI_FAILURE); 1610 1611 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1612 return (DDI_FAILURE); 1613 1614 /* 1615 * If the PSM module is "APIX" then pass the request for it 1616 * to allocate the vector now. 1617 */ 1618 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1619 info_hdl.ih_private = &type_info; 1620 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1621 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1622 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1623 free_phdl = 1; 1624 i_ddi_alloc_intr_phdl(hdlp); 1625 } 1626 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1627 ret = (*psm_intr_ops)(rdip, hdlp, 1628 PSM_INTR_OP_ALLOC_VECTORS, result); 1629 if (free_phdl) { /* free up the phdl structure */ 1630 free_phdl = 0; 1631 i_ddi_free_intr_phdl(hdlp); 1632 hdlp->ih_private = NULL; 1633 } 1634 } else { 1635 /* 1636 * No APIX module; fall back to the old scheme where the 1637 * interrupt vector is allocated during ddi_enable_intr() call. 1638 */ 1639 hdlp->ih_pri = ispec->intrspec_pri; 1640 *(int *)result = hdlp->ih_scratch1; 1641 ret = DDI_SUCCESS; 1642 } 1643 1644 return (ret); 1645 } 1646 1647 /* 1648 * Free up interrupt vector for FIXED (legacy) type. 1649 */ 1650 static int 1651 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1652 { 1653 struct intrspec *ispec; 1654 struct ddi_parent_private_data *pdp; 1655 ddi_intr_handle_impl_t info_hdl; 1656 int ret; 1657 apic_get_type_t type_info; 1658 1659 if (psm_intr_ops == NULL) 1660 return (DDI_FAILURE); 1661 1662 /* 1663 * If the PSM module is "APIX" then pass the request for it 1664 * to free up the vector now. 1665 */ 1666 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1667 info_hdl.ih_private = &type_info; 1668 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1669 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1670 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1671 return (DDI_FAILURE); 1672 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1673 ret = (*psm_intr_ops)(rdip, hdlp, 1674 PSM_INTR_OP_FREE_VECTORS, NULL); 1675 } else { 1676 /* 1677 * No APIX module; fall back to the old scheme where 1678 * the interrupt vector was already freed during 1679 * ddi_disable_intr() call. 1680 */ 1681 ret = DDI_SUCCESS; 1682 } 1683 1684 pdp = ddi_get_parent_data(rdip); 1685 1686 /* 1687 * Special case for 'pcic' driver' only. 1688 * If an intrspec was created for it, clean it up here 1689 * See detailed comments on this in the function 1690 * rootnex_get_ispec(). 1691 */ 1692 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1693 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1694 pdp->par_nintr); 1695 /* 1696 * Set it to zero; so that 1697 * DDI framework doesn't free it again 1698 */ 1699 pdp->par_intr = NULL; 1700 pdp->par_nintr = 0; 1701 } 1702 1703 return (ret); 1704 } 1705 1706 1707 /* 1708 * ****************** 1709 * dma related code 1710 * ****************** 1711 */ 1712 1713 /*ARGSUSED*/ 1714 static int 1715 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1716 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1717 ddi_dma_handle_t *handlep) 1718 { 1719 uint64_t maxsegmentsize_ll; 1720 uint_t maxsegmentsize; 1721 ddi_dma_impl_t *hp; 1722 rootnex_dma_t *dma; 1723 uint64_t count_max; 1724 uint64_t seg; 1725 int kmflag; 1726 int e; 1727 1728 1729 /* convert our sleep flags */ 1730 if (waitfp == DDI_DMA_SLEEP) { 1731 kmflag = KM_SLEEP; 1732 } else { 1733 kmflag = KM_NOSLEEP; 1734 } 1735 1736 /* 1737 * We try to do only one memory allocation here. We'll do a little 1738 * pointer manipulation later. If the bind ends up taking more than 1739 * our prealloc's space, we'll have to allocate more memory in the 1740 * bind operation. Not great, but much better than before and the 1741 * best we can do with the current bind interfaces. 1742 */ 1743 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1744 if (hp == NULL) 1745 return (DDI_DMA_NORESOURCES); 1746 1747 /* Do our pointer manipulation now, align the structures */ 1748 hp->dmai_private = (void *)(((uintptr_t)hp + 1749 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1750 dma = (rootnex_dma_t *)hp->dmai_private; 1751 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1752 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1753 1754 /* setup the handle */ 1755 rootnex_clean_dmahdl(hp); 1756 hp->dmai_error.err_fep = NULL; 1757 hp->dmai_error.err_cf = NULL; 1758 dma->dp_dip = rdip; 1759 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1760 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1761 1762 /* 1763 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1764 * is being used. Set the upper limit to the seg value. 1765 * There will be enough DVMA space to always get addresses 1766 * that will match the constraints. 1767 */ 1768 if (IOMMU_USED(rdip) && 1769 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1770 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1771 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1772 } else 1773 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1774 1775 hp->dmai_minxfer = attr->dma_attr_minxfer; 1776 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1777 hp->dmai_rdip = rdip; 1778 hp->dmai_attr = *attr; 1779 1780 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1781 dma->dp_sglinfo.si_cancross = B_FALSE; 1782 else 1783 dma->dp_sglinfo.si_cancross = B_TRUE; 1784 1785 /* we don't need to worry about the SPL since we do a tryenter */ 1786 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1787 1788 /* 1789 * Figure out our maximum segment size. If the segment size is greater 1790 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1791 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1792 * dma_attr_count_max are size-1 type values. 1793 * 1794 * Maximum segment size is the largest physically contiguous chunk of 1795 * memory that we can return from a bind (i.e. the maximum size of a 1796 * single cookie). 1797 */ 1798 1799 /* handle the rollover cases */ 1800 seg = attr->dma_attr_seg + 1; 1801 if (seg < attr->dma_attr_seg) { 1802 seg = attr->dma_attr_seg; 1803 } 1804 count_max = attr->dma_attr_count_max + 1; 1805 if (count_max < attr->dma_attr_count_max) { 1806 count_max = attr->dma_attr_count_max; 1807 } 1808 1809 /* 1810 * granularity may or may not be a power of two. If it isn't, we can't 1811 * use a simple mask. 1812 */ 1813 if (!ISP2(attr->dma_attr_granular)) { 1814 dma->dp_granularity_power_2 = B_FALSE; 1815 } else { 1816 dma->dp_granularity_power_2 = B_TRUE; 1817 } 1818 1819 /* 1820 * maxxfer should be a whole multiple of granularity. If we're going to 1821 * break up a window because we're greater than maxxfer, we might as 1822 * well make sure it's maxxfer is a whole multiple so we don't have to 1823 * worry about triming the window later on for this case. 1824 */ 1825 if (attr->dma_attr_granular > 1) { 1826 if (dma->dp_granularity_power_2) { 1827 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1828 (attr->dma_attr_maxxfer & 1829 (attr->dma_attr_granular - 1)); 1830 } else { 1831 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1832 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1833 } 1834 } else { 1835 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1836 } 1837 1838 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1839 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1840 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1841 maxsegmentsize = 0xFFFFFFFF; 1842 } else { 1843 maxsegmentsize = maxsegmentsize_ll; 1844 } 1845 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1846 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1847 1848 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1849 if (rootnex_alloc_check_parms) { 1850 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1851 if (e != DDI_SUCCESS) { 1852 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1853 (void) rootnex_dma_freehdl(dip, rdip, 1854 (ddi_dma_handle_t)hp); 1855 return (e); 1856 } 1857 } 1858 1859 *handlep = (ddi_dma_handle_t)hp; 1860 1861 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1862 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1863 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1864 1865 return (DDI_SUCCESS); 1866 } 1867 1868 1869 /* 1870 * rootnex_dma_allochdl() 1871 * called from ddi_dma_alloc_handle(). 1872 */ 1873 static int 1874 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1875 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1876 { 1877 int retval = DDI_SUCCESS; 1878 #if defined(__amd64) && !defined(__xpv) 1879 1880 if (IOMMU_UNITIALIZED(rdip)) { 1881 retval = iommulib_nex_open(dip, rdip); 1882 1883 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1884 return (retval); 1885 } 1886 1887 if (IOMMU_UNUSED(rdip)) { 1888 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1889 handlep); 1890 } else { 1891 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1892 waitfp, arg, handlep); 1893 } 1894 #else 1895 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1896 handlep); 1897 #endif 1898 switch (retval) { 1899 case DDI_DMA_NORESOURCES: 1900 if (waitfp != DDI_DMA_DONTWAIT) { 1901 ddi_set_callback(waitfp, arg, 1902 &rootnex_state->r_dvma_call_list_id); 1903 } 1904 break; 1905 case DDI_SUCCESS: 1906 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1907 break; 1908 default: 1909 break; 1910 } 1911 return (retval); 1912 } 1913 1914 /*ARGSUSED*/ 1915 static int 1916 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1917 ddi_dma_handle_t handle) 1918 { 1919 ddi_dma_impl_t *hp; 1920 rootnex_dma_t *dma; 1921 1922 1923 hp = (ddi_dma_impl_t *)handle; 1924 dma = (rootnex_dma_t *)hp->dmai_private; 1925 1926 /* unbind should have been called first */ 1927 ASSERT(!dma->dp_inuse); 1928 1929 mutex_destroy(&dma->dp_mutex); 1930 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1931 1932 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1933 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1934 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1935 1936 return (DDI_SUCCESS); 1937 } 1938 1939 /* 1940 * rootnex_dma_freehdl() 1941 * called from ddi_dma_free_handle(). 1942 */ 1943 static int 1944 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1945 { 1946 int ret; 1947 1948 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1949 #if defined(__amd64) && !defined(__xpv) 1950 if (IOMMU_USED(rdip)) 1951 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1952 else 1953 #endif 1954 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1955 1956 if (rootnex_state->r_dvma_call_list_id) 1957 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1958 1959 return (ret); 1960 } 1961 1962 /*ARGSUSED*/ 1963 static int 1964 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1965 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1966 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1967 { 1968 rootnex_sglinfo_t *sinfo; 1969 ddi_dma_obj_t *dmao; 1970 #if defined(__amd64) && !defined(__xpv) 1971 struct dvmaseg *dvs; 1972 ddi_dma_cookie_t *cookie; 1973 #endif 1974 ddi_dma_attr_t *attr; 1975 ddi_dma_impl_t *hp; 1976 rootnex_dma_t *dma; 1977 int kmflag; 1978 int e; 1979 uint_t ncookies; 1980 1981 hp = (ddi_dma_impl_t *)handle; 1982 dma = (rootnex_dma_t *)hp->dmai_private; 1983 dmao = &dma->dp_dma; 1984 sinfo = &dma->dp_sglinfo; 1985 attr = &hp->dmai_attr; 1986 1987 /* convert the sleep flags */ 1988 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1989 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1990 } else { 1991 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1992 } 1993 1994 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1995 1996 /* 1997 * This is useful for debugging a driver. Not as useful in a production 1998 * system. The only time this will fail is if you have a driver bug. 1999 */ 2000 if (rootnex_bind_check_inuse) { 2001 /* 2002 * No one else should ever have this lock unless someone else 2003 * is trying to use this handle. So contention on the lock 2004 * is the same as inuse being set. 2005 */ 2006 e = mutex_tryenter(&dma->dp_mutex); 2007 if (e == 0) { 2008 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2009 return (DDI_DMA_INUSE); 2010 } 2011 if (dma->dp_inuse) { 2012 mutex_exit(&dma->dp_mutex); 2013 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2014 return (DDI_DMA_INUSE); 2015 } 2016 dma->dp_inuse = B_TRUE; 2017 mutex_exit(&dma->dp_mutex); 2018 } 2019 2020 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2021 if (rootnex_bind_check_parms) { 2022 e = rootnex_valid_bind_parms(dmareq, attr); 2023 if (e != DDI_SUCCESS) { 2024 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2025 rootnex_clean_dmahdl(hp); 2026 return (e); 2027 } 2028 } 2029 2030 /* save away the original bind info */ 2031 dma->dp_dma = dmareq->dmar_object; 2032 2033 #if defined(__amd64) && !defined(__xpv) 2034 if (IOMMU_USED(rdip)) { 2035 dmao = &dma->dp_dvma; 2036 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2037 switch (e) { 2038 case DDI_SUCCESS: 2039 if (sinfo->si_cancross || 2040 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2041 dmao->dmao_size > sinfo->si_max_cookie_size) { 2042 dma->dp_dvma_used = B_TRUE; 2043 break; 2044 } 2045 sinfo->si_sgl_size = 1; 2046 hp->dmai_rflags |= DMP_NOSYNC; 2047 2048 dma->dp_dvma_used = B_TRUE; 2049 dma->dp_need_to_free_cookie = B_FALSE; 2050 2051 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2052 cookie = hp->dmai_cookie = dma->dp_cookies = 2053 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2054 cookie->dmac_laddress = dvs->dvs_start + 2055 dmao->dmao_obj.dvma_obj.dv_off; 2056 cookie->dmac_size = dvs->dvs_len; 2057 cookie->dmac_type = 0; 2058 2059 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2060 rdip); 2061 goto fast; 2062 case DDI_ENOTSUP: 2063 break; 2064 default: 2065 rootnex_clean_dmahdl(hp); 2066 return (e); 2067 } 2068 } 2069 #endif 2070 2071 /* 2072 * Figure out a rough estimate of what maximum number of pages 2073 * this buffer could use (a high estimate of course). 2074 */ 2075 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2076 2077 if (dma->dp_dvma_used) { 2078 /* 2079 * The number of physical pages is the worst case. 2080 * 2081 * For DVMA, the worst case is the length divided 2082 * by the maximum cookie length, plus 1. Add to that 2083 * the number of segment boundaries potentially crossed, and 2084 * the additional number of DVMA segments that was returned. 2085 * 2086 * In the normal case, for modern devices, si_cancross will 2087 * be false, and dv_nseg will be 1, and the fast path will 2088 * have been taken above. 2089 */ 2090 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2091 + 1; 2092 if (sinfo->si_cancross) 2093 ncookies += 2094 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2095 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2096 2097 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2098 } 2099 2100 /* 2101 * We'll use the pre-allocated cookies for any bind that will *always* 2102 * fit (more important to be consistent, we don't want to create 2103 * additional degenerate cases). 2104 */ 2105 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2106 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2107 dma->dp_need_to_free_cookie = B_FALSE; 2108 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2109 uint_t, sinfo->si_max_pages); 2110 2111 /* 2112 * For anything larger than that, we'll go ahead and allocate the 2113 * maximum number of pages we expect to see. Hopefuly, we won't be 2114 * seeing this path in the fast path for high performance devices very 2115 * frequently. 2116 * 2117 * a ddi bind interface that allowed the driver to provide storage to 2118 * the bind interface would speed this case up. 2119 */ 2120 } else { 2121 /* 2122 * Save away how much memory we allocated. If we're doing a 2123 * nosleep, the alloc could fail... 2124 */ 2125 dma->dp_cookie_size = sinfo->si_max_pages * 2126 sizeof (ddi_dma_cookie_t); 2127 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2128 if (dma->dp_cookies == NULL) { 2129 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2130 rootnex_clean_dmahdl(hp); 2131 return (DDI_DMA_NORESOURCES); 2132 } 2133 dma->dp_need_to_free_cookie = B_TRUE; 2134 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2135 uint_t, sinfo->si_max_pages); 2136 } 2137 hp->dmai_cookie = dma->dp_cookies; 2138 2139 /* 2140 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2141 * looking at the constraints in the dma structure. It will then put 2142 * some additional state about the sgl in the dma struct (i.e. is 2143 * the sgl clean, or do we need to do some munging; how many pages 2144 * need to be copied, etc.) 2145 */ 2146 if (dma->dp_dvma_used) 2147 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2148 else 2149 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2150 2151 out: 2152 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2153 /* if we don't need a copy buffer, we don't need to sync */ 2154 if (sinfo->si_copybuf_req == 0) { 2155 hp->dmai_rflags |= DMP_NOSYNC; 2156 } 2157 2158 /* 2159 * if we don't need the copybuf and we don't need to do a partial, we 2160 * hit the fast path. All the high performance devices should be trying 2161 * to hit this path. To hit this path, a device should be able to reach 2162 * all of memory, shouldn't try to bind more than it can transfer, and 2163 * the buffer shouldn't require more cookies than the driver/device can 2164 * handle [sgllen]). 2165 * 2166 * Note that negative values of dma_attr_sgllen are supposed 2167 * to mean unlimited, but we just cast them to mean a 2168 * "ridiculous large limit". This saves some extra checks on 2169 * hot paths. 2170 */ 2171 if ((sinfo->si_copybuf_req == 0) && 2172 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2173 (dmao->dmao_size <= dma->dp_maxxfer)) { 2174 fast: 2175 /* 2176 * If the driver supports FMA, insert the handle in the FMA DMA 2177 * handle cache. 2178 */ 2179 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2180 hp->dmai_error.err_cf = rootnex_dma_check; 2181 2182 /* 2183 * copy out the first cookie and ccountp, set the cookie 2184 * pointer to the second cookie. The first cookie is passed 2185 * back on the stack. Additional cookies are accessed via 2186 * ddi_dma_nextcookie() 2187 */ 2188 *cookiep = dma->dp_cookies[0]; 2189 *ccountp = sinfo->si_sgl_size; 2190 hp->dmai_cookie++; 2191 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2192 hp->dmai_ncookies = *ccountp; 2193 hp->dmai_curcookie = 1; 2194 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2195 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2196 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2197 uint_t, dmao->dmao_size, uint_t, *ccountp); 2198 2199 2200 return (DDI_DMA_MAPPED); 2201 } 2202 2203 /* 2204 * go to the slow path, we may need to alloc more memory, create 2205 * multiple windows, and munge up a sgl to make the device happy. 2206 */ 2207 2208 /* 2209 * With the IOMMU mapobject method used, we should never hit 2210 * the slow path. If we do, something is seriously wrong. 2211 * Clean up and return an error. 2212 */ 2213 2214 #if defined(__amd64) && !defined(__xpv) 2215 2216 if (dma->dp_dvma_used) { 2217 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2218 &dma->dp_dvma); 2219 e = DDI_DMA_NOMAPPING; 2220 } else { 2221 #endif 2222 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2223 kmflag); 2224 #if defined(__amd64) && !defined(__xpv) 2225 } 2226 #endif 2227 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2228 if (dma->dp_need_to_free_cookie) { 2229 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2230 } 2231 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2232 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2233 return (e); 2234 } 2235 2236 /* 2237 * If the driver supports FMA, insert the handle in the FMA DMA handle 2238 * cache. 2239 */ 2240 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2241 hp->dmai_error.err_cf = rootnex_dma_check; 2242 2243 /* if the first window uses the copy buffer, sync it for the device */ 2244 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2245 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2246 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2247 DDI_DMA_SYNC_FORDEV); 2248 } 2249 2250 /* 2251 * copy out the first cookie and ccountp, set the cookie pointer to the 2252 * second cookie. Make sure the partial flag is set/cleared correctly. 2253 * If we have a partial map (i.e. multiple windows), the number of 2254 * cookies we return is the number of cookies in the first window. 2255 */ 2256 if (e == DDI_DMA_MAPPED) { 2257 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2258 *ccountp = sinfo->si_sgl_size; 2259 hp->dmai_nwin = 1; 2260 } else { 2261 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2262 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2263 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2264 } 2265 *cookiep = dma->dp_cookies[0]; 2266 hp->dmai_cookie++; 2267 hp->dmai_ncookies = *ccountp; 2268 hp->dmai_curcookie = 1; 2269 2270 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2271 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2272 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2273 dmao->dmao_size, uint_t, *ccountp); 2274 return (e); 2275 } 2276 2277 /* 2278 * rootnex_dma_bindhdl() 2279 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2280 */ 2281 static int 2282 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2283 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2284 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2285 { 2286 int ret; 2287 #if defined(__amd64) && !defined(__xpv) 2288 if (IOMMU_USED(rdip)) 2289 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2290 cookiep, ccountp); 2291 else 2292 #endif 2293 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2294 cookiep, ccountp); 2295 2296 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2297 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2298 &rootnex_state->r_dvma_call_list_id); 2299 } 2300 2301 return (ret); 2302 } 2303 2304 2305 2306 /*ARGSUSED*/ 2307 static int 2308 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2309 ddi_dma_handle_t handle) 2310 { 2311 ddi_dma_impl_t *hp; 2312 rootnex_dma_t *dma; 2313 int e; 2314 2315 hp = (ddi_dma_impl_t *)handle; 2316 dma = (rootnex_dma_t *)hp->dmai_private; 2317 2318 /* make sure the buffer wasn't free'd before calling unbind */ 2319 if (rootnex_unbind_verify_buffer) { 2320 e = rootnex_verify_buffer(dma); 2321 if (e != DDI_SUCCESS) { 2322 ASSERT(0); 2323 return (DDI_FAILURE); 2324 } 2325 } 2326 2327 /* sync the current window before unbinding the buffer */ 2328 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2329 (hp->dmai_rflags & DDI_DMA_READ)) { 2330 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2331 DDI_DMA_SYNC_FORCPU); 2332 } 2333 2334 /* 2335 * cleanup and copy buffer or window state. if we didn't use the copy 2336 * buffer or windows, there won't be much to do :-) 2337 */ 2338 rootnex_teardown_copybuf(dma); 2339 rootnex_teardown_windows(dma); 2340 2341 #if defined(__amd64) && !defined(__xpv) 2342 if (IOMMU_USED(rdip) && dma->dp_dvma_used) 2343 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2344 &dma->dp_dvma); 2345 #endif 2346 2347 /* 2348 * If we had to allocate space to for the worse case sgl (it didn't 2349 * fit into our pre-allocate buffer), free that up now 2350 */ 2351 if (dma->dp_need_to_free_cookie) { 2352 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2353 } 2354 2355 /* 2356 * clean up the handle so it's ready for the next bind (i.e. if the 2357 * handle is reused). 2358 */ 2359 rootnex_clean_dmahdl(hp); 2360 hp->dmai_error.err_cf = NULL; 2361 2362 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2363 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2364 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2365 2366 return (DDI_SUCCESS); 2367 } 2368 2369 /* 2370 * rootnex_dma_unbindhdl() 2371 * called from ddi_dma_unbind_handle() 2372 */ 2373 /*ARGSUSED*/ 2374 static int 2375 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2376 ddi_dma_handle_t handle) 2377 { 2378 int ret; 2379 2380 #if defined(__amd64) && !defined(__xpv) 2381 if (IOMMU_USED(rdip)) 2382 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2383 else 2384 #endif 2385 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2386 2387 if (rootnex_state->r_dvma_call_list_id) 2388 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2389 2390 return (ret); 2391 } 2392 2393 #if defined(__amd64) && !defined(__xpv) 2394 2395 static int 2396 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2397 { 2398 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2399 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2400 2401 if (dma->dp_sleep_flags != KM_SLEEP && 2402 dma->dp_sleep_flags != KM_NOSLEEP) 2403 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2404 return (dma->dp_sleep_flags); 2405 } 2406 /*ARGSUSED*/ 2407 static void 2408 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2409 { 2410 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2411 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2412 rootnex_window_t *window; 2413 2414 if (dma->dp_window) { 2415 window = &dma->dp_window[dma->dp_current_win]; 2416 hp->dmai_cookie = window->wd_first_cookie; 2417 } else { 2418 hp->dmai_cookie = dma->dp_cookies; 2419 } 2420 hp->dmai_cookie++; 2421 hp->dmai_curcookie = 1; 2422 } 2423 2424 /*ARGSUSED*/ 2425 static int 2426 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2427 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2428 { 2429 int i; 2430 int km_flags; 2431 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2432 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2433 rootnex_window_t *window; 2434 ddi_dma_cookie_t *cp; 2435 ddi_dma_cookie_t *cookie; 2436 2437 ASSERT(*cookiepp == NULL); 2438 ASSERT(*ccountp == 0); 2439 2440 if (dma->dp_window) { 2441 window = &dma->dp_window[dma->dp_current_win]; 2442 cp = window->wd_first_cookie; 2443 *ccountp = window->wd_cookie_cnt; 2444 } else { 2445 cp = dma->dp_cookies; 2446 *ccountp = dma->dp_sglinfo.si_sgl_size; 2447 } 2448 2449 km_flags = rootnex_coredma_get_sleep_flags(handle); 2450 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2451 if (cookie == NULL) { 2452 return (DDI_DMA_NORESOURCES); 2453 } 2454 2455 for (i = 0; i < *ccountp; i++) { 2456 cookie[i].dmac_notused = cp[i].dmac_notused; 2457 cookie[i].dmac_type = cp[i].dmac_type; 2458 cookie[i].dmac_address = cp[i].dmac_address; 2459 cookie[i].dmac_size = cp[i].dmac_size; 2460 } 2461 2462 *cookiepp = cookie; 2463 2464 return (DDI_SUCCESS); 2465 } 2466 2467 /*ARGSUSED*/ 2468 static int 2469 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2470 ddi_dma_cookie_t *cookiep, uint_t ccount) 2471 { 2472 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2473 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2474 rootnex_window_t *window; 2475 ddi_dma_cookie_t *cur_cookiep; 2476 2477 ASSERT(cookiep); 2478 ASSERT(ccount != 0); 2479 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2480 2481 if (dma->dp_window) { 2482 window = &dma->dp_window[dma->dp_current_win]; 2483 dma->dp_saved_cookies = window->wd_first_cookie; 2484 window->wd_first_cookie = cookiep; 2485 ASSERT(ccount == window->wd_cookie_cnt); 2486 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2487 + window->wd_first_cookie; 2488 } else { 2489 dma->dp_saved_cookies = dma->dp_cookies; 2490 dma->dp_cookies = cookiep; 2491 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2492 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2493 + dma->dp_cookies; 2494 } 2495 2496 dma->dp_need_to_switch_cookies = B_TRUE; 2497 hp->dmai_cookie = cur_cookiep; 2498 2499 return (DDI_SUCCESS); 2500 } 2501 2502 /*ARGSUSED*/ 2503 static int 2504 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2505 { 2506 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2507 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2508 rootnex_window_t *window; 2509 ddi_dma_cookie_t *cur_cookiep; 2510 ddi_dma_cookie_t *cookie_array; 2511 uint_t ccount; 2512 2513 /* check if cookies have not been switched */ 2514 if (dma->dp_need_to_switch_cookies == B_FALSE) 2515 return (DDI_SUCCESS); 2516 2517 ASSERT(dma->dp_saved_cookies); 2518 2519 if (dma->dp_window) { 2520 window = &dma->dp_window[dma->dp_current_win]; 2521 cookie_array = window->wd_first_cookie; 2522 window->wd_first_cookie = dma->dp_saved_cookies; 2523 dma->dp_saved_cookies = NULL; 2524 ccount = window->wd_cookie_cnt; 2525 cur_cookiep = (hp->dmai_cookie - cookie_array) 2526 + window->wd_first_cookie; 2527 } else { 2528 cookie_array = dma->dp_cookies; 2529 dma->dp_cookies = dma->dp_saved_cookies; 2530 dma->dp_saved_cookies = NULL; 2531 ccount = dma->dp_sglinfo.si_sgl_size; 2532 cur_cookiep = (hp->dmai_cookie - cookie_array) 2533 + dma->dp_cookies; 2534 } 2535 2536 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2537 2538 hp->dmai_cookie = cur_cookiep; 2539 2540 dma->dp_need_to_switch_cookies = B_FALSE; 2541 2542 return (DDI_SUCCESS); 2543 } 2544 2545 #endif 2546 2547 static struct as * 2548 rootnex_get_as(ddi_dma_obj_t *dmao) 2549 { 2550 struct as *asp; 2551 2552 switch (dmao->dmao_type) { 2553 case DMA_OTYP_VADDR: 2554 case DMA_OTYP_BUFVADDR: 2555 asp = dmao->dmao_obj.virt_obj.v_as; 2556 if (asp == NULL) 2557 asp = &kas; 2558 break; 2559 default: 2560 asp = NULL; 2561 break; 2562 } 2563 return (asp); 2564 } 2565 2566 /* 2567 * rootnex_verify_buffer() 2568 * verify buffer wasn't free'd 2569 */ 2570 static int 2571 rootnex_verify_buffer(rootnex_dma_t *dma) 2572 { 2573 page_t **pplist; 2574 caddr_t vaddr; 2575 uint_t pcnt; 2576 uint_t poff; 2577 page_t *pp; 2578 char b; 2579 int i; 2580 2581 /* Figure out how many pages this buffer occupies */ 2582 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2583 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2584 } else { 2585 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2586 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2587 } 2588 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2589 2590 switch (dma->dp_dma.dmao_type) { 2591 case DMA_OTYP_PAGES: 2592 /* 2593 * for a linked list of pp's walk through them to make sure 2594 * they're locked and not free. 2595 */ 2596 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2597 for (i = 0; i < pcnt; i++) { 2598 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2599 return (DDI_FAILURE); 2600 } 2601 pp = pp->p_next; 2602 } 2603 break; 2604 2605 case DMA_OTYP_VADDR: 2606 case DMA_OTYP_BUFVADDR: 2607 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2608 /* 2609 * for an array of pp's walk through them to make sure they're 2610 * not free. It's possible that they may not be locked. 2611 */ 2612 if (pplist) { 2613 for (i = 0; i < pcnt; i++) { 2614 if (PP_ISFREE(pplist[i])) { 2615 return (DDI_FAILURE); 2616 } 2617 } 2618 2619 /* For a virtual address, try to peek at each page */ 2620 } else { 2621 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2622 for (i = 0; i < pcnt; i++) { 2623 if (ddi_peek8(NULL, vaddr, &b) == 2624 DDI_FAILURE) 2625 return (DDI_FAILURE); 2626 vaddr += MMU_PAGESIZE; 2627 } 2628 } 2629 } 2630 break; 2631 2632 default: 2633 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2634 break; 2635 } 2636 2637 return (DDI_SUCCESS); 2638 } 2639 2640 2641 /* 2642 * rootnex_clean_dmahdl() 2643 * Clean the dma handle. This should be called on a handle alloc and an 2644 * unbind handle. Set the handle state to the default settings. 2645 */ 2646 static void 2647 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2648 { 2649 rootnex_dma_t *dma; 2650 2651 2652 dma = (rootnex_dma_t *)hp->dmai_private; 2653 2654 hp->dmai_nwin = 0; 2655 dma->dp_current_cookie = 0; 2656 dma->dp_copybuf_size = 0; 2657 dma->dp_window = NULL; 2658 dma->dp_cbaddr = NULL; 2659 dma->dp_inuse = B_FALSE; 2660 dma->dp_dvma_used = B_FALSE; 2661 dma->dp_need_to_free_cookie = B_FALSE; 2662 dma->dp_need_to_switch_cookies = B_FALSE; 2663 dma->dp_saved_cookies = NULL; 2664 dma->dp_sleep_flags = KM_PANIC; 2665 dma->dp_need_to_free_window = B_FALSE; 2666 dma->dp_partial_required = B_FALSE; 2667 dma->dp_trim_required = B_FALSE; 2668 dma->dp_sglinfo.si_copybuf_req = 0; 2669 #if !defined(__amd64) 2670 dma->dp_cb_remaping = B_FALSE; 2671 dma->dp_kva = NULL; 2672 #endif 2673 2674 /* FMA related initialization */ 2675 hp->dmai_fault = 0; 2676 hp->dmai_fault_check = NULL; 2677 hp->dmai_fault_notify = NULL; 2678 hp->dmai_error.err_ena = 0; 2679 hp->dmai_error.err_status = DDI_FM_OK; 2680 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2681 hp->dmai_error.err_ontrap = NULL; 2682 2683 /* Cookie tracking */ 2684 hp->dmai_ncookies = 0; 2685 hp->dmai_curcookie = 0; 2686 } 2687 2688 2689 /* 2690 * rootnex_valid_alloc_parms() 2691 * Called in ddi_dma_alloc_handle path to validate its parameters. 2692 */ 2693 static int 2694 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2695 { 2696 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2697 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2698 (attr->dma_attr_granular > MMU_PAGESIZE) || 2699 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2700 return (DDI_DMA_BADATTR); 2701 } 2702 2703 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2704 return (DDI_DMA_BADATTR); 2705 } 2706 2707 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2708 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2709 attr->dma_attr_sgllen == 0) { 2710 return (DDI_DMA_BADATTR); 2711 } 2712 2713 /* We should be able to DMA into every byte offset in a page */ 2714 if (maxsegmentsize < MMU_PAGESIZE) { 2715 return (DDI_DMA_BADATTR); 2716 } 2717 2718 /* if we're bouncing on seg, seg must be <= addr_hi */ 2719 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2720 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2721 return (DDI_DMA_BADATTR); 2722 } 2723 return (DDI_SUCCESS); 2724 } 2725 2726 /* 2727 * rootnex_valid_bind_parms() 2728 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2729 */ 2730 /* ARGSUSED */ 2731 static int 2732 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2733 { 2734 #if !defined(__amd64) 2735 /* 2736 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2737 * we can track the offset for the obsoleted interfaces. 2738 */ 2739 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2740 return (DDI_DMA_TOOBIG); 2741 } 2742 #endif 2743 2744 return (DDI_SUCCESS); 2745 } 2746 2747 2748 /* 2749 * rootnex_need_bounce_seg() 2750 * check to see if the buffer lives on both side of the seg. 2751 */ 2752 static boolean_t 2753 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2754 { 2755 ddi_dma_atyp_t buftype; 2756 rootnex_addr_t raddr; 2757 boolean_t lower_addr; 2758 boolean_t upper_addr; 2759 uint64_t offset; 2760 page_t **pplist; 2761 uint64_t paddr; 2762 uint32_t psize; 2763 uint32_t size; 2764 caddr_t vaddr; 2765 uint_t pcnt; 2766 page_t *pp; 2767 2768 pp = NULL; 2769 /* shortcuts */ 2770 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2771 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2772 buftype = dmar_object->dmao_type; 2773 size = dmar_object->dmao_size; 2774 2775 lower_addr = B_FALSE; 2776 upper_addr = B_FALSE; 2777 pcnt = 0; 2778 2779 /* 2780 * Process the first page to handle the initial offset of the buffer. 2781 * We'll use the base address we get later when we loop through all 2782 * the pages. 2783 */ 2784 if (buftype == DMA_OTYP_PAGES) { 2785 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2786 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2787 MMU_PAGEOFFSET; 2788 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2789 psize = MIN(size, (MMU_PAGESIZE - offset)); 2790 pp = pp->p_next; 2791 sglinfo->si_asp = NULL; 2792 } else if (pplist != NULL) { 2793 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2794 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2795 if (sglinfo->si_asp == NULL) { 2796 sglinfo->si_asp = &kas; 2797 } 2798 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2799 paddr += offset; 2800 psize = MIN(size, (MMU_PAGESIZE - offset)); 2801 pcnt++; 2802 } else { 2803 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2804 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2805 if (sglinfo->si_asp == NULL) { 2806 sglinfo->si_asp = &kas; 2807 } 2808 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2809 paddr += offset; 2810 psize = MIN(size, (MMU_PAGESIZE - offset)); 2811 vaddr += psize; 2812 } 2813 2814 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2815 2816 if ((raddr + psize) > sglinfo->si_segmask) { 2817 upper_addr = B_TRUE; 2818 } else { 2819 lower_addr = B_TRUE; 2820 } 2821 size -= psize; 2822 2823 /* 2824 * Walk through the rest of the pages in the buffer. Track to see 2825 * if we have pages on both sides of the segment boundary. 2826 */ 2827 while (size > 0) { 2828 /* partial or full page */ 2829 psize = MIN(size, MMU_PAGESIZE); 2830 2831 if (buftype == DMA_OTYP_PAGES) { 2832 /* get the paddr from the page_t */ 2833 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2834 paddr = pfn_to_pa(pp->p_pagenum); 2835 pp = pp->p_next; 2836 } else if (pplist != NULL) { 2837 /* index into the array of page_t's to get the paddr */ 2838 ASSERT(!PP_ISFREE(pplist[pcnt])); 2839 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2840 pcnt++; 2841 } else { 2842 /* call into the VM to get the paddr */ 2843 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2844 vaddr)); 2845 vaddr += psize; 2846 } 2847 2848 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2849 2850 if ((raddr + psize) > sglinfo->si_segmask) { 2851 upper_addr = B_TRUE; 2852 } else { 2853 lower_addr = B_TRUE; 2854 } 2855 /* 2856 * if the buffer lives both above and below the segment 2857 * boundary, or the current page is the page immediately 2858 * after the segment, we will use a copy/bounce buffer for 2859 * all pages > seg. 2860 */ 2861 if ((lower_addr && upper_addr) || 2862 (raddr == (sglinfo->si_segmask + 1))) { 2863 return (B_TRUE); 2864 } 2865 2866 size -= psize; 2867 } 2868 2869 return (B_FALSE); 2870 } 2871 2872 /* 2873 * rootnex_get_sgl() 2874 * Called in bind fastpath to get the sgl. Most of this will be replaced 2875 * with a call to the vm layer when vm2.0 comes around... 2876 */ 2877 static void 2878 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2879 rootnex_sglinfo_t *sglinfo) 2880 { 2881 ddi_dma_atyp_t buftype; 2882 rootnex_addr_t raddr; 2883 uint64_t last_page; 2884 uint64_t offset; 2885 uint64_t addrhi; 2886 uint64_t addrlo; 2887 uint64_t maxseg; 2888 page_t **pplist; 2889 uint64_t paddr; 2890 uint32_t psize; 2891 uint32_t size; 2892 caddr_t vaddr; 2893 uint_t pcnt; 2894 page_t *pp; 2895 uint_t cnt; 2896 2897 pp = NULL; 2898 /* shortcuts */ 2899 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2900 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2901 maxseg = sglinfo->si_max_cookie_size; 2902 buftype = dmar_object->dmao_type; 2903 addrhi = sglinfo->si_max_addr; 2904 addrlo = sglinfo->si_min_addr; 2905 size = dmar_object->dmao_size; 2906 2907 pcnt = 0; 2908 cnt = 0; 2909 2910 2911 /* 2912 * check to see if we need to use the copy buffer for pages over 2913 * the segment attr. 2914 */ 2915 sglinfo->si_bounce_on_seg = B_FALSE; 2916 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2917 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2918 dmar_object, sglinfo); 2919 } 2920 2921 /* 2922 * if we were passed down a linked list of pages, i.e. pointer to 2923 * page_t, use this to get our physical address and buf offset. 2924 */ 2925 if (buftype == DMA_OTYP_PAGES) { 2926 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2927 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2928 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2929 MMU_PAGEOFFSET; 2930 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2931 psize = MIN(size, (MMU_PAGESIZE - offset)); 2932 pp = pp->p_next; 2933 sglinfo->si_asp = NULL; 2934 2935 /* 2936 * We weren't passed down a linked list of pages, but if we were passed 2937 * down an array of pages, use this to get our physical address and buf 2938 * offset. 2939 */ 2940 } else if (pplist != NULL) { 2941 ASSERT((buftype == DMA_OTYP_VADDR) || 2942 (buftype == DMA_OTYP_BUFVADDR)); 2943 2944 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2945 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2946 if (sglinfo->si_asp == NULL) { 2947 sglinfo->si_asp = &kas; 2948 } 2949 2950 ASSERT(!PP_ISFREE(pplist[pcnt])); 2951 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2952 paddr += offset; 2953 psize = MIN(size, (MMU_PAGESIZE - offset)); 2954 pcnt++; 2955 2956 /* 2957 * All we have is a virtual address, we'll need to call into the VM 2958 * to get the physical address. 2959 */ 2960 } else { 2961 ASSERT((buftype == DMA_OTYP_VADDR) || 2962 (buftype == DMA_OTYP_BUFVADDR)); 2963 2964 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2965 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2966 if (sglinfo->si_asp == NULL) { 2967 sglinfo->si_asp = &kas; 2968 } 2969 2970 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2971 paddr += offset; 2972 psize = MIN(size, (MMU_PAGESIZE - offset)); 2973 vaddr += psize; 2974 } 2975 2976 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2977 2978 /* 2979 * Setup the first cookie with the physical address of the page and the 2980 * size of the page (which takes into account the initial offset into 2981 * the page. 2982 */ 2983 sgl[cnt].dmac_laddress = raddr; 2984 sgl[cnt].dmac_size = psize; 2985 sgl[cnt].dmac_type = 0; 2986 2987 /* 2988 * Save away the buffer offset into the page. We'll need this later in 2989 * the copy buffer code to help figure out the page index within the 2990 * buffer and the offset into the current page. 2991 */ 2992 sglinfo->si_buf_offset = offset; 2993 2994 /* 2995 * If we are using the copy buffer for anything over the segment 2996 * boundary, and this page is over the segment boundary. 2997 * OR 2998 * if the DMA engine can't reach the physical address. 2999 */ 3000 if (((sglinfo->si_bounce_on_seg) && 3001 ((raddr + psize) > sglinfo->si_segmask)) || 3002 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3003 /* 3004 * Increase how much copy buffer we use. We always increase by 3005 * pagesize so we don't have to worry about converting offsets. 3006 * Set a flag in the cookies dmac_type to indicate that it uses 3007 * the copy buffer. If this isn't the last cookie, go to the 3008 * next cookie (since we separate each page which uses the copy 3009 * buffer in case the copy buffer is not physically contiguous. 3010 */ 3011 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3012 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3013 if ((cnt + 1) < sglinfo->si_max_pages) { 3014 cnt++; 3015 sgl[cnt].dmac_laddress = 0; 3016 sgl[cnt].dmac_size = 0; 3017 sgl[cnt].dmac_type = 0; 3018 } 3019 } 3020 3021 /* 3022 * save this page's physical address so we can figure out if the next 3023 * page is physically contiguous. Keep decrementing size until we are 3024 * done with the buffer. 3025 */ 3026 last_page = raddr & MMU_PAGEMASK; 3027 size -= psize; 3028 3029 while (size > 0) { 3030 /* Get the size for this page (i.e. partial or full page) */ 3031 psize = MIN(size, MMU_PAGESIZE); 3032 3033 if (buftype == DMA_OTYP_PAGES) { 3034 /* get the paddr from the page_t */ 3035 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3036 paddr = pfn_to_pa(pp->p_pagenum); 3037 pp = pp->p_next; 3038 } else if (pplist != NULL) { 3039 /* index into the array of page_t's to get the paddr */ 3040 ASSERT(!PP_ISFREE(pplist[pcnt])); 3041 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3042 pcnt++; 3043 } else { 3044 /* call into the VM to get the paddr */ 3045 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3046 vaddr)); 3047 vaddr += psize; 3048 } 3049 3050 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3051 3052 /* 3053 * If we are using the copy buffer for anything over the 3054 * segment boundary, and this page is over the segment 3055 * boundary. 3056 * OR 3057 * if the DMA engine can't reach the physical address. 3058 */ 3059 if (((sglinfo->si_bounce_on_seg) && 3060 ((raddr + psize) > sglinfo->si_segmask)) || 3061 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3062 3063 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3064 3065 /* 3066 * if there is something in the current cookie, go to 3067 * the next one. We only want one page in a cookie which 3068 * uses the copybuf since the copybuf doesn't have to 3069 * be physically contiguous. 3070 */ 3071 if (sgl[cnt].dmac_size != 0) { 3072 cnt++; 3073 } 3074 sgl[cnt].dmac_laddress = raddr; 3075 sgl[cnt].dmac_size = psize; 3076 #if defined(__amd64) 3077 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3078 #else 3079 /* 3080 * save the buf offset for 32-bit kernel. used in the 3081 * obsoleted interfaces. 3082 */ 3083 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3084 (dmar_object->dmao_size - size); 3085 #endif 3086 /* if this isn't the last cookie, go to the next one */ 3087 if ((cnt + 1) < sglinfo->si_max_pages) { 3088 cnt++; 3089 sgl[cnt].dmac_laddress = 0; 3090 sgl[cnt].dmac_size = 0; 3091 sgl[cnt].dmac_type = 0; 3092 } 3093 3094 /* 3095 * this page didn't need the copy buffer, if it's not physically 3096 * contiguous, or it would put us over a segment boundary, or it 3097 * puts us over the max cookie size, or the current sgl doesn't 3098 * have anything in it. 3099 */ 3100 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3101 !(raddr & sglinfo->si_segmask) || 3102 ((sgl[cnt].dmac_size + psize) > maxseg) || 3103 (sgl[cnt].dmac_size == 0)) { 3104 /* 3105 * if we're not already in a new cookie, go to the next 3106 * cookie. 3107 */ 3108 if (sgl[cnt].dmac_size != 0) { 3109 cnt++; 3110 } 3111 3112 /* save the cookie information */ 3113 sgl[cnt].dmac_laddress = raddr; 3114 sgl[cnt].dmac_size = psize; 3115 #if defined(__amd64) 3116 sgl[cnt].dmac_type = 0; 3117 #else 3118 /* 3119 * save the buf offset for 32-bit kernel. used in the 3120 * obsoleted interfaces. 3121 */ 3122 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3123 #endif 3124 3125 /* 3126 * this page didn't need the copy buffer, it is physically 3127 * contiguous with the last page, and it's <= the max cookie 3128 * size. 3129 */ 3130 } else { 3131 sgl[cnt].dmac_size += psize; 3132 3133 /* 3134 * If this cookie is used up, and more cookies 3135 * are available, then move onto the next one. 3136 */ 3137 if ((sgl[cnt].dmac_size == maxseg) && 3138 ((cnt + 1) < sglinfo->si_max_pages)) { 3139 cnt++; 3140 sgl[cnt].dmac_laddress = 0; 3141 sgl[cnt].dmac_size = 0; 3142 sgl[cnt].dmac_type = 0; 3143 } 3144 } 3145 3146 /* 3147 * save this page's physical address so we can figure out if the 3148 * next page is physically contiguous. Keep decrementing size 3149 * until we are done with the buffer. 3150 */ 3151 last_page = raddr; 3152 size -= psize; 3153 } 3154 3155 /* we're done, save away how many cookies the sgl has */ 3156 if (sgl[cnt].dmac_size == 0) { 3157 ASSERT(cnt < sglinfo->si_max_pages); 3158 sglinfo->si_sgl_size = cnt; 3159 } else { 3160 sglinfo->si_sgl_size = cnt + 1; 3161 } 3162 } 3163 3164 static void 3165 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3166 rootnex_sglinfo_t *sglinfo) 3167 { 3168 uint64_t offset; 3169 uint64_t maxseg; 3170 uint64_t dvaddr; 3171 struct dvmaseg *dvs; 3172 uint64_t paddr; 3173 uint32_t psize, ssize; 3174 uint32_t size; 3175 uint_t cnt; 3176 int physcontig; 3177 3178 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3179 3180 /* shortcuts */ 3181 maxseg = sglinfo->si_max_cookie_size; 3182 size = dmar_object->dmao_size; 3183 3184 cnt = 0; 3185 sglinfo->si_bounce_on_seg = B_FALSE; 3186 3187 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3188 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3189 ssize = dvs->dvs_len; 3190 paddr = dvs->dvs_start; 3191 paddr += offset; 3192 psize = MIN(ssize, (maxseg - offset)); 3193 dvaddr = paddr + psize; 3194 ssize -= psize; 3195 3196 sgl[cnt].dmac_laddress = paddr; 3197 sgl[cnt].dmac_size = psize; 3198 sgl[cnt].dmac_type = 0; 3199 3200 size -= psize; 3201 while (size > 0) { 3202 if (ssize == 0) { 3203 dvs++; 3204 ssize = dvs->dvs_len; 3205 dvaddr = dvs->dvs_start; 3206 physcontig = 0; 3207 } else 3208 physcontig = 1; 3209 3210 paddr = dvaddr; 3211 psize = MIN(ssize, maxseg); 3212 dvaddr += psize; 3213 ssize -= psize; 3214 3215 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3216 ((sgl[cnt].dmac_size + psize) > maxseg) || 3217 (sgl[cnt].dmac_size == 0)) { 3218 /* 3219 * if we're not already in a new cookie, go to the next 3220 * cookie. 3221 */ 3222 if (sgl[cnt].dmac_size != 0) { 3223 cnt++; 3224 } 3225 3226 /* save the cookie information */ 3227 sgl[cnt].dmac_laddress = paddr; 3228 sgl[cnt].dmac_size = psize; 3229 sgl[cnt].dmac_type = 0; 3230 } else { 3231 sgl[cnt].dmac_size += psize; 3232 3233 /* 3234 * If this cookie is used up, and more cookies 3235 * are available, then move onto the next one. 3236 */ 3237 if ((sgl[cnt].dmac_size == maxseg) && 3238 ((cnt + 1) < sglinfo->si_max_pages)) { 3239 cnt++; 3240 sgl[cnt].dmac_laddress = 0; 3241 sgl[cnt].dmac_size = 0; 3242 sgl[cnt].dmac_type = 0; 3243 } 3244 } 3245 size -= psize; 3246 } 3247 3248 /* we're done, save away how many cookies the sgl has */ 3249 if (sgl[cnt].dmac_size == 0) { 3250 sglinfo->si_sgl_size = cnt; 3251 } else { 3252 sglinfo->si_sgl_size = cnt + 1; 3253 } 3254 } 3255 3256 /* 3257 * rootnex_bind_slowpath() 3258 * Call in the bind path if the calling driver can't use the sgl without 3259 * modifying it. We either need to use the copy buffer and/or we will end up 3260 * with a partial bind. 3261 */ 3262 static int 3263 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3264 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3265 { 3266 rootnex_sglinfo_t *sinfo; 3267 rootnex_window_t *window; 3268 ddi_dma_cookie_t *cookie; 3269 size_t copybuf_used; 3270 size_t dmac_size; 3271 boolean_t partial; 3272 off_t cur_offset; 3273 page_t *cur_pp; 3274 major_t mnum; 3275 int e; 3276 int i; 3277 3278 3279 sinfo = &dma->dp_sglinfo; 3280 copybuf_used = 0; 3281 partial = B_FALSE; 3282 3283 /* 3284 * If we're using the copybuf, set the copybuf state in dma struct. 3285 * Needs to be first since it sets the copy buffer size. 3286 */ 3287 if (sinfo->si_copybuf_req != 0) { 3288 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3289 if (e != DDI_SUCCESS) { 3290 return (e); 3291 } 3292 } else { 3293 dma->dp_copybuf_size = 0; 3294 } 3295 3296 /* 3297 * Figure out if we need to do a partial mapping. If so, figure out 3298 * if we need to trim the buffers when we munge the sgl. 3299 */ 3300 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3301 (dmao->dmao_size > dma->dp_maxxfer) || 3302 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3303 dma->dp_partial_required = B_TRUE; 3304 if (attr->dma_attr_granular != 1) { 3305 dma->dp_trim_required = B_TRUE; 3306 } 3307 } else { 3308 dma->dp_partial_required = B_FALSE; 3309 dma->dp_trim_required = B_FALSE; 3310 } 3311 3312 /* If we need to do a partial bind, make sure the driver supports it */ 3313 if (dma->dp_partial_required && 3314 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3315 3316 mnum = ddi_driver_major(dma->dp_dip); 3317 /* 3318 * patchable which allows us to print one warning per major 3319 * number. 3320 */ 3321 if ((rootnex_bind_warn) && 3322 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3323 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3324 cmn_err(CE_WARN, "!%s: coding error detected, the " 3325 "driver is using ddi_dma_attr(9S) incorrectly. " 3326 "There is a small risk of data corruption in " 3327 "particular with large I/Os. The driver should be " 3328 "replaced with a corrected version for proper " 3329 "system operation. To disable this warning, add " 3330 "'set rootnex:rootnex_bind_warn=0' to " 3331 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3332 } 3333 return (DDI_DMA_TOOBIG); 3334 } 3335 3336 /* 3337 * we might need multiple windows, setup state to handle them. In this 3338 * code path, we will have at least one window. 3339 */ 3340 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3341 if (e != DDI_SUCCESS) { 3342 rootnex_teardown_copybuf(dma); 3343 return (e); 3344 } 3345 3346 window = &dma->dp_window[0]; 3347 cookie = &dma->dp_cookies[0]; 3348 cur_offset = 0; 3349 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3350 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3351 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3352 } 3353 3354 /* loop though all the cookies we got back from get_sgl() */ 3355 for (i = 0; i < sinfo->si_sgl_size; i++) { 3356 /* 3357 * If we're using the copy buffer, check this cookie and setup 3358 * its associated copy buffer state. If this cookie uses the 3359 * copy buffer, make sure we sync this window during dma_sync. 3360 */ 3361 if (dma->dp_copybuf_size > 0) { 3362 rootnex_setup_cookie(dmao, dma, cookie, 3363 cur_offset, ©buf_used, &cur_pp); 3364 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3365 window->wd_dosync = B_TRUE; 3366 } 3367 } 3368 3369 /* 3370 * save away the cookie size, since it could be modified in 3371 * the windowing code. 3372 */ 3373 dmac_size = cookie->dmac_size; 3374 3375 /* if we went over max copybuf size */ 3376 if (dma->dp_copybuf_size && 3377 (copybuf_used > dma->dp_copybuf_size)) { 3378 partial = B_TRUE; 3379 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3380 cookie, cur_offset, ©buf_used); 3381 if (e != DDI_SUCCESS) { 3382 rootnex_teardown_copybuf(dma); 3383 rootnex_teardown_windows(dma); 3384 return (e); 3385 } 3386 3387 /* 3388 * if the coookie uses the copy buffer, make sure the 3389 * new window we just moved to is set to sync. 3390 */ 3391 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3392 window->wd_dosync = B_TRUE; 3393 } 3394 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3395 dma->dp_dip); 3396 3397 /* if the cookie cnt == max sgllen, move to the next window */ 3398 } else if (window->wd_cookie_cnt >= 3399 (unsigned)attr->dma_attr_sgllen) { 3400 partial = B_TRUE; 3401 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3402 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3403 cookie, attr, cur_offset); 3404 if (e != DDI_SUCCESS) { 3405 rootnex_teardown_copybuf(dma); 3406 rootnex_teardown_windows(dma); 3407 return (e); 3408 } 3409 3410 /* 3411 * if the coookie uses the copy buffer, make sure the 3412 * new window we just moved to is set to sync. 3413 */ 3414 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3415 window->wd_dosync = B_TRUE; 3416 } 3417 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3418 dma->dp_dip); 3419 3420 /* else if we will be over maxxfer */ 3421 } else if ((window->wd_size + dmac_size) > 3422 dma->dp_maxxfer) { 3423 partial = B_TRUE; 3424 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3425 cookie); 3426 if (e != DDI_SUCCESS) { 3427 rootnex_teardown_copybuf(dma); 3428 rootnex_teardown_windows(dma); 3429 return (e); 3430 } 3431 3432 /* 3433 * if the coookie uses the copy buffer, make sure the 3434 * new window we just moved to is set to sync. 3435 */ 3436 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3437 window->wd_dosync = B_TRUE; 3438 } 3439 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3440 dma->dp_dip); 3441 3442 /* else this cookie fits in the current window */ 3443 } else { 3444 window->wd_cookie_cnt++; 3445 window->wd_size += dmac_size; 3446 } 3447 3448 /* track our offset into the buffer, go to the next cookie */ 3449 ASSERT(dmac_size <= dmao->dmao_size); 3450 ASSERT(cookie->dmac_size <= dmac_size); 3451 cur_offset += dmac_size; 3452 cookie++; 3453 } 3454 3455 /* if we ended up with a zero sized window in the end, clean it up */ 3456 if (window->wd_size == 0) { 3457 hp->dmai_nwin--; 3458 window--; 3459 } 3460 3461 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3462 3463 if (!partial) { 3464 return (DDI_DMA_MAPPED); 3465 } 3466 3467 ASSERT(dma->dp_partial_required); 3468 return (DDI_DMA_PARTIAL_MAP); 3469 } 3470 3471 /* 3472 * rootnex_setup_copybuf() 3473 * Called in bind slowpath. Figures out if we're going to use the copy 3474 * buffer, and if we do, sets up the basic state to handle it. 3475 */ 3476 static int 3477 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3478 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3479 { 3480 rootnex_sglinfo_t *sinfo; 3481 ddi_dma_attr_t lattr; 3482 size_t max_copybuf; 3483 int cansleep; 3484 int e; 3485 #if !defined(__amd64) 3486 int vmflag; 3487 #endif 3488 3489 ASSERT(!dma->dp_dvma_used); 3490 3491 sinfo = &dma->dp_sglinfo; 3492 3493 /* read this first so it's consistent through the routine */ 3494 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3495 3496 /* We need to call into the rootnex on ddi_dma_sync() */ 3497 hp->dmai_rflags &= ~DMP_NOSYNC; 3498 3499 /* make sure the copybuf size <= the max size */ 3500 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3501 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3502 3503 #if !defined(__amd64) 3504 /* 3505 * if we don't have kva space to copy to/from, allocate the KVA space 3506 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3507 * the 64-bit kernel. 3508 */ 3509 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3510 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3511 3512 /* convert the sleep flags */ 3513 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3514 vmflag = VM_SLEEP; 3515 } else { 3516 vmflag = VM_NOSLEEP; 3517 } 3518 3519 /* allocate Kernel VA space that we can bcopy to/from */ 3520 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3521 vmflag); 3522 if (dma->dp_kva == NULL) { 3523 return (DDI_DMA_NORESOURCES); 3524 } 3525 } 3526 #endif 3527 3528 /* convert the sleep flags */ 3529 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3530 cansleep = 1; 3531 } else { 3532 cansleep = 0; 3533 } 3534 3535 /* 3536 * Allocate the actual copy buffer. This needs to fit within the DMA 3537 * engine limits, so we can't use kmem_alloc... We don't need 3538 * contiguous memory (sgllen) since we will be forcing windows on 3539 * sgllen anyway. 3540 */ 3541 lattr = *attr; 3542 lattr.dma_attr_align = MMU_PAGESIZE; 3543 lattr.dma_attr_sgllen = -1; /* no limit */ 3544 /* 3545 * if we're using the copy buffer because of seg, use that for our 3546 * upper address limit. 3547 */ 3548 if (sinfo->si_bounce_on_seg) { 3549 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3550 } 3551 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3552 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3553 if (e != DDI_SUCCESS) { 3554 #if !defined(__amd64) 3555 if (dma->dp_kva != NULL) { 3556 vmem_free(heap_arena, dma->dp_kva, 3557 dma->dp_copybuf_size); 3558 } 3559 #endif 3560 return (DDI_DMA_NORESOURCES); 3561 } 3562 3563 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3564 size_t, dma->dp_copybuf_size); 3565 3566 return (DDI_SUCCESS); 3567 } 3568 3569 3570 /* 3571 * rootnex_setup_windows() 3572 * Called in bind slowpath to setup the window state. We always have windows 3573 * in the slowpath. Even if the window count = 1. 3574 */ 3575 static int 3576 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3577 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3578 { 3579 rootnex_window_t *windowp; 3580 rootnex_sglinfo_t *sinfo; 3581 size_t copy_state_size; 3582 size_t win_state_size; 3583 size_t state_available; 3584 size_t space_needed; 3585 uint_t copybuf_win; 3586 uint_t maxxfer_win; 3587 size_t space_used; 3588 uint_t sglwin; 3589 3590 3591 sinfo = &dma->dp_sglinfo; 3592 3593 dma->dp_current_win = 0; 3594 hp->dmai_nwin = 0; 3595 3596 /* If we don't need to do a partial, we only have one window */ 3597 if (!dma->dp_partial_required) { 3598 dma->dp_max_win = 1; 3599 3600 /* 3601 * we need multiple windows, need to figure out the worse case number 3602 * of windows. 3603 */ 3604 } else { 3605 /* 3606 * if we need windows because we need more copy buffer that 3607 * we allow, the worse case number of windows we could need 3608 * here would be (copybuf space required / copybuf space that 3609 * we have) plus one for remainder, and plus 2 to handle the 3610 * extra pages on the trim for the first and last pages of the 3611 * buffer (a page is the minimum window size so under the right 3612 * attr settings, you could have a window for each page). 3613 * The last page will only be hit here if the size is not a 3614 * multiple of the granularity (which theoretically shouldn't 3615 * be the case but never has been enforced, so we could have 3616 * broken things without it). 3617 */ 3618 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3619 ASSERT(dma->dp_copybuf_size > 0); 3620 copybuf_win = (sinfo->si_copybuf_req / 3621 dma->dp_copybuf_size) + 1 + 2; 3622 } else { 3623 copybuf_win = 0; 3624 } 3625 3626 /* 3627 * if we need windows because we have more cookies than the H/W 3628 * can handle, the number of windows we would need here would 3629 * be (cookie count / cookies count H/W supports minus 1[for 3630 * trim]) plus one for remainder. 3631 */ 3632 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3633 sglwin = (sinfo->si_sgl_size / 3634 (attr->dma_attr_sgllen - 1)) + 1; 3635 } else { 3636 sglwin = 0; 3637 } 3638 3639 /* 3640 * if we need windows because we're binding more memory than the 3641 * H/W can transfer at once, the number of windows we would need 3642 * here would be (xfer count / max xfer H/W supports) plus one 3643 * for remainder, and plus 2 to handle the extra pages on the 3644 * trim (see above comment about trim) 3645 */ 3646 if (dmao->dmao_size > dma->dp_maxxfer) { 3647 maxxfer_win = (dmao->dmao_size / 3648 dma->dp_maxxfer) + 1 + 2; 3649 } else { 3650 maxxfer_win = 0; 3651 } 3652 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3653 ASSERT(dma->dp_max_win > 0); 3654 } 3655 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3656 3657 /* 3658 * Get space for window and potential copy buffer state. Before we 3659 * go and allocate memory, see if we can get away with using what's 3660 * left in the pre-allocted state or the dynamically allocated sgl. 3661 */ 3662 space_used = (uintptr_t)(sinfo->si_sgl_size * 3663 sizeof (ddi_dma_cookie_t)); 3664 3665 /* if we dynamically allocated space for the cookies */ 3666 if (dma->dp_need_to_free_cookie) { 3667 /* if we have more space in the pre-allocted buffer, use it */ 3668 ASSERT(space_used <= dma->dp_cookie_size); 3669 if ((dma->dp_cookie_size - space_used) <= 3670 rootnex_state->r_prealloc_size) { 3671 state_available = rootnex_state->r_prealloc_size; 3672 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3673 3674 /* 3675 * else, we have more free space in the dynamically allocated 3676 * buffer, i.e. the buffer wasn't worse case fragmented so we 3677 * didn't need a lot of cookies. 3678 */ 3679 } else { 3680 state_available = dma->dp_cookie_size - space_used; 3681 windowp = (rootnex_window_t *) 3682 &dma->dp_cookies[sinfo->si_sgl_size]; 3683 } 3684 3685 /* we used the pre-alloced buffer */ 3686 } else { 3687 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3688 state_available = rootnex_state->r_prealloc_size - space_used; 3689 windowp = (rootnex_window_t *) 3690 &dma->dp_cookies[sinfo->si_sgl_size]; 3691 } 3692 3693 /* 3694 * figure out how much state we need to track the copy buffer. Add an 3695 * addition 8 bytes for pointer alignemnt later. 3696 */ 3697 if (dma->dp_copybuf_size > 0) { 3698 copy_state_size = sinfo->si_max_pages * 3699 sizeof (rootnex_pgmap_t); 3700 } else { 3701 copy_state_size = 0; 3702 } 3703 /* add an additional 8 bytes for pointer alignment */ 3704 space_needed = win_state_size + copy_state_size + 0x8; 3705 3706 /* if we have enough space already, use it */ 3707 if (state_available >= space_needed) { 3708 dma->dp_window = windowp; 3709 dma->dp_need_to_free_window = B_FALSE; 3710 3711 /* not enough space, need to allocate more. */ 3712 } else { 3713 dma->dp_window = kmem_alloc(space_needed, kmflag); 3714 if (dma->dp_window == NULL) { 3715 return (DDI_DMA_NORESOURCES); 3716 } 3717 dma->dp_need_to_free_window = B_TRUE; 3718 dma->dp_window_size = space_needed; 3719 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3720 dma->dp_dip, size_t, space_needed); 3721 } 3722 3723 /* 3724 * we allocate copy buffer state and window state at the same time. 3725 * setup our copy buffer state pointers. Make sure it's aligned. 3726 */ 3727 if (dma->dp_copybuf_size > 0) { 3728 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3729 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3730 3731 #if !defined(__amd64) 3732 /* 3733 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3734 * false/NULL. Should be quicker to bzero vs loop and set. 3735 */ 3736 bzero(dma->dp_pgmap, copy_state_size); 3737 #endif 3738 } else { 3739 dma->dp_pgmap = NULL; 3740 } 3741 3742 return (DDI_SUCCESS); 3743 } 3744 3745 3746 /* 3747 * rootnex_teardown_copybuf() 3748 * cleans up after rootnex_setup_copybuf() 3749 */ 3750 static void 3751 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3752 { 3753 #if !defined(__amd64) 3754 int i; 3755 3756 /* 3757 * if we allocated kernel heap VMEM space, go through all the pages and 3758 * map out any of the ones that we're mapped into the kernel heap VMEM 3759 * arena. Then free the VMEM space. 3760 */ 3761 if (dma->dp_kva != NULL) { 3762 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3763 if (dma->dp_pgmap[i].pm_mapped) { 3764 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3765 MMU_PAGESIZE, HAT_UNLOAD); 3766 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3767 } 3768 } 3769 3770 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3771 } 3772 3773 #endif 3774 3775 /* if we allocated a copy buffer, free it */ 3776 if (dma->dp_cbaddr != NULL) { 3777 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3778 } 3779 } 3780 3781 3782 /* 3783 * rootnex_teardown_windows() 3784 * cleans up after rootnex_setup_windows() 3785 */ 3786 static void 3787 rootnex_teardown_windows(rootnex_dma_t *dma) 3788 { 3789 /* 3790 * if we had to allocate window state on the last bind (because we 3791 * didn't have enough pre-allocated space in the handle), free it. 3792 */ 3793 if (dma->dp_need_to_free_window) { 3794 kmem_free(dma->dp_window, dma->dp_window_size); 3795 } 3796 } 3797 3798 3799 /* 3800 * rootnex_init_win() 3801 * Called in bind slow path during creation of a new window. Initializes 3802 * window state to default values. 3803 */ 3804 /*ARGSUSED*/ 3805 static void 3806 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3807 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3808 { 3809 hp->dmai_nwin++; 3810 window->wd_dosync = B_FALSE; 3811 window->wd_offset = cur_offset; 3812 window->wd_size = 0; 3813 window->wd_first_cookie = cookie; 3814 window->wd_cookie_cnt = 0; 3815 window->wd_trim.tr_trim_first = B_FALSE; 3816 window->wd_trim.tr_trim_last = B_FALSE; 3817 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3818 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3819 #if !defined(__amd64) 3820 window->wd_remap_copybuf = dma->dp_cb_remaping; 3821 #endif 3822 } 3823 3824 3825 /* 3826 * rootnex_setup_cookie() 3827 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3828 * the sgl uses the copy buffer, we need to go through each cookie, figure 3829 * out if it uses the copy buffer, and if it does, save away everything we'll 3830 * need during sync. 3831 */ 3832 static void 3833 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3834 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3835 page_t **cur_pp) 3836 { 3837 boolean_t copybuf_sz_power_2; 3838 rootnex_sglinfo_t *sinfo; 3839 paddr_t paddr; 3840 uint_t pidx; 3841 uint_t pcnt; 3842 off_t poff; 3843 #if defined(__amd64) 3844 pfn_t pfn; 3845 #else 3846 page_t **pplist; 3847 #endif 3848 3849 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3850 3851 sinfo = &dma->dp_sglinfo; 3852 3853 /* 3854 * Calculate the page index relative to the start of the buffer. The 3855 * index to the current page for our buffer is the offset into the 3856 * first page of the buffer plus our current offset into the buffer 3857 * itself, shifted of course... 3858 */ 3859 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3860 ASSERT(pidx < sinfo->si_max_pages); 3861 3862 /* if this cookie uses the copy buffer */ 3863 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3864 /* 3865 * NOTE: we know that since this cookie uses the copy buffer, it 3866 * is <= MMU_PAGESIZE. 3867 */ 3868 3869 /* 3870 * get the offset into the page. For the 64-bit kernel, get the 3871 * pfn which we'll use with seg kpm. 3872 */ 3873 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3874 #if defined(__amd64) 3875 /* mfn_to_pfn() is a NOP on i86pc */ 3876 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3877 #endif /* __amd64 */ 3878 3879 /* figure out if the copybuf size is a power of 2 */ 3880 if (!ISP2(dma->dp_copybuf_size)) { 3881 copybuf_sz_power_2 = B_FALSE; 3882 } else { 3883 copybuf_sz_power_2 = B_TRUE; 3884 } 3885 3886 /* This page uses the copy buffer */ 3887 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3888 3889 /* 3890 * save the copy buffer KVA that we'll use with this page. 3891 * if we still fit within the copybuf, it's a simple add. 3892 * otherwise, we need to wrap over using & or % accordingly. 3893 */ 3894 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3895 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3896 *copybuf_used; 3897 } else { 3898 if (copybuf_sz_power_2) { 3899 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3900 (uintptr_t)dma->dp_cbaddr + 3901 (*copybuf_used & 3902 (dma->dp_copybuf_size - 1))); 3903 } else { 3904 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3905 (uintptr_t)dma->dp_cbaddr + 3906 (*copybuf_used % dma->dp_copybuf_size)); 3907 } 3908 } 3909 3910 /* 3911 * over write the cookie physical address with the address of 3912 * the physical address of the copy buffer page that we will 3913 * use. 3914 */ 3915 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3916 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3917 3918 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3919 3920 /* if we have a kernel VA, it's easy, just save that address */ 3921 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3922 (sinfo->si_asp == &kas)) { 3923 /* 3924 * save away the page aligned virtual address of the 3925 * driver buffer. Offsets are handled in the sync code. 3926 */ 3927 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3928 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3929 & MMU_PAGEMASK); 3930 #if !defined(__amd64) 3931 /* 3932 * we didn't need to, and will never need to map this 3933 * page. 3934 */ 3935 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3936 #endif 3937 3938 /* we don't have a kernel VA. We need one for the bcopy. */ 3939 } else { 3940 #if defined(__amd64) 3941 /* 3942 * for the 64-bit kernel, it's easy. We use seg kpm to 3943 * get a Kernel VA for the corresponding pfn. 3944 */ 3945 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3946 #else 3947 /* 3948 * for the 32-bit kernel, this is a pain. First we'll 3949 * save away the page_t or user VA for this page. This 3950 * is needed in rootnex_dma_win() when we switch to a 3951 * new window which requires us to re-map the copy 3952 * buffer. 3953 */ 3954 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3955 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3956 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3957 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3958 } else if (pplist != NULL) { 3959 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3960 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3961 } else { 3962 dma->dp_pgmap[pidx].pm_pp = NULL; 3963 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3964 (((uintptr_t) 3965 dmar_object->dmao_obj.virt_obj.v_addr + 3966 cur_offset) & MMU_PAGEMASK); 3967 } 3968 3969 /* 3970 * save away the page aligned virtual address which was 3971 * allocated from the kernel heap arena (taking into 3972 * account if we need more copy buffer than we alloced 3973 * and use multiple windows to handle this, i.e. &,%). 3974 * NOTE: there isn't and physical memory backing up this 3975 * virtual address space currently. 3976 */ 3977 if ((*copybuf_used + MMU_PAGESIZE) <= 3978 dma->dp_copybuf_size) { 3979 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3980 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3981 MMU_PAGEMASK); 3982 } else { 3983 if (copybuf_sz_power_2) { 3984 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3985 (((uintptr_t)dma->dp_kva + 3986 (*copybuf_used & 3987 (dma->dp_copybuf_size - 1))) & 3988 MMU_PAGEMASK); 3989 } else { 3990 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3991 (((uintptr_t)dma->dp_kva + 3992 (*copybuf_used % 3993 dma->dp_copybuf_size)) & 3994 MMU_PAGEMASK); 3995 } 3996 } 3997 3998 /* 3999 * if we haven't used up the available copy buffer yet, 4000 * map the kva to the physical page. 4001 */ 4002 if (!dma->dp_cb_remaping && ((*copybuf_used + 4003 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 4004 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 4005 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 4006 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 4007 dma->dp_pgmap[pidx].pm_kaddr); 4008 } else { 4009 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4010 sinfo->si_asp, 4011 dma->dp_pgmap[pidx].pm_kaddr); 4012 } 4013 4014 /* 4015 * we've used up the available copy buffer, this page 4016 * will have to be mapped during rootnex_dma_win() when 4017 * we switch to a new window which requires a re-map 4018 * the copy buffer. (32-bit kernel only) 4019 */ 4020 } else { 4021 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4022 } 4023 #endif 4024 /* go to the next page_t */ 4025 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4026 *cur_pp = (*cur_pp)->p_next; 4027 } 4028 } 4029 4030 /* add to the copy buffer count */ 4031 *copybuf_used += MMU_PAGESIZE; 4032 4033 /* 4034 * This cookie doesn't use the copy buffer. Walk through the pages this 4035 * cookie occupies to reflect this. 4036 */ 4037 } else { 4038 /* 4039 * figure out how many pages the cookie occupies. We need to 4040 * use the original page offset of the buffer and the cookies 4041 * offset in the buffer to do this. 4042 */ 4043 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4044 pcnt = mmu_btopr(cookie->dmac_size + poff); 4045 4046 while (pcnt > 0) { 4047 #if !defined(__amd64) 4048 /* 4049 * the 32-bit kernel doesn't have seg kpm, so we need 4050 * to map in the driver buffer (if it didn't come down 4051 * with a kernel VA) on the fly. Since this page doesn't 4052 * use the copy buffer, it's not, or will it ever, have 4053 * to be mapped in. 4054 */ 4055 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4056 #endif 4057 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4058 4059 /* 4060 * we need to update pidx and cur_pp or we'll loose 4061 * track of where we are. 4062 */ 4063 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4064 *cur_pp = (*cur_pp)->p_next; 4065 } 4066 pidx++; 4067 pcnt--; 4068 } 4069 } 4070 } 4071 4072 4073 /* 4074 * rootnex_sgllen_window_boundary() 4075 * Called in the bind slow path when the next cookie causes us to exceed (in 4076 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4077 * length supported by the DMA H/W. 4078 */ 4079 static int 4080 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4081 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4082 off_t cur_offset) 4083 { 4084 off_t new_offset; 4085 size_t trim_sz; 4086 off_t coffset; 4087 4088 4089 /* 4090 * if we know we'll never have to trim, it's pretty easy. Just move to 4091 * the next window and init it. We're done. 4092 */ 4093 if (!dma->dp_trim_required) { 4094 (*windowp)++; 4095 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4096 (*windowp)->wd_cookie_cnt++; 4097 (*windowp)->wd_size = cookie->dmac_size; 4098 return (DDI_SUCCESS); 4099 } 4100 4101 /* figure out how much we need to trim from the window */ 4102 ASSERT(attr->dma_attr_granular != 0); 4103 if (dma->dp_granularity_power_2) { 4104 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4105 } else { 4106 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4107 } 4108 4109 /* The window's a whole multiple of granularity. We're done */ 4110 if (trim_sz == 0) { 4111 (*windowp)++; 4112 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4113 (*windowp)->wd_cookie_cnt++; 4114 (*windowp)->wd_size = cookie->dmac_size; 4115 return (DDI_SUCCESS); 4116 } 4117 4118 /* 4119 * The window's not a whole multiple of granularity, since we know this 4120 * is due to the sgllen, we need to go back to the last cookie and trim 4121 * that one, add the left over part of the old cookie into the new 4122 * window, and then add in the new cookie into the new window. 4123 */ 4124 4125 /* 4126 * make sure the driver isn't making us do something bad... Trimming and 4127 * sgllen == 1 don't go together. 4128 */ 4129 if (attr->dma_attr_sgllen == 1) { 4130 return (DDI_DMA_NOMAPPING); 4131 } 4132 4133 /* 4134 * first, setup the current window to account for the trim. Need to go 4135 * back to the last cookie for this. 4136 */ 4137 cookie--; 4138 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4139 (*windowp)->wd_trim.tr_last_cookie = cookie; 4140 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4141 ASSERT(cookie->dmac_size > trim_sz); 4142 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4143 (*windowp)->wd_size -= trim_sz; 4144 4145 /* save the buffer offsets for the next window */ 4146 coffset = cookie->dmac_size - trim_sz; 4147 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4148 4149 /* 4150 * set this now in case this is the first window. all other cases are 4151 * set in dma_win() 4152 */ 4153 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4154 4155 /* 4156 * initialize the next window using what's left over in the previous 4157 * cookie. 4158 */ 4159 (*windowp)++; 4160 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4161 (*windowp)->wd_cookie_cnt++; 4162 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4163 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4164 (*windowp)->wd_trim.tr_first_size = trim_sz; 4165 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4166 (*windowp)->wd_dosync = B_TRUE; 4167 } 4168 4169 /* 4170 * now go back to the current cookie and add it to the new window. set 4171 * the new window size to the what was left over from the previous 4172 * cookie and what's in the current cookie. 4173 */ 4174 cookie++; 4175 (*windowp)->wd_cookie_cnt++; 4176 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4177 4178 /* 4179 * trim plus the next cookie could put us over maxxfer (a cookie can be 4180 * a max size of maxxfer). Handle that case. 4181 */ 4182 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4183 /* 4184 * maxxfer is already a whole multiple of granularity, and this 4185 * trim will be <= the previous trim (since a cookie can't be 4186 * larger than maxxfer). Make things simple here. 4187 */ 4188 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4189 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4190 (*windowp)->wd_trim.tr_last_cookie = cookie; 4191 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4192 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4193 (*windowp)->wd_size -= trim_sz; 4194 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4195 4196 /* save the buffer offsets for the next window */ 4197 coffset = cookie->dmac_size - trim_sz; 4198 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4199 4200 /* setup the next window */ 4201 (*windowp)++; 4202 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4203 (*windowp)->wd_cookie_cnt++; 4204 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4205 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4206 coffset; 4207 (*windowp)->wd_trim.tr_first_size = trim_sz; 4208 } 4209 4210 return (DDI_SUCCESS); 4211 } 4212 4213 4214 /* 4215 * rootnex_copybuf_window_boundary() 4216 * Called in bind slowpath when we get to a window boundary because we used 4217 * up all the copy buffer that we have. 4218 */ 4219 static int 4220 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4221 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4222 size_t *copybuf_used) 4223 { 4224 rootnex_sglinfo_t *sinfo; 4225 off_t new_offset; 4226 size_t trim_sz; 4227 paddr_t paddr; 4228 off_t coffset; 4229 uint_t pidx; 4230 off_t poff; 4231 4232 pidx = 0; 4233 sinfo = &dma->dp_sglinfo; 4234 4235 /* 4236 * the copy buffer should be a whole multiple of page size. We know that 4237 * this cookie is <= MMU_PAGESIZE. 4238 */ 4239 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4240 4241 /* 4242 * from now on, all new windows in this bind need to be re-mapped during 4243 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4244 * space... 4245 */ 4246 #if !defined(__amd64) 4247 dma->dp_cb_remaping = B_TRUE; 4248 #endif 4249 4250 /* reset copybuf used */ 4251 *copybuf_used = 0; 4252 4253 /* 4254 * if we don't have to trim (since granularity is set to 1), go to the 4255 * next window and add the current cookie to it. We know the current 4256 * cookie uses the copy buffer since we're in this code path. 4257 */ 4258 if (!dma->dp_trim_required) { 4259 (*windowp)++; 4260 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4261 4262 /* Add this cookie to the new window */ 4263 (*windowp)->wd_cookie_cnt++; 4264 (*windowp)->wd_size += cookie->dmac_size; 4265 *copybuf_used += MMU_PAGESIZE; 4266 return (DDI_SUCCESS); 4267 } 4268 4269 /* 4270 * *** may need to trim, figure it out. 4271 */ 4272 4273 /* figure out how much we need to trim from the window */ 4274 if (dma->dp_granularity_power_2) { 4275 trim_sz = (*windowp)->wd_size & 4276 (hp->dmai_attr.dma_attr_granular - 1); 4277 } else { 4278 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4279 } 4280 4281 /* 4282 * if the window's a whole multiple of granularity, go to the next 4283 * window, init it, then add in the current cookie. We know the current 4284 * cookie uses the copy buffer since we're in this code path. 4285 */ 4286 if (trim_sz == 0) { 4287 (*windowp)++; 4288 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4289 4290 /* Add this cookie to the new window */ 4291 (*windowp)->wd_cookie_cnt++; 4292 (*windowp)->wd_size += cookie->dmac_size; 4293 *copybuf_used += MMU_PAGESIZE; 4294 return (DDI_SUCCESS); 4295 } 4296 4297 /* 4298 * *** We figured it out, we definitly need to trim 4299 */ 4300 4301 /* 4302 * make sure the driver isn't making us do something bad... 4303 * Trimming and sgllen == 1 don't go together. 4304 */ 4305 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4306 return (DDI_DMA_NOMAPPING); 4307 } 4308 4309 /* 4310 * first, setup the current window to account for the trim. Need to go 4311 * back to the last cookie for this. Some of the last cookie will be in 4312 * the current window, and some of the last cookie will be in the new 4313 * window. All of the current cookie will be in the new window. 4314 */ 4315 cookie--; 4316 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4317 (*windowp)->wd_trim.tr_last_cookie = cookie; 4318 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4319 ASSERT(cookie->dmac_size > trim_sz); 4320 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4321 (*windowp)->wd_size -= trim_sz; 4322 4323 /* 4324 * we're trimming the last cookie (not the current cookie). So that 4325 * last cookie may have or may not have been using the copy buffer ( 4326 * we know the cookie passed in uses the copy buffer since we're in 4327 * this code path). 4328 * 4329 * If the last cookie doesn't use the copy buffer, nothing special to 4330 * do. However, if it does uses the copy buffer, it will be both the 4331 * last page in the current window and the first page in the next 4332 * window. Since we are reusing the copy buffer (and KVA space on the 4333 * 32-bit kernel), this page will use the end of the copy buffer in the 4334 * current window, and the start of the copy buffer in the next window. 4335 * Track that info... The cookie physical address was already set to 4336 * the copy buffer physical address in setup_cookie.. 4337 */ 4338 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4339 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4340 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4341 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4342 (*windowp)->wd_trim.tr_last_pidx = pidx; 4343 (*windowp)->wd_trim.tr_last_cbaddr = 4344 dma->dp_pgmap[pidx].pm_cbaddr; 4345 #if !defined(__amd64) 4346 (*windowp)->wd_trim.tr_last_kaddr = 4347 dma->dp_pgmap[pidx].pm_kaddr; 4348 #endif 4349 } 4350 4351 /* save the buffer offsets for the next window */ 4352 coffset = cookie->dmac_size - trim_sz; 4353 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4354 4355 /* 4356 * set this now in case this is the first window. all other cases are 4357 * set in dma_win() 4358 */ 4359 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4360 4361 /* 4362 * initialize the next window using what's left over in the previous 4363 * cookie. 4364 */ 4365 (*windowp)++; 4366 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4367 (*windowp)->wd_cookie_cnt++; 4368 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4369 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4370 (*windowp)->wd_trim.tr_first_size = trim_sz; 4371 4372 /* 4373 * again, we're tracking if the last cookie uses the copy buffer. 4374 * read the comment above for more info on why we need to track 4375 * additional state. 4376 * 4377 * For the first cookie in the new window, we need reset the physical 4378 * address to DMA into to the start of the copy buffer plus any 4379 * initial page offset which may be present. 4380 */ 4381 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4382 (*windowp)->wd_dosync = B_TRUE; 4383 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4384 (*windowp)->wd_trim.tr_first_pidx = pidx; 4385 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4386 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4387 4388 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4389 poff; 4390 (*windowp)->wd_trim.tr_first_paddr = 4391 ROOTNEX_PADDR_TO_RBASE(paddr); 4392 4393 #if !defined(__amd64) 4394 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4395 #endif 4396 /* account for the cookie copybuf usage in the new window */ 4397 *copybuf_used += MMU_PAGESIZE; 4398 4399 /* 4400 * every piece of code has to have a hack, and here is this 4401 * ones :-) 4402 * 4403 * There is a complex interaction between setup_cookie and the 4404 * copybuf window boundary. The complexity had to be in either 4405 * the maxxfer window, or the copybuf window, and I chose the 4406 * copybuf code. 4407 * 4408 * So in this code path, we have taken the last cookie, 4409 * virtually broken it in half due to the trim, and it happens 4410 * to use the copybuf which further complicates life. At the 4411 * same time, we have already setup the current cookie, which 4412 * is now wrong. More background info: the current cookie uses 4413 * the copybuf, so it is only a page long max. So we need to 4414 * fix the current cookies copy buffer address, physical 4415 * address, and kva for the 32-bit kernel. We due this by 4416 * bumping them by page size (of course, we can't due this on 4417 * the physical address since the copy buffer may not be 4418 * physically contiguous). 4419 */ 4420 cookie++; 4421 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4422 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4423 4424 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4425 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4426 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4427 4428 #if !defined(__amd64) 4429 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4430 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4431 #endif 4432 } else { 4433 /* go back to the current cookie */ 4434 cookie++; 4435 } 4436 4437 /* 4438 * add the current cookie to the new window. set the new window size to 4439 * the what was left over from the previous cookie and what's in the 4440 * current cookie. 4441 */ 4442 (*windowp)->wd_cookie_cnt++; 4443 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4444 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4445 4446 /* 4447 * we know that the cookie passed in always uses the copy buffer. We 4448 * wouldn't be here if it didn't. 4449 */ 4450 *copybuf_used += MMU_PAGESIZE; 4451 4452 return (DDI_SUCCESS); 4453 } 4454 4455 4456 /* 4457 * rootnex_maxxfer_window_boundary() 4458 * Called in bind slowpath when we get to a window boundary because we will 4459 * go over maxxfer. 4460 */ 4461 static int 4462 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4463 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4464 { 4465 size_t dmac_size; 4466 off_t new_offset; 4467 size_t trim_sz; 4468 off_t coffset; 4469 4470 4471 /* 4472 * calculate how much we have to trim off of the current cookie to equal 4473 * maxxfer. We don't have to account for granularity here since our 4474 * maxxfer already takes that into account. 4475 */ 4476 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4477 ASSERT(trim_sz <= cookie->dmac_size); 4478 ASSERT(trim_sz <= dma->dp_maxxfer); 4479 4480 /* save cookie size since we need it later and we might change it */ 4481 dmac_size = cookie->dmac_size; 4482 4483 /* 4484 * if we're not trimming the entire cookie, setup the current window to 4485 * account for the trim. 4486 */ 4487 if (trim_sz < cookie->dmac_size) { 4488 (*windowp)->wd_cookie_cnt++; 4489 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4490 (*windowp)->wd_trim.tr_last_cookie = cookie; 4491 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4492 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4493 (*windowp)->wd_size = dma->dp_maxxfer; 4494 4495 /* 4496 * set the adjusted cookie size now in case this is the first 4497 * window. All other windows are taken care of in get win 4498 */ 4499 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4500 } 4501 4502 /* 4503 * coffset is the current offset within the cookie, new_offset is the 4504 * current offset with the entire buffer. 4505 */ 4506 coffset = dmac_size - trim_sz; 4507 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4508 4509 /* initialize the next window */ 4510 (*windowp)++; 4511 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4512 (*windowp)->wd_cookie_cnt++; 4513 (*windowp)->wd_size = trim_sz; 4514 if (trim_sz < dmac_size) { 4515 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4516 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4517 coffset; 4518 (*windowp)->wd_trim.tr_first_size = trim_sz; 4519 } 4520 4521 return (DDI_SUCCESS); 4522 } 4523 4524 4525 /*ARGSUSED*/ 4526 static int 4527 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4528 off_t off, size_t len, uint_t cache_flags) 4529 { 4530 rootnex_sglinfo_t *sinfo; 4531 rootnex_pgmap_t *cbpage; 4532 rootnex_window_t *win; 4533 ddi_dma_impl_t *hp; 4534 rootnex_dma_t *dma; 4535 caddr_t fromaddr; 4536 caddr_t toaddr; 4537 uint_t psize; 4538 off_t offset; 4539 uint_t pidx; 4540 size_t size; 4541 off_t poff; 4542 int e; 4543 4544 4545 hp = (ddi_dma_impl_t *)handle; 4546 dma = (rootnex_dma_t *)hp->dmai_private; 4547 sinfo = &dma->dp_sglinfo; 4548 4549 /* 4550 * if we don't have any windows, we don't need to sync. A copybuf 4551 * will cause us to have at least one window. 4552 */ 4553 if (dma->dp_window == NULL) { 4554 return (DDI_SUCCESS); 4555 } 4556 4557 /* This window may not need to be sync'd */ 4558 win = &dma->dp_window[dma->dp_current_win]; 4559 if (!win->wd_dosync) { 4560 return (DDI_SUCCESS); 4561 } 4562 4563 /* handle off and len special cases */ 4564 if ((off == 0) || (rootnex_sync_ignore_params)) { 4565 offset = win->wd_offset; 4566 } else { 4567 offset = off; 4568 } 4569 if ((len == 0) || (rootnex_sync_ignore_params)) { 4570 size = win->wd_size; 4571 } else { 4572 size = len; 4573 } 4574 4575 /* check the sync args to make sure they make a little sense */ 4576 if (rootnex_sync_check_parms) { 4577 e = rootnex_valid_sync_parms(hp, win, offset, size, 4578 cache_flags); 4579 if (e != DDI_SUCCESS) { 4580 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4581 return (DDI_FAILURE); 4582 } 4583 } 4584 4585 /* 4586 * special case the first page to handle the offset into the page. The 4587 * offset to the current page for our buffer is the offset into the 4588 * first page of the buffer plus our current offset into the buffer 4589 * itself, masked of course. 4590 */ 4591 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4592 psize = MIN((MMU_PAGESIZE - poff), size); 4593 4594 /* go through all the pages that we want to sync */ 4595 while (size > 0) { 4596 /* 4597 * Calculate the page index relative to the start of the buffer. 4598 * The index to the current page for our buffer is the offset 4599 * into the first page of the buffer plus our current offset 4600 * into the buffer itself, shifted of course... 4601 */ 4602 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4603 ASSERT(pidx < sinfo->si_max_pages); 4604 4605 /* 4606 * if this page uses the copy buffer, we need to sync it, 4607 * otherwise, go on to the next page. 4608 */ 4609 cbpage = &dma->dp_pgmap[pidx]; 4610 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4611 (cbpage->pm_uses_copybuf == B_FALSE)); 4612 if (cbpage->pm_uses_copybuf) { 4613 /* cbaddr and kaddr should be page aligned */ 4614 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4615 MMU_PAGEOFFSET) == 0); 4616 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4617 MMU_PAGEOFFSET) == 0); 4618 4619 /* 4620 * if we're copying for the device, we are going to 4621 * copy from the drivers buffer and to the rootnex 4622 * allocated copy buffer. 4623 */ 4624 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4625 fromaddr = cbpage->pm_kaddr + poff; 4626 toaddr = cbpage->pm_cbaddr + poff; 4627 ROOTNEX_DPROBE2(rootnex__sync__dev, 4628 dev_info_t *, dma->dp_dip, size_t, psize); 4629 4630 /* 4631 * if we're copying for the cpu/kernel, we are going to 4632 * copy from the rootnex allocated copy buffer to the 4633 * drivers buffer. 4634 */ 4635 } else { 4636 fromaddr = cbpage->pm_cbaddr + poff; 4637 toaddr = cbpage->pm_kaddr + poff; 4638 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4639 dev_info_t *, dma->dp_dip, size_t, psize); 4640 } 4641 4642 bcopy(fromaddr, toaddr, psize); 4643 } 4644 4645 /* 4646 * decrement size until we're done, update our offset into the 4647 * buffer, and get the next page size. 4648 */ 4649 size -= psize; 4650 offset += psize; 4651 psize = MIN(MMU_PAGESIZE, size); 4652 4653 /* page offset is zero for the rest of this loop */ 4654 poff = 0; 4655 } 4656 4657 return (DDI_SUCCESS); 4658 } 4659 4660 /* 4661 * rootnex_dma_sync() 4662 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4663 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4664 * is set, ddi_dma_sync() returns immediately passing back success. 4665 */ 4666 /*ARGSUSED*/ 4667 static int 4668 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4669 off_t off, size_t len, uint_t cache_flags) 4670 { 4671 #if defined(__amd64) && !defined(__xpv) 4672 if (IOMMU_USED(rdip)) { 4673 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4674 cache_flags)); 4675 } 4676 #endif 4677 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4678 cache_flags)); 4679 } 4680 4681 /* 4682 * rootnex_valid_sync_parms() 4683 * checks the parameters passed to sync to verify they are correct. 4684 */ 4685 static int 4686 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4687 off_t offset, size_t size, uint_t cache_flags) 4688 { 4689 off_t woffset; 4690 4691 4692 /* 4693 * the first part of the test to make sure the offset passed in is 4694 * within the window. 4695 */ 4696 if (offset < win->wd_offset) { 4697 return (DDI_FAILURE); 4698 } 4699 4700 /* 4701 * second and last part of the test to make sure the offset and length 4702 * passed in is within the window. 4703 */ 4704 woffset = offset - win->wd_offset; 4705 if ((woffset + size) > win->wd_size) { 4706 return (DDI_FAILURE); 4707 } 4708 4709 /* 4710 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4711 * be set too. 4712 */ 4713 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4714 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4715 return (DDI_SUCCESS); 4716 } 4717 4718 /* 4719 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4720 * should be set. Also DDI_DMA_READ should be set in the flags. 4721 */ 4722 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4723 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4724 (hp->dmai_rflags & DDI_DMA_READ)) { 4725 return (DDI_SUCCESS); 4726 } 4727 4728 return (DDI_FAILURE); 4729 } 4730 4731 4732 /*ARGSUSED*/ 4733 static int 4734 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4735 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4736 uint_t *ccountp) 4737 { 4738 rootnex_window_t *window; 4739 rootnex_trim_t *trim; 4740 ddi_dma_impl_t *hp; 4741 rootnex_dma_t *dma; 4742 ddi_dma_obj_t *dmao; 4743 #if !defined(__amd64) 4744 rootnex_sglinfo_t *sinfo; 4745 rootnex_pgmap_t *pmap; 4746 uint_t pidx; 4747 uint_t pcnt; 4748 off_t poff; 4749 int i; 4750 #endif 4751 4752 4753 hp = (ddi_dma_impl_t *)handle; 4754 dma = (rootnex_dma_t *)hp->dmai_private; 4755 #if !defined(__amd64) 4756 sinfo = &dma->dp_sglinfo; 4757 #endif 4758 4759 /* If we try and get a window which doesn't exist, return failure */ 4760 if (win >= hp->dmai_nwin) { 4761 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4762 return (DDI_FAILURE); 4763 } 4764 4765 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4766 4767 /* 4768 * if we don't have any windows, and they're asking for the first 4769 * window, setup the cookie pointer to the first cookie in the bind. 4770 * setup our return values, then increment the cookie since we return 4771 * the first cookie on the stack. 4772 */ 4773 if (dma->dp_window == NULL) { 4774 if (win != 0) { 4775 ROOTNEX_DPROF_INC( 4776 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4777 return (DDI_FAILURE); 4778 } 4779 hp->dmai_cookie = dma->dp_cookies; 4780 *offp = 0; 4781 *lenp = dmao->dmao_size; 4782 *ccountp = dma->dp_sglinfo.si_sgl_size; 4783 *cookiep = hp->dmai_cookie[0]; 4784 hp->dmai_cookie++; 4785 hp->dmai_ncookies = *ccountp; 4786 hp->dmai_curcookie = 1; 4787 return (DDI_SUCCESS); 4788 } 4789 4790 /* sync the old window before moving on to the new one */ 4791 window = &dma->dp_window[dma->dp_current_win]; 4792 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4793 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4794 DDI_DMA_SYNC_FORCPU); 4795 } 4796 4797 #if !defined(__amd64) 4798 /* 4799 * before we move to the next window, if we need to re-map, unmap all 4800 * the pages in this window. 4801 */ 4802 if (dma->dp_cb_remaping) { 4803 /* 4804 * If we switch to this window again, we'll need to map in 4805 * on the fly next time. 4806 */ 4807 window->wd_remap_copybuf = B_TRUE; 4808 4809 /* 4810 * calculate the page index into the buffer where this window 4811 * starts, and the number of pages this window takes up. 4812 */ 4813 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4814 MMU_PAGESHIFT; 4815 poff = (sinfo->si_buf_offset + window->wd_offset) & 4816 MMU_PAGEOFFSET; 4817 pcnt = mmu_btopr(window->wd_size + poff); 4818 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4819 4820 /* unmap pages which are currently mapped in this window */ 4821 for (i = 0; i < pcnt; i++) { 4822 if (dma->dp_pgmap[pidx].pm_mapped) { 4823 hat_unload(kas.a_hat, 4824 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4825 HAT_UNLOAD); 4826 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4827 } 4828 pidx++; 4829 } 4830 } 4831 #endif 4832 4833 /* 4834 * Move to the new window. 4835 * NOTE: current_win must be set for sync to work right 4836 */ 4837 dma->dp_current_win = win; 4838 window = &dma->dp_window[win]; 4839 4840 /* if needed, adjust the first and/or last cookies for trim */ 4841 trim = &window->wd_trim; 4842 if (trim->tr_trim_first) { 4843 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4844 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4845 #if !defined(__amd64) 4846 window->wd_first_cookie->dmac_type = 4847 (window->wd_first_cookie->dmac_type & 4848 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4849 #endif 4850 if (trim->tr_first_copybuf_win) { 4851 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4852 trim->tr_first_cbaddr; 4853 #if !defined(__amd64) 4854 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4855 trim->tr_first_kaddr; 4856 #endif 4857 } 4858 } 4859 if (trim->tr_trim_last) { 4860 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4861 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4862 if (trim->tr_last_copybuf_win) { 4863 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4864 trim->tr_last_cbaddr; 4865 #if !defined(__amd64) 4866 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4867 trim->tr_last_kaddr; 4868 #endif 4869 } 4870 } 4871 4872 /* 4873 * setup the cookie pointer to the first cookie in the window. setup 4874 * our return values, then increment the cookie since we return the 4875 * first cookie on the stack. 4876 */ 4877 hp->dmai_cookie = window->wd_first_cookie; 4878 *offp = window->wd_offset; 4879 *lenp = window->wd_size; 4880 *ccountp = window->wd_cookie_cnt; 4881 *cookiep = hp->dmai_cookie[0]; 4882 hp->dmai_ncookies = *ccountp; 4883 hp->dmai_curcookie = 1; 4884 hp->dmai_cookie++; 4885 4886 #if !defined(__amd64) 4887 /* re-map copybuf if required for this window */ 4888 if (dma->dp_cb_remaping) { 4889 /* 4890 * calculate the page index into the buffer where this 4891 * window starts. 4892 */ 4893 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4894 MMU_PAGESHIFT; 4895 ASSERT(pidx < sinfo->si_max_pages); 4896 4897 /* 4898 * the first page can get unmapped if it's shared with the 4899 * previous window. Even if the rest of this window is already 4900 * mapped in, we need to still check this one. 4901 */ 4902 pmap = &dma->dp_pgmap[pidx]; 4903 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4904 if (pmap->pm_pp != NULL) { 4905 pmap->pm_mapped = B_TRUE; 4906 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4907 } else if (pmap->pm_vaddr != NULL) { 4908 pmap->pm_mapped = B_TRUE; 4909 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4910 pmap->pm_kaddr); 4911 } 4912 } 4913 pidx++; 4914 4915 /* map in the rest of the pages if required */ 4916 if (window->wd_remap_copybuf) { 4917 window->wd_remap_copybuf = B_FALSE; 4918 4919 /* figure out many pages this window takes up */ 4920 poff = (sinfo->si_buf_offset + window->wd_offset) & 4921 MMU_PAGEOFFSET; 4922 pcnt = mmu_btopr(window->wd_size + poff); 4923 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4924 4925 /* map pages which require it */ 4926 for (i = 1; i < pcnt; i++) { 4927 pmap = &dma->dp_pgmap[pidx]; 4928 if (pmap->pm_uses_copybuf) { 4929 ASSERT(pmap->pm_mapped == B_FALSE); 4930 if (pmap->pm_pp != NULL) { 4931 pmap->pm_mapped = B_TRUE; 4932 i86_pp_map(pmap->pm_pp, 4933 pmap->pm_kaddr); 4934 } else if (pmap->pm_vaddr != NULL) { 4935 pmap->pm_mapped = B_TRUE; 4936 i86_va_map(pmap->pm_vaddr, 4937 sinfo->si_asp, 4938 pmap->pm_kaddr); 4939 } 4940 } 4941 pidx++; 4942 } 4943 } 4944 } 4945 #endif 4946 4947 /* if the new window uses the copy buffer, sync it for the device */ 4948 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4949 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4950 DDI_DMA_SYNC_FORDEV); 4951 } 4952 4953 return (DDI_SUCCESS); 4954 } 4955 4956 /* 4957 * rootnex_dma_win() 4958 * called from ddi_dma_getwin() 4959 */ 4960 /*ARGSUSED*/ 4961 static int 4962 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4963 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4964 uint_t *ccountp) 4965 { 4966 #if defined(__amd64) && !defined(__xpv) 4967 if (IOMMU_USED(rdip)) { 4968 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4969 cookiep, ccountp)); 4970 } 4971 #endif 4972 4973 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4974 cookiep, ccountp)); 4975 } 4976 4977 #if defined(__amd64) && !defined(__xpv) 4978 /*ARGSUSED*/ 4979 static int 4980 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4981 ddi_dma_handle_t handle, void *v) 4982 { 4983 ddi_dma_impl_t *hp; 4984 rootnex_dma_t *dma; 4985 4986 hp = (ddi_dma_impl_t *)handle; 4987 dma = (rootnex_dma_t *)hp->dmai_private; 4988 dma->dp_iommu_private = v; 4989 4990 return (DDI_SUCCESS); 4991 } 4992 4993 /*ARGSUSED*/ 4994 static void * 4995 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4996 ddi_dma_handle_t handle) 4997 { 4998 ddi_dma_impl_t *hp; 4999 rootnex_dma_t *dma; 5000 5001 hp = (ddi_dma_impl_t *)handle; 5002 dma = (rootnex_dma_t *)hp->dmai_private; 5003 5004 return (dma->dp_iommu_private); 5005 } 5006 #endif 5007 5008 /* 5009 * ************************ 5010 * obsoleted dma routines 5011 * ************************ 5012 */ 5013 5014 /* 5015 * rootnex_dma_mctl() 5016 * 5017 * We don't support this legacy interface any more on x86. 5018 */ 5019 /* ARGSUSED */ 5020 static int 5021 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5022 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5023 uint_t cache_flags) 5024 { 5025 /* 5026 * The only thing dma_mctl is usef for anymore is legacy SPARC 5027 * dvma and sbus-specific routines. 5028 */ 5029 return (DDI_FAILURE); 5030 } 5031 5032 /* 5033 * ********* 5034 * FMA Code 5035 * ********* 5036 */ 5037 5038 /* 5039 * rootnex_fm_init() 5040 * FMA init busop 5041 */ 5042 /* ARGSUSED */ 5043 static int 5044 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5045 ddi_iblock_cookie_t *ibc) 5046 { 5047 *ibc = rootnex_state->r_err_ibc; 5048 5049 return (ddi_system_fmcap); 5050 } 5051 5052 /* 5053 * rootnex_dma_check() 5054 * Function called after a dma fault occurred to find out whether the 5055 * fault address is associated with a driver that is able to handle faults 5056 * and recover from faults. 5057 */ 5058 /* ARGSUSED */ 5059 static int 5060 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5061 const void *not_used) 5062 { 5063 rootnex_window_t *window; 5064 uint64_t start_addr; 5065 uint64_t fault_addr; 5066 ddi_dma_impl_t *hp; 5067 rootnex_dma_t *dma; 5068 uint64_t end_addr; 5069 size_t csize; 5070 int i; 5071 int j; 5072 5073 5074 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5075 hp = (ddi_dma_impl_t *)handle; 5076 ASSERT(hp); 5077 5078 dma = (rootnex_dma_t *)hp->dmai_private; 5079 5080 /* Get the address that we need to search for */ 5081 fault_addr = *(uint64_t *)addr; 5082 5083 /* 5084 * if we don't have any windows, we can just walk through all the 5085 * cookies. 5086 */ 5087 if (dma->dp_window == NULL) { 5088 /* for each cookie */ 5089 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5090 /* 5091 * if the faulted address is within the physical address 5092 * range of the cookie, return DDI_FM_NONFATAL. 5093 */ 5094 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5095 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5096 dma->dp_cookies[i].dmac_size))) { 5097 return (DDI_FM_NONFATAL); 5098 } 5099 } 5100 5101 /* fault_addr not within this DMA handle */ 5102 return (DDI_FM_UNKNOWN); 5103 } 5104 5105 /* we have mutiple windows, walk through each window */ 5106 for (i = 0; i < hp->dmai_nwin; i++) { 5107 window = &dma->dp_window[i]; 5108 5109 /* Go through all the cookies in the window */ 5110 for (j = 0; j < window->wd_cookie_cnt; j++) { 5111 5112 start_addr = window->wd_first_cookie[j].dmac_laddress; 5113 csize = window->wd_first_cookie[j].dmac_size; 5114 5115 /* 5116 * if we are trimming the first cookie in the window, 5117 * and this is the first cookie, adjust the start 5118 * address and size of the cookie to account for the 5119 * trim. 5120 */ 5121 if (window->wd_trim.tr_trim_first && (j == 0)) { 5122 start_addr = window->wd_trim.tr_first_paddr; 5123 csize = window->wd_trim.tr_first_size; 5124 } 5125 5126 /* 5127 * if we are trimming the last cookie in the window, 5128 * and this is the last cookie, adjust the start 5129 * address and size of the cookie to account for the 5130 * trim. 5131 */ 5132 if (window->wd_trim.tr_trim_last && 5133 (j == (window->wd_cookie_cnt - 1))) { 5134 start_addr = window->wd_trim.tr_last_paddr; 5135 csize = window->wd_trim.tr_last_size; 5136 } 5137 5138 end_addr = start_addr + csize; 5139 5140 /* 5141 * if the faulted address is within the physical 5142 * address of the cookie, return DDI_FM_NONFATAL. 5143 */ 5144 if ((fault_addr >= start_addr) && 5145 (fault_addr <= end_addr)) { 5146 return (DDI_FM_NONFATAL); 5147 } 5148 } 5149 } 5150 5151 /* fault_addr not within this DMA handle */ 5152 return (DDI_FM_UNKNOWN); 5153 } 5154 5155 /*ARGSUSED*/ 5156 static int 5157 rootnex_quiesce(dev_info_t *dip) 5158 { 5159 #if defined(__amd64) && !defined(__xpv) 5160 return (immu_quiesce()); 5161 #else 5162 return (DDI_SUCCESS); 5163 #endif 5164 } 5165 5166 #if defined(__xpv) 5167 void 5168 immu_init(void) 5169 { 5170 ; 5171 } 5172 5173 void 5174 immu_startup(void) 5175 { 5176 ; 5177 } 5178 /*ARGSUSED*/ 5179 void 5180 immu_physmem_update(uint64_t addr, uint64_t size) 5181 { 5182 ; 5183 } 5184 #endif 5185