1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 * Copyright 2017 Joyent, Inc. 29 */ 30 31 /* 32 * x86 root nexus driver 33 */ 34 35 #include <sys/sysmacros.h> 36 #include <sys/conf.h> 37 #include <sys/autoconf.h> 38 #include <sys/sysmacros.h> 39 #include <sys/debug.h> 40 #include <sys/psw.h> 41 #include <sys/ddidmareq.h> 42 #include <sys/promif.h> 43 #include <sys/devops.h> 44 #include <sys/kmem.h> 45 #include <sys/cmn_err.h> 46 #include <vm/seg.h> 47 #include <vm/seg_kmem.h> 48 #include <vm/seg_dev.h> 49 #include <sys/vmem.h> 50 #include <sys/mman.h> 51 #include <vm/hat.h> 52 #include <vm/as.h> 53 #include <vm/page.h> 54 #include <sys/avintr.h> 55 #include <sys/errno.h> 56 #include <sys/modctl.h> 57 #include <sys/ddi_impldefs.h> 58 #include <sys/sunddi.h> 59 #include <sys/sunndi.h> 60 #include <sys/mach_intr.h> 61 #include <sys/psm.h> 62 #include <sys/ontrap.h> 63 #include <sys/atomic.h> 64 #include <sys/sdt.h> 65 #include <sys/rootnex.h> 66 #include <vm/hat_i86.h> 67 #include <sys/ddifm.h> 68 #include <sys/ddi_isa.h> 69 #include <sys/apic.h> 70 71 #ifdef __xpv 72 #include <sys/bootinfo.h> 73 #include <sys/hypervisor.h> 74 #include <sys/bootconf.h> 75 #include <vm/kboot_mmu.h> 76 #endif 77 78 #if defined(__amd64) && !defined(__xpv) 79 #include <sys/immu.h> 80 #endif 81 82 83 /* 84 * enable/disable extra checking of function parameters. Useful for debugging 85 * drivers. 86 */ 87 #ifdef DEBUG 88 int rootnex_alloc_check_parms = 1; 89 int rootnex_bind_check_parms = 1; 90 int rootnex_bind_check_inuse = 1; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 1; 93 #else 94 int rootnex_alloc_check_parms = 0; 95 int rootnex_bind_check_parms = 0; 96 int rootnex_bind_check_inuse = 0; 97 int rootnex_unbind_verify_buffer = 0; 98 int rootnex_sync_check_parms = 0; 99 #endif 100 101 boolean_t rootnex_dmar_not_setup; 102 103 /* Master Abort and Target Abort panic flag */ 104 int rootnex_fm_ma_ta_panic_flag = 0; 105 106 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 107 int rootnex_bind_fail = 1; 108 int rootnex_bind_warn = 1; 109 uint8_t *rootnex_warn_list; 110 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 111 #define ROOTNEX_BIND_WARNING (0x1 << 0) 112 113 /* 114 * revert back to old broken behavior of always sync'ing entire copy buffer. 115 * This is useful if be have a buggy driver which doesn't correctly pass in 116 * the offset and size into ddi_dma_sync(). 117 */ 118 int rootnex_sync_ignore_params = 0; 119 120 /* 121 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 122 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 123 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 124 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 125 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 126 * (< 8K). We will still need to allocate the copy buffer during bind though 127 * (if we need one). These can only be modified in /etc/system before rootnex 128 * attach. 129 */ 130 #if defined(__amd64) 131 int rootnex_prealloc_cookies = 65; 132 int rootnex_prealloc_windows = 4; 133 int rootnex_prealloc_copybuf = 2; 134 #else 135 int rootnex_prealloc_cookies = 33; 136 int rootnex_prealloc_windows = 4; 137 int rootnex_prealloc_copybuf = 2; 138 #endif 139 140 /* driver global state */ 141 static rootnex_state_t *rootnex_state; 142 143 #ifdef DEBUG 144 /* shortcut to rootnex counters */ 145 static uint64_t *rootnex_cnt; 146 #endif 147 148 /* 149 * XXX - does x86 even need these or are they left over from the SPARC days? 150 */ 151 /* statically defined integer/boolean properties for the root node */ 152 static rootnex_intprop_t rootnex_intprp[] = { 153 { "PAGESIZE", PAGESIZE }, 154 { "MMU_PAGESIZE", MMU_PAGESIZE }, 155 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 156 { DDI_RELATIVE_ADDRESSING, 1 }, 157 }; 158 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 159 160 /* 161 * If we're dom0, we're using a real device so we need to load 162 * the cookies with MFNs instead of PFNs. 163 */ 164 #ifdef __xpv 165 typedef maddr_t rootnex_addr_t; 166 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 167 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 168 #else 169 typedef paddr_t rootnex_addr_t; 170 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 171 #endif 172 173 static struct cb_ops rootnex_cb_ops = { 174 nodev, /* open */ 175 nodev, /* close */ 176 nodev, /* strategy */ 177 nodev, /* print */ 178 nodev, /* dump */ 179 nodev, /* read */ 180 nodev, /* write */ 181 nodev, /* ioctl */ 182 nodev, /* devmap */ 183 nodev, /* mmap */ 184 nodev, /* segmap */ 185 nochpoll, /* chpoll */ 186 ddi_prop_op, /* cb_prop_op */ 187 NULL, /* struct streamtab */ 188 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 189 CB_REV, /* Rev */ 190 nodev, /* cb_aread */ 191 nodev /* cb_awrite */ 192 }; 193 194 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 195 off_t offset, off_t len, caddr_t *vaddrp); 196 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 197 struct hat *hat, struct seg *seg, caddr_t addr, 198 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 201 ddi_dma_handle_t *handlep); 202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle); 204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle); 209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 213 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 218 ddi_ctl_enum_t ctlop, void *arg, void *result); 219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 220 ddi_iblock_cookie_t *ibc); 221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 224 void *); 225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 226 227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 229 ddi_dma_handle_t *handlep); 230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 231 ddi_dma_handle_t handle); 232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 234 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 236 ddi_dma_handle_t handle); 237 #if defined(__amd64) && !defined(__xpv) 238 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 239 ddi_dma_handle_t handle); 240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 243 ddi_dma_cookie_t *cookiep, uint_t ccount); 244 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 245 ddi_dma_handle_t handle); 246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 247 #endif 248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 252 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 253 254 #if defined(__amd64) && !defined(__xpv) 255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 256 ddi_dma_handle_t handle, void *v); 257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 258 ddi_dma_handle_t handle); 259 #endif 260 261 262 static struct bus_ops rootnex_bus_ops = { 263 BUSO_REV, 264 rootnex_map, 265 NULL, 266 NULL, 267 NULL, 268 rootnex_map_fault, 269 0, 270 rootnex_dma_allochdl, 271 rootnex_dma_freehdl, 272 rootnex_dma_bindhdl, 273 rootnex_dma_unbindhdl, 274 rootnex_dma_sync, 275 rootnex_dma_win, 276 rootnex_dma_mctl, 277 rootnex_ctlops, 278 ddi_bus_prop_op, 279 i_ddi_rootnex_get_eventcookie, 280 i_ddi_rootnex_add_eventcall, 281 i_ddi_rootnex_remove_eventcall, 282 i_ddi_rootnex_post_event, 283 0, /* bus_intr_ctl */ 284 0, /* bus_config */ 285 0, /* bus_unconfig */ 286 rootnex_fm_init, /* bus_fm_init */ 287 NULL, /* bus_fm_fini */ 288 NULL, /* bus_fm_access_enter */ 289 NULL, /* bus_fm_access_exit */ 290 NULL, /* bus_powr */ 291 rootnex_intr_ops /* bus_intr_op */ 292 }; 293 294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 296 static int rootnex_quiesce(dev_info_t *dip); 297 298 static struct dev_ops rootnex_ops = { 299 DEVO_REV, 300 0, 301 ddi_no_info, 302 nulldev, 303 nulldev, 304 rootnex_attach, 305 rootnex_detach, 306 nulldev, 307 &rootnex_cb_ops, 308 &rootnex_bus_ops, 309 NULL, 310 rootnex_quiesce, /* quiesce */ 311 }; 312 313 static struct modldrv rootnex_modldrv = { 314 &mod_driverops, 315 "i86pc root nexus", 316 &rootnex_ops 317 }; 318 319 static struct modlinkage rootnex_modlinkage = { 320 MODREV_1, 321 (void *)&rootnex_modldrv, 322 NULL 323 }; 324 325 #if defined(__amd64) && !defined(__xpv) 326 static iommulib_nexops_t iommulib_nexops = { 327 IOMMU_NEXOPS_VERSION, 328 "Rootnex IOMMU ops Vers 1.1", 329 NULL, 330 rootnex_coredma_allochdl, 331 rootnex_coredma_freehdl, 332 rootnex_coredma_bindhdl, 333 rootnex_coredma_unbindhdl, 334 rootnex_coredma_reset_cookies, 335 rootnex_coredma_get_cookies, 336 rootnex_coredma_set_cookies, 337 rootnex_coredma_clear_cookies, 338 rootnex_coredma_get_sleep_flags, 339 rootnex_coredma_sync, 340 rootnex_coredma_win, 341 rootnex_coredma_hdl_setprivate, 342 rootnex_coredma_hdl_getprivate 343 }; 344 #endif 345 346 /* 347 * extern hacks 348 */ 349 extern struct seg_ops segdev_ops; 350 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 351 #ifdef DDI_MAP_DEBUG 352 extern int ddi_map_debug_flag; 353 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 354 #endif 355 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 356 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 357 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 358 psm_intr_op_t, int *); 359 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 360 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 361 362 /* 363 * Use device arena to use for device control register mappings. 364 * Various kernel memory walkers (debugger, dtrace) need to know 365 * to avoid this address range to prevent undesired device activity. 366 */ 367 extern void *device_arena_alloc(size_t size, int vm_flag); 368 extern void device_arena_free(void * vaddr, size_t size); 369 370 371 /* 372 * Internal functions 373 */ 374 static int rootnex_dma_init(); 375 static void rootnex_add_props(dev_info_t *); 376 static int rootnex_ctl_reportdev(dev_info_t *dip); 377 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 378 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 379 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 380 static int rootnex_map_handle(ddi_map_req_t *mp); 381 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 382 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 383 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 384 ddi_dma_attr_t *attr); 385 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 386 rootnex_sglinfo_t *sglinfo); 387 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 388 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 389 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 390 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 391 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 392 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 393 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 394 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 395 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 396 static void rootnex_teardown_windows(rootnex_dma_t *dma); 397 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 398 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 399 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 400 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 401 size_t *copybuf_used, page_t **cur_pp); 402 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 403 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 404 ddi_dma_attr_t *attr, off_t cur_offset); 405 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 406 rootnex_dma_t *dma, rootnex_window_t **windowp, 407 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 408 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 409 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 410 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 411 off_t offset, size_t size, uint_t cache_flags); 412 static int rootnex_verify_buffer(rootnex_dma_t *dma); 413 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 414 const void *comp_addr, const void *not_used); 415 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 416 rootnex_sglinfo_t *sglinfo); 417 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 418 419 /* 420 * _init() 421 * 422 */ 423 int 424 _init(void) 425 { 426 427 rootnex_state = NULL; 428 return (mod_install(&rootnex_modlinkage)); 429 } 430 431 432 /* 433 * _info() 434 * 435 */ 436 int 437 _info(struct modinfo *modinfop) 438 { 439 return (mod_info(&rootnex_modlinkage, modinfop)); 440 } 441 442 443 /* 444 * _fini() 445 * 446 */ 447 int 448 _fini(void) 449 { 450 return (EBUSY); 451 } 452 453 454 /* 455 * rootnex_attach() 456 * 457 */ 458 static int 459 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 460 { 461 int fmcap; 462 int e; 463 464 switch (cmd) { 465 case DDI_ATTACH: 466 break; 467 case DDI_RESUME: 468 #if defined(__amd64) && !defined(__xpv) 469 return (immu_unquiesce()); 470 #else 471 return (DDI_SUCCESS); 472 #endif 473 default: 474 return (DDI_FAILURE); 475 } 476 477 /* 478 * We should only have one instance of rootnex. Save it away since we 479 * don't have an easy way to get it back later. 480 */ 481 ASSERT(rootnex_state == NULL); 482 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 483 484 rootnex_state->r_dip = dip; 485 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 486 rootnex_state->r_reserved_msg_printed = B_FALSE; 487 #ifdef DEBUG 488 rootnex_cnt = &rootnex_state->r_counters[0]; 489 #endif 490 491 /* 492 * Set minimum fm capability level for i86pc platforms and then 493 * initialize error handling. Since we're the rootnex, we don't 494 * care what's returned in the fmcap field. 495 */ 496 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 497 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 498 fmcap = ddi_system_fmcap; 499 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 500 501 /* initialize DMA related state */ 502 e = rootnex_dma_init(); 503 if (e != DDI_SUCCESS) { 504 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 505 return (DDI_FAILURE); 506 } 507 508 /* Add static root node properties */ 509 rootnex_add_props(dip); 510 511 /* since we can't call ddi_report_dev() */ 512 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 513 514 /* Initialize rootnex event handle */ 515 i_ddi_rootnex_init_events(dip); 516 517 #if defined(__amd64) && !defined(__xpv) 518 e = iommulib_nexus_register(dip, &iommulib_nexops, 519 &rootnex_state->r_iommulib_handle); 520 521 ASSERT(e == DDI_SUCCESS); 522 #endif 523 524 return (DDI_SUCCESS); 525 } 526 527 528 /* 529 * rootnex_detach() 530 * 531 */ 532 /*ARGSUSED*/ 533 static int 534 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 535 { 536 switch (cmd) { 537 case DDI_SUSPEND: 538 #if defined(__amd64) && !defined(__xpv) 539 return (immu_quiesce()); 540 #else 541 return (DDI_SUCCESS); 542 #endif 543 default: 544 return (DDI_FAILURE); 545 } 546 /*NOTREACHED*/ 547 548 } 549 550 551 /* 552 * rootnex_dma_init() 553 * 554 */ 555 /*ARGSUSED*/ 556 static int 557 rootnex_dma_init() 558 { 559 size_t bufsize; 560 561 562 /* 563 * size of our cookie/window/copybuf state needed in dma bind that we 564 * pre-alloc in dma_alloc_handle 565 */ 566 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 567 rootnex_state->r_prealloc_size = 568 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 569 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 570 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 571 572 /* 573 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 574 * allocate 16 extra bytes for struct pointer alignment 575 * (p->dmai_private & dma->dp_prealloc_buffer) 576 */ 577 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 578 rootnex_state->r_prealloc_size + 0x10; 579 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 580 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 581 if (rootnex_state->r_dmahdl_cache == NULL) { 582 return (DDI_FAILURE); 583 } 584 585 /* 586 * allocate array to track which major numbers we have printed warnings 587 * for. 588 */ 589 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 590 KM_SLEEP); 591 592 return (DDI_SUCCESS); 593 } 594 595 596 /* 597 * rootnex_add_props() 598 * 599 */ 600 static void 601 rootnex_add_props(dev_info_t *dip) 602 { 603 rootnex_intprop_t *rpp; 604 int i; 605 606 /* Add static integer/boolean properties to the root node */ 607 rpp = rootnex_intprp; 608 for (i = 0; i < NROOT_INTPROPS; i++) { 609 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 610 rpp[i].prop_name, rpp[i].prop_value); 611 } 612 } 613 614 615 616 /* 617 * ************************* 618 * ctlops related routines 619 * ************************* 620 */ 621 622 /* 623 * rootnex_ctlops() 624 * 625 */ 626 /*ARGSUSED*/ 627 static int 628 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 629 void *arg, void *result) 630 { 631 int n, *ptr; 632 struct ddi_parent_private_data *pdp; 633 634 switch (ctlop) { 635 case DDI_CTLOPS_DMAPMAPC: 636 /* 637 * Return 'partial' to indicate that dma mapping 638 * has to be done in the main MMU. 639 */ 640 return (DDI_DMA_PARTIAL); 641 642 case DDI_CTLOPS_BTOP: 643 /* 644 * Convert byte count input to physical page units. 645 * (byte counts that are not a page-size multiple 646 * are rounded down) 647 */ 648 *(ulong_t *)result = btop(*(ulong_t *)arg); 649 return (DDI_SUCCESS); 650 651 case DDI_CTLOPS_PTOB: 652 /* 653 * Convert size in physical pages to bytes 654 */ 655 *(ulong_t *)result = ptob(*(ulong_t *)arg); 656 return (DDI_SUCCESS); 657 658 case DDI_CTLOPS_BTOPR: 659 /* 660 * Convert byte count input to physical page units 661 * (byte counts that are not a page-size multiple 662 * are rounded up) 663 */ 664 *(ulong_t *)result = btopr(*(ulong_t *)arg); 665 return (DDI_SUCCESS); 666 667 case DDI_CTLOPS_INITCHILD: 668 return (impl_ddi_sunbus_initchild(arg)); 669 670 case DDI_CTLOPS_UNINITCHILD: 671 impl_ddi_sunbus_removechild(arg); 672 return (DDI_SUCCESS); 673 674 case DDI_CTLOPS_REPORTDEV: 675 return (rootnex_ctl_reportdev(rdip)); 676 677 case DDI_CTLOPS_IOMIN: 678 /* 679 * Nothing to do here but reflect back.. 680 */ 681 return (DDI_SUCCESS); 682 683 case DDI_CTLOPS_REGSIZE: 684 case DDI_CTLOPS_NREGS: 685 break; 686 687 case DDI_CTLOPS_SIDDEV: 688 if (ndi_dev_is_prom_node(rdip)) 689 return (DDI_SUCCESS); 690 if (ndi_dev_is_persistent_node(rdip)) 691 return (DDI_SUCCESS); 692 return (DDI_FAILURE); 693 694 case DDI_CTLOPS_POWER: 695 return ((*pm_platform_power)((power_req_t *)arg)); 696 697 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 698 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 699 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 700 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 701 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 702 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 703 if (!rootnex_state->r_reserved_msg_printed) { 704 rootnex_state->r_reserved_msg_printed = B_TRUE; 705 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 706 "1 or more reserved/obsolete operations."); 707 } 708 return (DDI_FAILURE); 709 710 default: 711 return (DDI_FAILURE); 712 } 713 /* 714 * The rest are for "hardware" properties 715 */ 716 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 717 return (DDI_FAILURE); 718 719 if (ctlop == DDI_CTLOPS_NREGS) { 720 ptr = (int *)result; 721 *ptr = pdp->par_nreg; 722 } else { 723 off_t *size = (off_t *)result; 724 725 ptr = (int *)arg; 726 n = *ptr; 727 if (n >= pdp->par_nreg) { 728 return (DDI_FAILURE); 729 } 730 *size = (off_t)pdp->par_reg[n].regspec_size; 731 } 732 return (DDI_SUCCESS); 733 } 734 735 736 /* 737 * rootnex_ctl_reportdev() 738 * 739 */ 740 static int 741 rootnex_ctl_reportdev(dev_info_t *dev) 742 { 743 int i, n, len, f_len = 0; 744 char *buf; 745 746 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 747 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 748 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 749 len = strlen(buf); 750 751 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 752 753 struct regspec *rp = sparc_pd_getreg(dev, i); 754 755 if (i == 0) 756 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 757 ": "); 758 else 759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 760 " and "); 761 len = strlen(buf); 762 763 switch (rp->regspec_bustype) { 764 765 case BTEISA: 766 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 767 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 768 break; 769 770 case BTISA: 771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 772 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 773 break; 774 775 default: 776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 777 "space %x offset %x", 778 rp->regspec_bustype, rp->regspec_addr); 779 break; 780 } 781 len = strlen(buf); 782 } 783 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 784 int pri; 785 786 if (i != 0) { 787 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 788 ","); 789 len = strlen(buf); 790 } 791 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 793 " sparc ipl %d", pri); 794 len = strlen(buf); 795 } 796 #ifdef DEBUG 797 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 798 cmn_err(CE_NOTE, "next message is truncated: " 799 "printed length 1024, real length %d", f_len); 800 } 801 #endif /* DEBUG */ 802 cmn_err(CE_CONT, "?%s\n", buf); 803 kmem_free(buf, REPORTDEV_BUFSIZE); 804 return (DDI_SUCCESS); 805 } 806 807 808 /* 809 * ****************** 810 * map related code 811 * ****************** 812 */ 813 814 /* 815 * rootnex_map() 816 * 817 */ 818 static int 819 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 820 off_t len, caddr_t *vaddrp) 821 { 822 struct regspec *orp = NULL; 823 struct regspec64 rp = { 0 }; 824 ddi_map_req_t mr = *mp; /* Get private copy of request */ 825 826 mp = &mr; 827 828 switch (mp->map_op) { 829 case DDI_MO_MAP_LOCKED: 830 case DDI_MO_UNMAP: 831 case DDI_MO_MAP_HANDLE: 832 break; 833 default: 834 #ifdef DDI_MAP_DEBUG 835 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 836 mp->map_op); 837 #endif /* DDI_MAP_DEBUG */ 838 return (DDI_ME_UNIMPLEMENTED); 839 } 840 841 if (mp->map_flags & DDI_MF_USER_MAPPING) { 842 #ifdef DDI_MAP_DEBUG 843 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 844 #endif /* DDI_MAP_DEBUG */ 845 return (DDI_ME_UNIMPLEMENTED); 846 } 847 848 /* 849 * First, we need to get the original regspec out before we convert it 850 * to the extended format. If we have a register number, then we need to 851 * convert that to a regspec. 852 */ 853 if (mp->map_type == DDI_MT_RNUMBER) { 854 855 int rnumber = mp->map_obj.rnumber; 856 #ifdef DDI_MAP_DEBUG 857 static char *out_of_range = 858 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 859 #endif /* DDI_MAP_DEBUG */ 860 861 orp = i_ddi_rnumber_to_regspec(rdip, rnumber); 862 if (orp == NULL) { 863 #ifdef DDI_MAP_DEBUG 864 cmn_err(CE_WARN, out_of_range, rnumber, 865 ddi_get_name(rdip)); 866 #endif /* DDI_MAP_DEBUG */ 867 return (DDI_ME_RNUMBER_RANGE); 868 } 869 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) { 870 orp = mp->map_obj.rp; 871 } 872 873 /* 874 * Ensure that we are always using a 64-bit extended regspec regardless 875 * of what was passed into us. If the child driver is using a 64-bit 876 * regspec, then we need to make sure that we copy this to the local 877 * regspec64, rp. 878 */ 879 if (orp != NULL) { 880 rp.regspec_bustype = orp->regspec_bustype; 881 rp.regspec_addr = orp->regspec_addr; 882 rp.regspec_size = orp->regspec_size; 883 } else { 884 struct regspec64 *rp64; 885 rp64 = (struct regspec64 *)mp->map_obj.rp; 886 rp = *rp64; 887 } 888 889 mp->map_type = DDI_MT_REGSPEC; 890 mp->map_flags |= DDI_MF_EXT_REGSPEC; 891 mp->map_obj.rp = (struct regspec *)&rp; 892 893 /* 894 * Adjust offset and length correspnding to called values... 895 * XXX: A non-zero length means override the one in the regspec 896 * XXX: (regardless of what's in the parent's range?) 897 */ 898 899 #ifdef DDI_MAP_DEBUG 900 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 901 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 902 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset, 903 len, mp->map_handlep); 904 #endif /* DDI_MAP_DEBUG */ 905 906 /* 907 * I/O or memory mapping: 908 * 909 * <bustype=0, addr=x, len=x>: memory 910 * <bustype=1, addr=x, len=x>: i/o 911 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 912 */ 913 914 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) { 915 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 916 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">", 917 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 918 rp.regspec_addr, rp.regspec_size); 919 return (DDI_ME_INVAL); 920 } 921 922 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) { 923 /* 924 * compatibility i/o mapping 925 */ 926 rp.regspec_bustype += offset; 927 } else { 928 /* 929 * Normal memory or i/o mapping 930 */ 931 rp.regspec_addr += offset; 932 } 933 934 if (len != 0) 935 rp.regspec_size = len; 936 937 #ifdef DDI_MAP_DEBUG 938 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64 939 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n", 940 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 941 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep); 942 #endif /* DDI_MAP_DEBUG */ 943 944 945 /* 946 * The x86 root nexus does not have any notion of valid ranges of 947 * addresses. Its children have valid ranges, but because there are none 948 * for the nexus, we don't need to call i_ddi_apply_range(). Verify 949 * that is the case. 950 */ 951 ASSERT0(sparc_pd_getnrng(dip)); 952 953 switch (mp->map_op) { 954 case DDI_MO_MAP_LOCKED: 955 956 /* 957 * Set up the locked down kernel mapping to the regspec... 958 */ 959 960 return (rootnex_map_regspec(mp, vaddrp)); 961 962 case DDI_MO_UNMAP: 963 964 /* 965 * Release mapping... 966 */ 967 968 return (rootnex_unmap_regspec(mp, vaddrp)); 969 970 case DDI_MO_MAP_HANDLE: 971 972 return (rootnex_map_handle(mp)); 973 974 default: 975 return (DDI_ME_UNIMPLEMENTED); 976 } 977 } 978 979 980 /* 981 * rootnex_map_fault() 982 * 983 * fault in mappings for requestors 984 */ 985 /*ARGSUSED*/ 986 static int 987 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 988 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 989 uint_t lock) 990 { 991 992 #ifdef DDI_MAP_DEBUG 993 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 994 ddi_map_debug(" Seg <%s>\n", 995 seg->s_ops == &segdev_ops ? "segdev" : 996 seg == &kvseg ? "segkmem" : "NONE!"); 997 #endif /* DDI_MAP_DEBUG */ 998 999 /* 1000 * This is all terribly broken, but it is a start 1001 * 1002 * XXX Note that this test means that segdev_ops 1003 * must be exported from seg_dev.c. 1004 * XXX What about devices with their own segment drivers? 1005 */ 1006 if (seg->s_ops == &segdev_ops) { 1007 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1008 1009 if (hat == NULL) { 1010 /* 1011 * This is one plausible interpretation of 1012 * a null hat i.e. use the first hat on the 1013 * address space hat list which by convention is 1014 * the hat of the system MMU. At alternative 1015 * would be to panic .. this might well be better .. 1016 */ 1017 ASSERT(AS_READ_HELD(seg->s_as)); 1018 hat = seg->s_as->a_hat; 1019 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1020 } 1021 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1022 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1023 } else if (seg == &kvseg && dp == NULL) { 1024 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1025 HAT_LOAD_LOCK); 1026 } else 1027 return (DDI_FAILURE); 1028 return (DDI_SUCCESS); 1029 } 1030 1031 1032 static int 1033 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1034 { 1035 rootnex_addr_t rbase; 1036 void *cvaddr; 1037 uint64_t npages, pgoffset; 1038 struct regspec64 *rp; 1039 ddi_acc_hdl_t *hp; 1040 ddi_acc_impl_t *ap; 1041 uint_t hat_acc_flags; 1042 paddr_t pbase; 1043 1044 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1045 rp = (struct regspec64 *)mp->map_obj.rp; 1046 hp = mp->map_handlep; 1047 1048 #ifdef DDI_MAP_DEBUG 1049 ddi_map_debug( 1050 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1051 rp->regspec_bustype, rp->regspec_addr, 1052 rp->regspec_size, mp->map_handlep); 1053 #endif /* DDI_MAP_DEBUG */ 1054 1055 /* 1056 * I/O or memory mapping 1057 * 1058 * <bustype=0, addr=x, len=x>: memory 1059 * <bustype=1, addr=x, len=x>: i/o 1060 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1061 */ 1062 1063 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1064 cmn_err(CE_WARN, "rootnex: invalid register spec" 1065 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">", 1066 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size); 1067 return (DDI_FAILURE); 1068 } 1069 1070 if (rp->regspec_bustype != 0) { 1071 /* 1072 * I/O space - needs a handle. 1073 */ 1074 if (hp == NULL) { 1075 return (DDI_FAILURE); 1076 } 1077 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1078 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1079 impl_acc_hdl_init(hp); 1080 1081 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1082 #ifdef DDI_MAP_DEBUG 1083 ddi_map_debug("rootnex_map_regspec: mmap() " 1084 "to I/O space is not supported.\n"); 1085 #endif /* DDI_MAP_DEBUG */ 1086 return (DDI_ME_INVAL); 1087 } else { 1088 /* 1089 * 1275-compliant vs. compatibility i/o mapping 1090 */ 1091 *vaddrp = 1092 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1093 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1094 ((caddr_t)(uintptr_t)rp->regspec_addr); 1095 #ifdef __xpv 1096 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1097 hp->ah_pfn = xen_assign_pfn( 1098 mmu_btop((ulong_t)rp->regspec_addr & 1099 MMU_PAGEMASK)); 1100 } else { 1101 hp->ah_pfn = mmu_btop( 1102 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1103 } 1104 #else 1105 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1106 MMU_PAGEMASK); 1107 #endif 1108 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1109 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1110 } 1111 1112 #ifdef DDI_MAP_DEBUG 1113 ddi_map_debug( 1114 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1115 rp->regspec_size, *vaddrp); 1116 #endif /* DDI_MAP_DEBUG */ 1117 return (DDI_SUCCESS); 1118 } 1119 1120 /* 1121 * Memory space 1122 */ 1123 1124 if (hp != NULL) { 1125 /* 1126 * hat layer ignores 1127 * hp->ah_acc.devacc_attr_endian_flags. 1128 */ 1129 switch (hp->ah_acc.devacc_attr_dataorder) { 1130 case DDI_STRICTORDER_ACC: 1131 hat_acc_flags = HAT_STRICTORDER; 1132 break; 1133 case DDI_UNORDERED_OK_ACC: 1134 hat_acc_flags = HAT_UNORDERED_OK; 1135 break; 1136 case DDI_MERGING_OK_ACC: 1137 hat_acc_flags = HAT_MERGING_OK; 1138 break; 1139 case DDI_LOADCACHING_OK_ACC: 1140 hat_acc_flags = HAT_LOADCACHING_OK; 1141 break; 1142 case DDI_STORECACHING_OK_ACC: 1143 hat_acc_flags = HAT_STORECACHING_OK; 1144 break; 1145 default: 1146 return (DDI_ME_INVAL); 1147 } 1148 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1149 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1150 impl_acc_hdl_init(hp); 1151 hp->ah_hat_flags = hat_acc_flags; 1152 } else { 1153 hat_acc_flags = HAT_STRICTORDER; 1154 } 1155 1156 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1157 #ifdef __xpv 1158 /* 1159 * If we're dom0, we're using a real device so we need to translate 1160 * the MA to a PA. 1161 */ 1162 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1163 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1164 } else { 1165 pbase = rbase; 1166 } 1167 #else 1168 pbase = rbase; 1169 #endif 1170 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1171 1172 if (rp->regspec_size == 0) { 1173 #ifdef DDI_MAP_DEBUG 1174 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1175 #endif /* DDI_MAP_DEBUG */ 1176 return (DDI_ME_INVAL); 1177 } 1178 1179 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1180 /* extra cast to make gcc happy */ 1181 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1182 } else { 1183 npages = mmu_btopr(rp->regspec_size + pgoffset); 1184 1185 #ifdef DDI_MAP_DEBUG 1186 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1187 "physical %llx", npages, pbase); 1188 #endif /* DDI_MAP_DEBUG */ 1189 1190 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1191 if (cvaddr == NULL) 1192 return (DDI_ME_NORESOURCES); 1193 1194 /* 1195 * Now map in the pages we've allocated... 1196 */ 1197 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1198 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1199 HAT_LOAD_LOCK); 1200 *vaddrp = (caddr_t)cvaddr + pgoffset; 1201 1202 /* save away pfn and npages for FMA */ 1203 hp = mp->map_handlep; 1204 if (hp) { 1205 hp->ah_pfn = mmu_btop(pbase); 1206 hp->ah_pnum = npages; 1207 } 1208 } 1209 1210 #ifdef DDI_MAP_DEBUG 1211 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1212 #endif /* DDI_MAP_DEBUG */ 1213 return (DDI_SUCCESS); 1214 } 1215 1216 1217 static int 1218 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1219 { 1220 caddr_t addr = (caddr_t)*vaddrp; 1221 uint64_t npages, pgoffset; 1222 struct regspec64 *rp; 1223 1224 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1225 return (0); 1226 1227 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1228 rp = (struct regspec64 *)mp->map_obj.rp; 1229 1230 if (rp->regspec_size == 0) { 1231 #ifdef DDI_MAP_DEBUG 1232 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1233 #endif /* DDI_MAP_DEBUG */ 1234 return (DDI_ME_INVAL); 1235 } 1236 1237 /* 1238 * I/O or memory mapping: 1239 * 1240 * <bustype=0, addr=x, len=x>: memory 1241 * <bustype=1, addr=x, len=x>: i/o 1242 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1243 */ 1244 if (rp->regspec_bustype != 0) { 1245 /* 1246 * This is I/O space, which requires no particular 1247 * processing on unmap since it isn't mapped in the 1248 * first place. 1249 */ 1250 return (DDI_SUCCESS); 1251 } 1252 1253 /* 1254 * Memory space 1255 */ 1256 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1257 npages = mmu_btopr(rp->regspec_size + pgoffset); 1258 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1259 device_arena_free(addr - pgoffset, ptob(npages)); 1260 1261 /* 1262 * Destroy the pointer - the mapping has logically gone 1263 */ 1264 *vaddrp = NULL; 1265 1266 return (DDI_SUCCESS); 1267 } 1268 1269 static int 1270 rootnex_map_handle(ddi_map_req_t *mp) 1271 { 1272 rootnex_addr_t rbase; 1273 ddi_acc_hdl_t *hp; 1274 uint64_t pgoffset; 1275 struct regspec64 *rp; 1276 paddr_t pbase; 1277 1278 rp = (struct regspec64 *)mp->map_obj.rp; 1279 1280 #ifdef DDI_MAP_DEBUG 1281 ddi_map_debug( 1282 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1283 rp->regspec_bustype, rp->regspec_addr, 1284 rp->regspec_size, mp->map_handlep); 1285 #endif /* DDI_MAP_DEBUG */ 1286 1287 /* 1288 * I/O or memory mapping: 1289 * 1290 * <bustype=0, addr=x, len=x>: memory 1291 * <bustype=1, addr=x, len=x>: i/o 1292 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1293 */ 1294 if (rp->regspec_bustype != 0) { 1295 /* 1296 * This refers to I/O space, and we don't support "mapping" 1297 * I/O space to a user. 1298 */ 1299 return (DDI_FAILURE); 1300 } 1301 1302 /* 1303 * Set up the hat_flags for the mapping. 1304 */ 1305 hp = mp->map_handlep; 1306 1307 switch (hp->ah_acc.devacc_attr_endian_flags) { 1308 case DDI_NEVERSWAP_ACC: 1309 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1310 break; 1311 case DDI_STRUCTURE_LE_ACC: 1312 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1313 break; 1314 case DDI_STRUCTURE_BE_ACC: 1315 return (DDI_FAILURE); 1316 default: 1317 return (DDI_REGS_ACC_CONFLICT); 1318 } 1319 1320 switch (hp->ah_acc.devacc_attr_dataorder) { 1321 case DDI_STRICTORDER_ACC: 1322 break; 1323 case DDI_UNORDERED_OK_ACC: 1324 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1325 break; 1326 case DDI_MERGING_OK_ACC: 1327 hp->ah_hat_flags |= HAT_MERGING_OK; 1328 break; 1329 case DDI_LOADCACHING_OK_ACC: 1330 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1331 break; 1332 case DDI_STORECACHING_OK_ACC: 1333 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1334 break; 1335 default: 1336 return (DDI_FAILURE); 1337 } 1338 1339 rbase = (rootnex_addr_t)rp->regspec_addr & 1340 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1341 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1342 1343 if (rp->regspec_size == 0) 1344 return (DDI_ME_INVAL); 1345 1346 #ifdef __xpv 1347 /* 1348 * If we're dom0, we're using a real device so we need to translate 1349 * the MA to a PA. 1350 */ 1351 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1352 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1353 (rbase & MMU_PAGEOFFSET); 1354 } else { 1355 pbase = rbase; 1356 } 1357 #else 1358 pbase = rbase; 1359 #endif 1360 1361 hp->ah_pfn = mmu_btop(pbase); 1362 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1363 1364 return (DDI_SUCCESS); 1365 } 1366 1367 1368 1369 /* 1370 * ************************ 1371 * interrupt related code 1372 * ************************ 1373 */ 1374 1375 /* 1376 * rootnex_intr_ops() 1377 * bus_intr_op() function for interrupt support 1378 */ 1379 /* ARGSUSED */ 1380 static int 1381 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1382 ddi_intr_handle_impl_t *hdlp, void *result) 1383 { 1384 struct intrspec *ispec; 1385 1386 DDI_INTR_NEXDBG((CE_CONT, 1387 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1388 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1389 1390 /* Process the interrupt operation */ 1391 switch (intr_op) { 1392 case DDI_INTROP_GETCAP: 1393 /* First check with pcplusmp */ 1394 if (psm_intr_ops == NULL) 1395 return (DDI_FAILURE); 1396 1397 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1398 *(int *)result = 0; 1399 return (DDI_FAILURE); 1400 } 1401 break; 1402 case DDI_INTROP_SETCAP: 1403 if (psm_intr_ops == NULL) 1404 return (DDI_FAILURE); 1405 1406 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1407 return (DDI_FAILURE); 1408 break; 1409 case DDI_INTROP_ALLOC: 1410 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1411 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1412 case DDI_INTROP_FREE: 1413 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1414 return (rootnex_free_intr_fixed(rdip, hdlp)); 1415 case DDI_INTROP_GETPRI: 1416 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1417 return (DDI_FAILURE); 1418 *(int *)result = ispec->intrspec_pri; 1419 break; 1420 case DDI_INTROP_SETPRI: 1421 /* Validate the interrupt priority passed to us */ 1422 if (*(int *)result > LOCK_LEVEL) 1423 return (DDI_FAILURE); 1424 1425 /* Ensure that PSM is all initialized and ispec is ok */ 1426 if ((psm_intr_ops == NULL) || 1427 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1428 return (DDI_FAILURE); 1429 1430 /* Change the priority */ 1431 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1432 PSM_FAILURE) 1433 return (DDI_FAILURE); 1434 1435 /* update the ispec with the new priority */ 1436 ispec->intrspec_pri = *(int *)result; 1437 break; 1438 case DDI_INTROP_ADDISR: 1439 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1440 return (DDI_FAILURE); 1441 ispec->intrspec_func = hdlp->ih_cb_func; 1442 break; 1443 case DDI_INTROP_REMISR: 1444 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 return (DDI_FAILURE); 1446 ispec->intrspec_func = (uint_t (*)()) 0; 1447 break; 1448 case DDI_INTROP_ENABLE: 1449 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1450 return (DDI_FAILURE); 1451 1452 /* Call psmi to translate irq with the dip */ 1453 if (psm_intr_ops == NULL) 1454 return (DDI_FAILURE); 1455 1456 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1457 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1458 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1459 return (DDI_FAILURE); 1460 1461 /* Add the interrupt handler */ 1462 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1463 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1464 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1465 return (DDI_FAILURE); 1466 break; 1467 case DDI_INTROP_DISABLE: 1468 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1469 return (DDI_FAILURE); 1470 1471 /* Call psm_ops() to translate irq with the dip */ 1472 if (psm_intr_ops == NULL) 1473 return (DDI_FAILURE); 1474 1475 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1476 (void) (*psm_intr_ops)(rdip, hdlp, 1477 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1478 1479 /* Remove the interrupt handler */ 1480 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1481 hdlp->ih_cb_func, hdlp->ih_vector); 1482 break; 1483 case DDI_INTROP_SETMASK: 1484 if (psm_intr_ops == NULL) 1485 return (DDI_FAILURE); 1486 1487 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1488 return (DDI_FAILURE); 1489 break; 1490 case DDI_INTROP_CLRMASK: 1491 if (psm_intr_ops == NULL) 1492 return (DDI_FAILURE); 1493 1494 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1495 return (DDI_FAILURE); 1496 break; 1497 case DDI_INTROP_GETPENDING: 1498 if (psm_intr_ops == NULL) 1499 return (DDI_FAILURE); 1500 1501 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1502 result)) { 1503 *(int *)result = 0; 1504 return (DDI_FAILURE); 1505 } 1506 break; 1507 case DDI_INTROP_NAVAIL: 1508 case DDI_INTROP_NINTRS: 1509 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1510 if (*(int *)result == 0) { 1511 /* 1512 * Special case for 'pcic' driver' only. This driver 1513 * driver is a child of 'isa' and 'rootnex' drivers. 1514 * 1515 * See detailed comments on this in the function 1516 * rootnex_get_ispec(). 1517 * 1518 * Children of 'pcic' send 'NINITR' request all the 1519 * way to rootnex driver. But, the 'pdp->par_nintr' 1520 * field may not initialized. So, we fake it here 1521 * to return 1 (a la what PCMCIA nexus does). 1522 */ 1523 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1524 *(int *)result = 1; 1525 else 1526 return (DDI_FAILURE); 1527 } 1528 break; 1529 case DDI_INTROP_SUPPORTED_TYPES: 1530 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1531 break; 1532 default: 1533 return (DDI_FAILURE); 1534 } 1535 1536 return (DDI_SUCCESS); 1537 } 1538 1539 1540 /* 1541 * rootnex_get_ispec() 1542 * convert an interrupt number to an interrupt specification. 1543 * The interrupt number determines which interrupt spec will be 1544 * returned if more than one exists. 1545 * 1546 * Look into the parent private data area of the 'rdip' to find out 1547 * the interrupt specification. First check to make sure there is 1548 * one that matchs "inumber" and then return a pointer to it. 1549 * 1550 * Return NULL if one could not be found. 1551 * 1552 * NOTE: This is needed for rootnex_intr_ops() 1553 */ 1554 static struct intrspec * 1555 rootnex_get_ispec(dev_info_t *rdip, int inum) 1556 { 1557 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1558 1559 /* 1560 * Special case handling for drivers that provide their own 1561 * intrspec structures instead of relying on the DDI framework. 1562 * 1563 * A broken hardware driver in ON could potentially provide its 1564 * own intrspec structure, instead of relying on the hardware. 1565 * If these drivers are children of 'rootnex' then we need to 1566 * continue to provide backward compatibility to them here. 1567 * 1568 * Following check is a special case for 'pcic' driver which 1569 * was found to have broken hardwre andby provides its own intrspec. 1570 * 1571 * Verbatim comments from this driver are shown here: 1572 * "Don't use the ddi_add_intr since we don't have a 1573 * default intrspec in all cases." 1574 * 1575 * Since an 'ispec' may not be always created for it, 1576 * check for that and create one if so. 1577 * 1578 * NOTE: Currently 'pcic' is the only driver found to do this. 1579 */ 1580 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1581 pdp->par_nintr = 1; 1582 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1583 pdp->par_nintr, KM_SLEEP); 1584 } 1585 1586 /* Validate the interrupt number */ 1587 if (inum >= pdp->par_nintr) 1588 return (NULL); 1589 1590 /* Get the interrupt structure pointer and return that */ 1591 return ((struct intrspec *)&pdp->par_intr[inum]); 1592 } 1593 1594 /* 1595 * Allocate interrupt vector for FIXED (legacy) type. 1596 */ 1597 static int 1598 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1599 void *result) 1600 { 1601 struct intrspec *ispec; 1602 ddi_intr_handle_impl_t info_hdl; 1603 int ret; 1604 int free_phdl = 0; 1605 apic_get_type_t type_info; 1606 1607 if (psm_intr_ops == NULL) 1608 return (DDI_FAILURE); 1609 1610 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1611 return (DDI_FAILURE); 1612 1613 /* 1614 * If the PSM module is "APIX" then pass the request for it 1615 * to allocate the vector now. 1616 */ 1617 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1618 info_hdl.ih_private = &type_info; 1619 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1620 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1621 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1622 free_phdl = 1; 1623 i_ddi_alloc_intr_phdl(hdlp); 1624 } 1625 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1626 ret = (*psm_intr_ops)(rdip, hdlp, 1627 PSM_INTR_OP_ALLOC_VECTORS, result); 1628 if (free_phdl) { /* free up the phdl structure */ 1629 free_phdl = 0; 1630 i_ddi_free_intr_phdl(hdlp); 1631 hdlp->ih_private = NULL; 1632 } 1633 } else { 1634 /* 1635 * No APIX module; fall back to the old scheme where the 1636 * interrupt vector is allocated during ddi_enable_intr() call. 1637 */ 1638 hdlp->ih_pri = ispec->intrspec_pri; 1639 *(int *)result = hdlp->ih_scratch1; 1640 ret = DDI_SUCCESS; 1641 } 1642 1643 return (ret); 1644 } 1645 1646 /* 1647 * Free up interrupt vector for FIXED (legacy) type. 1648 */ 1649 static int 1650 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1651 { 1652 struct intrspec *ispec; 1653 struct ddi_parent_private_data *pdp; 1654 ddi_intr_handle_impl_t info_hdl; 1655 int ret; 1656 apic_get_type_t type_info; 1657 1658 if (psm_intr_ops == NULL) 1659 return (DDI_FAILURE); 1660 1661 /* 1662 * If the PSM module is "APIX" then pass the request for it 1663 * to free up the vector now. 1664 */ 1665 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1666 info_hdl.ih_private = &type_info; 1667 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1668 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1669 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1670 return (DDI_FAILURE); 1671 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1672 ret = (*psm_intr_ops)(rdip, hdlp, 1673 PSM_INTR_OP_FREE_VECTORS, NULL); 1674 } else { 1675 /* 1676 * No APIX module; fall back to the old scheme where 1677 * the interrupt vector was already freed during 1678 * ddi_disable_intr() call. 1679 */ 1680 ret = DDI_SUCCESS; 1681 } 1682 1683 pdp = ddi_get_parent_data(rdip); 1684 1685 /* 1686 * Special case for 'pcic' driver' only. 1687 * If an intrspec was created for it, clean it up here 1688 * See detailed comments on this in the function 1689 * rootnex_get_ispec(). 1690 */ 1691 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1692 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1693 pdp->par_nintr); 1694 /* 1695 * Set it to zero; so that 1696 * DDI framework doesn't free it again 1697 */ 1698 pdp->par_intr = NULL; 1699 pdp->par_nintr = 0; 1700 } 1701 1702 return (ret); 1703 } 1704 1705 1706 /* 1707 * ****************** 1708 * dma related code 1709 * ****************** 1710 */ 1711 1712 /*ARGSUSED*/ 1713 static int 1714 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1715 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1716 ddi_dma_handle_t *handlep) 1717 { 1718 uint64_t maxsegmentsize_ll; 1719 uint_t maxsegmentsize; 1720 ddi_dma_impl_t *hp; 1721 rootnex_dma_t *dma; 1722 uint64_t count_max; 1723 uint64_t seg; 1724 int kmflag; 1725 int e; 1726 1727 1728 /* convert our sleep flags */ 1729 if (waitfp == DDI_DMA_SLEEP) { 1730 kmflag = KM_SLEEP; 1731 } else { 1732 kmflag = KM_NOSLEEP; 1733 } 1734 1735 /* 1736 * We try to do only one memory allocation here. We'll do a little 1737 * pointer manipulation later. If the bind ends up taking more than 1738 * our prealloc's space, we'll have to allocate more memory in the 1739 * bind operation. Not great, but much better than before and the 1740 * best we can do with the current bind interfaces. 1741 */ 1742 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1743 if (hp == NULL) 1744 return (DDI_DMA_NORESOURCES); 1745 1746 /* Do our pointer manipulation now, align the structures */ 1747 hp->dmai_private = (void *)(((uintptr_t)hp + 1748 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1749 dma = (rootnex_dma_t *)hp->dmai_private; 1750 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1751 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1752 1753 /* setup the handle */ 1754 rootnex_clean_dmahdl(hp); 1755 hp->dmai_error.err_fep = NULL; 1756 hp->dmai_error.err_cf = NULL; 1757 dma->dp_dip = rdip; 1758 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1759 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1760 1761 /* 1762 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1763 * is being used. Set the upper limit to the seg value. 1764 * There will be enough DVMA space to always get addresses 1765 * that will match the constraints. 1766 */ 1767 if (IOMMU_USED(rdip) && 1768 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1769 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1770 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1771 } else 1772 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1773 1774 hp->dmai_minxfer = attr->dma_attr_minxfer; 1775 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1776 hp->dmai_rdip = rdip; 1777 hp->dmai_attr = *attr; 1778 1779 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1780 dma->dp_sglinfo.si_cancross = B_FALSE; 1781 else 1782 dma->dp_sglinfo.si_cancross = B_TRUE; 1783 1784 /* we don't need to worry about the SPL since we do a tryenter */ 1785 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1786 1787 /* 1788 * Figure out our maximum segment size. If the segment size is greater 1789 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1790 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1791 * dma_attr_count_max are size-1 type values. 1792 * 1793 * Maximum segment size is the largest physically contiguous chunk of 1794 * memory that we can return from a bind (i.e. the maximum size of a 1795 * single cookie). 1796 */ 1797 1798 /* handle the rollover cases */ 1799 seg = attr->dma_attr_seg + 1; 1800 if (seg < attr->dma_attr_seg) { 1801 seg = attr->dma_attr_seg; 1802 } 1803 count_max = attr->dma_attr_count_max + 1; 1804 if (count_max < attr->dma_attr_count_max) { 1805 count_max = attr->dma_attr_count_max; 1806 } 1807 1808 /* 1809 * granularity may or may not be a power of two. If it isn't, we can't 1810 * use a simple mask. 1811 */ 1812 if (!ISP2(attr->dma_attr_granular)) { 1813 dma->dp_granularity_power_2 = B_FALSE; 1814 } else { 1815 dma->dp_granularity_power_2 = B_TRUE; 1816 } 1817 1818 /* 1819 * maxxfer should be a whole multiple of granularity. If we're going to 1820 * break up a window because we're greater than maxxfer, we might as 1821 * well make sure it's maxxfer is a whole multiple so we don't have to 1822 * worry about triming the window later on for this case. 1823 */ 1824 if (attr->dma_attr_granular > 1) { 1825 if (dma->dp_granularity_power_2) { 1826 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1827 (attr->dma_attr_maxxfer & 1828 (attr->dma_attr_granular - 1)); 1829 } else { 1830 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1831 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1832 } 1833 } else { 1834 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1835 } 1836 1837 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1838 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1839 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1840 maxsegmentsize = 0xFFFFFFFF; 1841 } else { 1842 maxsegmentsize = maxsegmentsize_ll; 1843 } 1844 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1845 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1846 1847 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1848 if (rootnex_alloc_check_parms) { 1849 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1850 if (e != DDI_SUCCESS) { 1851 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1852 (void) rootnex_dma_freehdl(dip, rdip, 1853 (ddi_dma_handle_t)hp); 1854 return (e); 1855 } 1856 } 1857 1858 *handlep = (ddi_dma_handle_t)hp; 1859 1860 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1861 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1862 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1863 1864 return (DDI_SUCCESS); 1865 } 1866 1867 1868 /* 1869 * rootnex_dma_allochdl() 1870 * called from ddi_dma_alloc_handle(). 1871 */ 1872 static int 1873 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1874 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1875 { 1876 int retval = DDI_SUCCESS; 1877 #if defined(__amd64) && !defined(__xpv) 1878 1879 if (IOMMU_UNITIALIZED(rdip)) { 1880 retval = iommulib_nex_open(dip, rdip); 1881 1882 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1883 return (retval); 1884 } 1885 1886 if (IOMMU_UNUSED(rdip)) { 1887 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1888 handlep); 1889 } else { 1890 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1891 waitfp, arg, handlep); 1892 } 1893 #else 1894 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1895 handlep); 1896 #endif 1897 switch (retval) { 1898 case DDI_DMA_NORESOURCES: 1899 if (waitfp != DDI_DMA_DONTWAIT) { 1900 ddi_set_callback(waitfp, arg, 1901 &rootnex_state->r_dvma_call_list_id); 1902 } 1903 break; 1904 case DDI_SUCCESS: 1905 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1906 break; 1907 default: 1908 break; 1909 } 1910 return (retval); 1911 } 1912 1913 /*ARGSUSED*/ 1914 static int 1915 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1916 ddi_dma_handle_t handle) 1917 { 1918 ddi_dma_impl_t *hp; 1919 rootnex_dma_t *dma; 1920 1921 1922 hp = (ddi_dma_impl_t *)handle; 1923 dma = (rootnex_dma_t *)hp->dmai_private; 1924 1925 /* unbind should have been called first */ 1926 ASSERT(!dma->dp_inuse); 1927 1928 mutex_destroy(&dma->dp_mutex); 1929 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1930 1931 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1932 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1933 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1934 1935 return (DDI_SUCCESS); 1936 } 1937 1938 /* 1939 * rootnex_dma_freehdl() 1940 * called from ddi_dma_free_handle(). 1941 */ 1942 static int 1943 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1944 { 1945 int ret; 1946 1947 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1948 #if defined(__amd64) && !defined(__xpv) 1949 if (IOMMU_USED(rdip)) 1950 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1951 else 1952 #endif 1953 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1954 1955 if (rootnex_state->r_dvma_call_list_id) 1956 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1957 1958 return (ret); 1959 } 1960 1961 /*ARGSUSED*/ 1962 static int 1963 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1964 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1965 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1966 { 1967 rootnex_sglinfo_t *sinfo; 1968 ddi_dma_obj_t *dmao; 1969 #if defined(__amd64) && !defined(__xpv) 1970 struct dvmaseg *dvs; 1971 ddi_dma_cookie_t *cookie; 1972 #endif 1973 ddi_dma_attr_t *attr; 1974 ddi_dma_impl_t *hp; 1975 rootnex_dma_t *dma; 1976 int kmflag; 1977 int e; 1978 uint_t ncookies; 1979 1980 hp = (ddi_dma_impl_t *)handle; 1981 dma = (rootnex_dma_t *)hp->dmai_private; 1982 dmao = &dma->dp_dma; 1983 sinfo = &dma->dp_sglinfo; 1984 attr = &hp->dmai_attr; 1985 1986 /* convert the sleep flags */ 1987 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1988 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1989 } else { 1990 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1991 } 1992 1993 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1994 1995 /* 1996 * This is useful for debugging a driver. Not as useful in a production 1997 * system. The only time this will fail is if you have a driver bug. 1998 */ 1999 if (rootnex_bind_check_inuse) { 2000 /* 2001 * No one else should ever have this lock unless someone else 2002 * is trying to use this handle. So contention on the lock 2003 * is the same as inuse being set. 2004 */ 2005 e = mutex_tryenter(&dma->dp_mutex); 2006 if (e == 0) { 2007 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2008 return (DDI_DMA_INUSE); 2009 } 2010 if (dma->dp_inuse) { 2011 mutex_exit(&dma->dp_mutex); 2012 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2013 return (DDI_DMA_INUSE); 2014 } 2015 dma->dp_inuse = B_TRUE; 2016 mutex_exit(&dma->dp_mutex); 2017 } 2018 2019 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2020 if (rootnex_bind_check_parms) { 2021 e = rootnex_valid_bind_parms(dmareq, attr); 2022 if (e != DDI_SUCCESS) { 2023 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2024 rootnex_clean_dmahdl(hp); 2025 return (e); 2026 } 2027 } 2028 2029 /* save away the original bind info */ 2030 dma->dp_dma = dmareq->dmar_object; 2031 2032 #if defined(__amd64) && !defined(__xpv) 2033 if (IOMMU_USED(rdip)) { 2034 dmao = &dma->dp_dvma; 2035 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2036 switch (e) { 2037 case DDI_SUCCESS: 2038 if (sinfo->si_cancross || 2039 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2040 dmao->dmao_size > sinfo->si_max_cookie_size) { 2041 dma->dp_dvma_used = B_TRUE; 2042 break; 2043 } 2044 sinfo->si_sgl_size = 1; 2045 hp->dmai_rflags |= DMP_NOSYNC; 2046 2047 dma->dp_dvma_used = B_TRUE; 2048 dma->dp_need_to_free_cookie = B_FALSE; 2049 2050 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2051 cookie = hp->dmai_cookie = dma->dp_cookies = 2052 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2053 cookie->dmac_laddress = dvs->dvs_start + 2054 dmao->dmao_obj.dvma_obj.dv_off; 2055 cookie->dmac_size = dvs->dvs_len; 2056 cookie->dmac_type = 0; 2057 2058 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2059 rdip); 2060 goto fast; 2061 case DDI_ENOTSUP: 2062 break; 2063 default: 2064 rootnex_clean_dmahdl(hp); 2065 return (e); 2066 } 2067 } 2068 #endif 2069 2070 /* 2071 * Figure out a rough estimate of what maximum number of pages 2072 * this buffer could use (a high estimate of course). 2073 */ 2074 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2075 2076 if (dma->dp_dvma_used) { 2077 /* 2078 * The number of physical pages is the worst case. 2079 * 2080 * For DVMA, the worst case is the length divided 2081 * by the maximum cookie length, plus 1. Add to that 2082 * the number of segment boundaries potentially crossed, and 2083 * the additional number of DVMA segments that was returned. 2084 * 2085 * In the normal case, for modern devices, si_cancross will 2086 * be false, and dv_nseg will be 1, and the fast path will 2087 * have been taken above. 2088 */ 2089 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2090 + 1; 2091 if (sinfo->si_cancross) 2092 ncookies += 2093 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2094 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2095 2096 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2097 } 2098 2099 /* 2100 * We'll use the pre-allocated cookies for any bind that will *always* 2101 * fit (more important to be consistent, we don't want to create 2102 * additional degenerate cases). 2103 */ 2104 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2105 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2106 dma->dp_need_to_free_cookie = B_FALSE; 2107 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2108 uint_t, sinfo->si_max_pages); 2109 2110 /* 2111 * For anything larger than that, we'll go ahead and allocate the 2112 * maximum number of pages we expect to see. Hopefuly, we won't be 2113 * seeing this path in the fast path for high performance devices very 2114 * frequently. 2115 * 2116 * a ddi bind interface that allowed the driver to provide storage to 2117 * the bind interface would speed this case up. 2118 */ 2119 } else { 2120 /* 2121 * Save away how much memory we allocated. If we're doing a 2122 * nosleep, the alloc could fail... 2123 */ 2124 dma->dp_cookie_size = sinfo->si_max_pages * 2125 sizeof (ddi_dma_cookie_t); 2126 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2127 if (dma->dp_cookies == NULL) { 2128 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2129 rootnex_clean_dmahdl(hp); 2130 return (DDI_DMA_NORESOURCES); 2131 } 2132 dma->dp_need_to_free_cookie = B_TRUE; 2133 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2134 uint_t, sinfo->si_max_pages); 2135 } 2136 hp->dmai_cookie = dma->dp_cookies; 2137 2138 /* 2139 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2140 * looking at the constraints in the dma structure. It will then put 2141 * some additional state about the sgl in the dma struct (i.e. is 2142 * the sgl clean, or do we need to do some munging; how many pages 2143 * need to be copied, etc.) 2144 */ 2145 if (dma->dp_dvma_used) 2146 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2147 else 2148 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2149 2150 out: 2151 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2152 /* if we don't need a copy buffer, we don't need to sync */ 2153 if (sinfo->si_copybuf_req == 0) { 2154 hp->dmai_rflags |= DMP_NOSYNC; 2155 } 2156 2157 /* 2158 * if we don't need the copybuf and we don't need to do a partial, we 2159 * hit the fast path. All the high performance devices should be trying 2160 * to hit this path. To hit this path, a device should be able to reach 2161 * all of memory, shouldn't try to bind more than it can transfer, and 2162 * the buffer shouldn't require more cookies than the driver/device can 2163 * handle [sgllen]). 2164 * 2165 * Note that negative values of dma_attr_sgllen are supposed 2166 * to mean unlimited, but we just cast them to mean a 2167 * "ridiculous large limit". This saves some extra checks on 2168 * hot paths. 2169 */ 2170 if ((sinfo->si_copybuf_req == 0) && 2171 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2172 (dmao->dmao_size <= dma->dp_maxxfer)) { 2173 fast: 2174 /* 2175 * If the driver supports FMA, insert the handle in the FMA DMA 2176 * handle cache. 2177 */ 2178 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2179 hp->dmai_error.err_cf = rootnex_dma_check; 2180 2181 /* 2182 * copy out the first cookie and ccountp, set the cookie 2183 * pointer to the second cookie. The first cookie is passed 2184 * back on the stack. Additional cookies are accessed via 2185 * ddi_dma_nextcookie() 2186 */ 2187 *cookiep = dma->dp_cookies[0]; 2188 *ccountp = sinfo->si_sgl_size; 2189 hp->dmai_cookie++; 2190 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2191 hp->dmai_ncookies = *ccountp; 2192 hp->dmai_curcookie = 1; 2193 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2194 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2195 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2196 uint_t, dmao->dmao_size, uint_t, *ccountp); 2197 2198 2199 return (DDI_DMA_MAPPED); 2200 } 2201 2202 /* 2203 * go to the slow path, we may need to alloc more memory, create 2204 * multiple windows, and munge up a sgl to make the device happy. 2205 */ 2206 2207 /* 2208 * With the IOMMU mapobject method used, we should never hit 2209 * the slow path. If we do, something is seriously wrong. 2210 * Clean up and return an error. 2211 */ 2212 2213 #if defined(__amd64) && !defined(__xpv) 2214 2215 if (dma->dp_dvma_used) { 2216 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2217 &dma->dp_dvma); 2218 e = DDI_DMA_NOMAPPING; 2219 } else { 2220 #endif 2221 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2222 kmflag); 2223 #if defined(__amd64) && !defined(__xpv) 2224 } 2225 #endif 2226 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2227 if (dma->dp_need_to_free_cookie) { 2228 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2229 } 2230 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2231 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2232 return (e); 2233 } 2234 2235 /* 2236 * If the driver supports FMA, insert the handle in the FMA DMA handle 2237 * cache. 2238 */ 2239 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2240 hp->dmai_error.err_cf = rootnex_dma_check; 2241 2242 /* if the first window uses the copy buffer, sync it for the device */ 2243 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2244 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2245 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2246 DDI_DMA_SYNC_FORDEV); 2247 } 2248 2249 /* 2250 * copy out the first cookie and ccountp, set the cookie pointer to the 2251 * second cookie. Make sure the partial flag is set/cleared correctly. 2252 * If we have a partial map (i.e. multiple windows), the number of 2253 * cookies we return is the number of cookies in the first window. 2254 */ 2255 if (e == DDI_DMA_MAPPED) { 2256 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2257 *ccountp = sinfo->si_sgl_size; 2258 hp->dmai_nwin = 1; 2259 } else { 2260 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2261 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2262 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2263 } 2264 *cookiep = dma->dp_cookies[0]; 2265 hp->dmai_cookie++; 2266 hp->dmai_ncookies = *ccountp; 2267 hp->dmai_curcookie = 1; 2268 2269 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2270 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2271 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2272 dmao->dmao_size, uint_t, *ccountp); 2273 return (e); 2274 } 2275 2276 /* 2277 * rootnex_dma_bindhdl() 2278 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2279 */ 2280 static int 2281 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2282 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2283 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2284 { 2285 int ret; 2286 #if defined(__amd64) && !defined(__xpv) 2287 if (IOMMU_USED(rdip)) 2288 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2289 cookiep, ccountp); 2290 else 2291 #endif 2292 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2293 cookiep, ccountp); 2294 2295 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2296 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2297 &rootnex_state->r_dvma_call_list_id); 2298 } 2299 2300 return (ret); 2301 } 2302 2303 2304 2305 /*ARGSUSED*/ 2306 static int 2307 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2308 ddi_dma_handle_t handle) 2309 { 2310 ddi_dma_impl_t *hp; 2311 rootnex_dma_t *dma; 2312 int e; 2313 2314 hp = (ddi_dma_impl_t *)handle; 2315 dma = (rootnex_dma_t *)hp->dmai_private; 2316 2317 /* make sure the buffer wasn't free'd before calling unbind */ 2318 if (rootnex_unbind_verify_buffer) { 2319 e = rootnex_verify_buffer(dma); 2320 if (e != DDI_SUCCESS) { 2321 ASSERT(0); 2322 return (DDI_FAILURE); 2323 } 2324 } 2325 2326 /* sync the current window before unbinding the buffer */ 2327 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2328 (hp->dmai_rflags & DDI_DMA_READ)) { 2329 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2330 DDI_DMA_SYNC_FORCPU); 2331 } 2332 2333 /* 2334 * cleanup and copy buffer or window state. if we didn't use the copy 2335 * buffer or windows, there won't be much to do :-) 2336 */ 2337 rootnex_teardown_copybuf(dma); 2338 rootnex_teardown_windows(dma); 2339 2340 #if defined(__amd64) && !defined(__xpv) 2341 if (IOMMU_USED(rdip) && dma->dp_dvma_used) 2342 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2343 &dma->dp_dvma); 2344 #endif 2345 2346 /* 2347 * If we had to allocate space to for the worse case sgl (it didn't 2348 * fit into our pre-allocate buffer), free that up now 2349 */ 2350 if (dma->dp_need_to_free_cookie) { 2351 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2352 } 2353 2354 /* 2355 * clean up the handle so it's ready for the next bind (i.e. if the 2356 * handle is reused). 2357 */ 2358 rootnex_clean_dmahdl(hp); 2359 hp->dmai_error.err_cf = NULL; 2360 2361 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2362 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2363 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2364 2365 return (DDI_SUCCESS); 2366 } 2367 2368 /* 2369 * rootnex_dma_unbindhdl() 2370 * called from ddi_dma_unbind_handle() 2371 */ 2372 /*ARGSUSED*/ 2373 static int 2374 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2375 ddi_dma_handle_t handle) 2376 { 2377 int ret; 2378 2379 #if defined(__amd64) && !defined(__xpv) 2380 if (IOMMU_USED(rdip)) 2381 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2382 else 2383 #endif 2384 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2385 2386 if (rootnex_state->r_dvma_call_list_id) 2387 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2388 2389 return (ret); 2390 } 2391 2392 #if defined(__amd64) && !defined(__xpv) 2393 2394 static int 2395 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2396 { 2397 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2398 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2399 2400 if (dma->dp_sleep_flags != KM_SLEEP && 2401 dma->dp_sleep_flags != KM_NOSLEEP) 2402 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2403 return (dma->dp_sleep_flags); 2404 } 2405 /*ARGSUSED*/ 2406 static void 2407 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2408 { 2409 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2410 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2411 rootnex_window_t *window; 2412 2413 if (dma->dp_window) { 2414 window = &dma->dp_window[dma->dp_current_win]; 2415 hp->dmai_cookie = window->wd_first_cookie; 2416 } else { 2417 hp->dmai_cookie = dma->dp_cookies; 2418 } 2419 hp->dmai_cookie++; 2420 hp->dmai_curcookie = 1; 2421 } 2422 2423 /*ARGSUSED*/ 2424 static int 2425 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2426 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2427 { 2428 int i; 2429 int km_flags; 2430 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2431 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2432 rootnex_window_t *window; 2433 ddi_dma_cookie_t *cp; 2434 ddi_dma_cookie_t *cookie; 2435 2436 ASSERT(*cookiepp == NULL); 2437 ASSERT(*ccountp == 0); 2438 2439 if (dma->dp_window) { 2440 window = &dma->dp_window[dma->dp_current_win]; 2441 cp = window->wd_first_cookie; 2442 *ccountp = window->wd_cookie_cnt; 2443 } else { 2444 cp = dma->dp_cookies; 2445 *ccountp = dma->dp_sglinfo.si_sgl_size; 2446 } 2447 2448 km_flags = rootnex_coredma_get_sleep_flags(handle); 2449 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2450 if (cookie == NULL) { 2451 return (DDI_DMA_NORESOURCES); 2452 } 2453 2454 for (i = 0; i < *ccountp; i++) { 2455 cookie[i].dmac_notused = cp[i].dmac_notused; 2456 cookie[i].dmac_type = cp[i].dmac_type; 2457 cookie[i].dmac_address = cp[i].dmac_address; 2458 cookie[i].dmac_size = cp[i].dmac_size; 2459 } 2460 2461 *cookiepp = cookie; 2462 2463 return (DDI_SUCCESS); 2464 } 2465 2466 /*ARGSUSED*/ 2467 static int 2468 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2469 ddi_dma_cookie_t *cookiep, uint_t ccount) 2470 { 2471 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2472 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2473 rootnex_window_t *window; 2474 ddi_dma_cookie_t *cur_cookiep; 2475 2476 ASSERT(cookiep); 2477 ASSERT(ccount != 0); 2478 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2479 2480 if (dma->dp_window) { 2481 window = &dma->dp_window[dma->dp_current_win]; 2482 dma->dp_saved_cookies = window->wd_first_cookie; 2483 window->wd_first_cookie = cookiep; 2484 ASSERT(ccount == window->wd_cookie_cnt); 2485 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2486 + window->wd_first_cookie; 2487 } else { 2488 dma->dp_saved_cookies = dma->dp_cookies; 2489 dma->dp_cookies = cookiep; 2490 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2491 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2492 + dma->dp_cookies; 2493 } 2494 2495 dma->dp_need_to_switch_cookies = B_TRUE; 2496 hp->dmai_cookie = cur_cookiep; 2497 2498 return (DDI_SUCCESS); 2499 } 2500 2501 /*ARGSUSED*/ 2502 static int 2503 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2504 { 2505 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2506 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2507 rootnex_window_t *window; 2508 ddi_dma_cookie_t *cur_cookiep; 2509 ddi_dma_cookie_t *cookie_array; 2510 uint_t ccount; 2511 2512 /* check if cookies have not been switched */ 2513 if (dma->dp_need_to_switch_cookies == B_FALSE) 2514 return (DDI_SUCCESS); 2515 2516 ASSERT(dma->dp_saved_cookies); 2517 2518 if (dma->dp_window) { 2519 window = &dma->dp_window[dma->dp_current_win]; 2520 cookie_array = window->wd_first_cookie; 2521 window->wd_first_cookie = dma->dp_saved_cookies; 2522 dma->dp_saved_cookies = NULL; 2523 ccount = window->wd_cookie_cnt; 2524 cur_cookiep = (hp->dmai_cookie - cookie_array) 2525 + window->wd_first_cookie; 2526 } else { 2527 cookie_array = dma->dp_cookies; 2528 dma->dp_cookies = dma->dp_saved_cookies; 2529 dma->dp_saved_cookies = NULL; 2530 ccount = dma->dp_sglinfo.si_sgl_size; 2531 cur_cookiep = (hp->dmai_cookie - cookie_array) 2532 + dma->dp_cookies; 2533 } 2534 2535 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2536 2537 hp->dmai_cookie = cur_cookiep; 2538 2539 dma->dp_need_to_switch_cookies = B_FALSE; 2540 2541 return (DDI_SUCCESS); 2542 } 2543 2544 #endif 2545 2546 static struct as * 2547 rootnex_get_as(ddi_dma_obj_t *dmao) 2548 { 2549 struct as *asp; 2550 2551 switch (dmao->dmao_type) { 2552 case DMA_OTYP_VADDR: 2553 case DMA_OTYP_BUFVADDR: 2554 asp = dmao->dmao_obj.virt_obj.v_as; 2555 if (asp == NULL) 2556 asp = &kas; 2557 break; 2558 default: 2559 asp = NULL; 2560 break; 2561 } 2562 return (asp); 2563 } 2564 2565 /* 2566 * rootnex_verify_buffer() 2567 * verify buffer wasn't free'd 2568 */ 2569 static int 2570 rootnex_verify_buffer(rootnex_dma_t *dma) 2571 { 2572 page_t **pplist; 2573 caddr_t vaddr; 2574 uint_t pcnt; 2575 uint_t poff; 2576 page_t *pp; 2577 char b; 2578 int i; 2579 2580 /* Figure out how many pages this buffer occupies */ 2581 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2582 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2583 } else { 2584 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2585 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2586 } 2587 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2588 2589 switch (dma->dp_dma.dmao_type) { 2590 case DMA_OTYP_PAGES: 2591 /* 2592 * for a linked list of pp's walk through them to make sure 2593 * they're locked and not free. 2594 */ 2595 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2596 for (i = 0; i < pcnt; i++) { 2597 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2598 return (DDI_FAILURE); 2599 } 2600 pp = pp->p_next; 2601 } 2602 break; 2603 2604 case DMA_OTYP_VADDR: 2605 case DMA_OTYP_BUFVADDR: 2606 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2607 /* 2608 * for an array of pp's walk through them to make sure they're 2609 * not free. It's possible that they may not be locked. 2610 */ 2611 if (pplist) { 2612 for (i = 0; i < pcnt; i++) { 2613 if (PP_ISFREE(pplist[i])) { 2614 return (DDI_FAILURE); 2615 } 2616 } 2617 2618 /* For a virtual address, try to peek at each page */ 2619 } else { 2620 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2621 for (i = 0; i < pcnt; i++) { 2622 if (ddi_peek8(NULL, vaddr, &b) == 2623 DDI_FAILURE) 2624 return (DDI_FAILURE); 2625 vaddr += MMU_PAGESIZE; 2626 } 2627 } 2628 } 2629 break; 2630 2631 default: 2632 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2633 break; 2634 } 2635 2636 return (DDI_SUCCESS); 2637 } 2638 2639 2640 /* 2641 * rootnex_clean_dmahdl() 2642 * Clean the dma handle. This should be called on a handle alloc and an 2643 * unbind handle. Set the handle state to the default settings. 2644 */ 2645 static void 2646 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2647 { 2648 rootnex_dma_t *dma; 2649 2650 2651 dma = (rootnex_dma_t *)hp->dmai_private; 2652 2653 hp->dmai_nwin = 0; 2654 dma->dp_current_cookie = 0; 2655 dma->dp_copybuf_size = 0; 2656 dma->dp_window = NULL; 2657 dma->dp_cbaddr = NULL; 2658 dma->dp_inuse = B_FALSE; 2659 dma->dp_dvma_used = B_FALSE; 2660 dma->dp_need_to_free_cookie = B_FALSE; 2661 dma->dp_need_to_switch_cookies = B_FALSE; 2662 dma->dp_saved_cookies = NULL; 2663 dma->dp_sleep_flags = KM_PANIC; 2664 dma->dp_need_to_free_window = B_FALSE; 2665 dma->dp_partial_required = B_FALSE; 2666 dma->dp_trim_required = B_FALSE; 2667 dma->dp_sglinfo.si_copybuf_req = 0; 2668 #if !defined(__amd64) 2669 dma->dp_cb_remaping = B_FALSE; 2670 dma->dp_kva = NULL; 2671 #endif 2672 2673 /* FMA related initialization */ 2674 hp->dmai_fault = 0; 2675 hp->dmai_fault_check = NULL; 2676 hp->dmai_fault_notify = NULL; 2677 hp->dmai_error.err_ena = 0; 2678 hp->dmai_error.err_status = DDI_FM_OK; 2679 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2680 hp->dmai_error.err_ontrap = NULL; 2681 2682 /* Cookie tracking */ 2683 hp->dmai_ncookies = 0; 2684 hp->dmai_curcookie = 0; 2685 } 2686 2687 2688 /* 2689 * rootnex_valid_alloc_parms() 2690 * Called in ddi_dma_alloc_handle path to validate its parameters. 2691 */ 2692 static int 2693 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2694 { 2695 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2696 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2697 (attr->dma_attr_granular > MMU_PAGESIZE) || 2698 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2699 return (DDI_DMA_BADATTR); 2700 } 2701 2702 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2703 return (DDI_DMA_BADATTR); 2704 } 2705 2706 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2707 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2708 attr->dma_attr_sgllen == 0) { 2709 return (DDI_DMA_BADATTR); 2710 } 2711 2712 /* We should be able to DMA into every byte offset in a page */ 2713 if (maxsegmentsize < MMU_PAGESIZE) { 2714 return (DDI_DMA_BADATTR); 2715 } 2716 2717 /* if we're bouncing on seg, seg must be <= addr_hi */ 2718 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2719 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2720 return (DDI_DMA_BADATTR); 2721 } 2722 return (DDI_SUCCESS); 2723 } 2724 2725 /* 2726 * rootnex_valid_bind_parms() 2727 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2728 */ 2729 /* ARGSUSED */ 2730 static int 2731 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2732 { 2733 #if !defined(__amd64) 2734 /* 2735 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2736 * we can track the offset for the obsoleted interfaces. 2737 */ 2738 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2739 return (DDI_DMA_TOOBIG); 2740 } 2741 #endif 2742 2743 return (DDI_SUCCESS); 2744 } 2745 2746 2747 /* 2748 * rootnex_need_bounce_seg() 2749 * check to see if the buffer lives on both side of the seg. 2750 */ 2751 static boolean_t 2752 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2753 { 2754 ddi_dma_atyp_t buftype; 2755 rootnex_addr_t raddr; 2756 boolean_t lower_addr; 2757 boolean_t upper_addr; 2758 uint64_t offset; 2759 page_t **pplist; 2760 uint64_t paddr; 2761 uint32_t psize; 2762 uint32_t size; 2763 caddr_t vaddr; 2764 uint_t pcnt; 2765 page_t *pp; 2766 2767 pp = NULL; 2768 /* shortcuts */ 2769 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2770 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2771 buftype = dmar_object->dmao_type; 2772 size = dmar_object->dmao_size; 2773 2774 lower_addr = B_FALSE; 2775 upper_addr = B_FALSE; 2776 pcnt = 0; 2777 2778 /* 2779 * Process the first page to handle the initial offset of the buffer. 2780 * We'll use the base address we get later when we loop through all 2781 * the pages. 2782 */ 2783 if (buftype == DMA_OTYP_PAGES) { 2784 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2785 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2786 MMU_PAGEOFFSET; 2787 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2788 psize = MIN(size, (MMU_PAGESIZE - offset)); 2789 pp = pp->p_next; 2790 sglinfo->si_asp = NULL; 2791 } else if (pplist != NULL) { 2792 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2793 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2794 if (sglinfo->si_asp == NULL) { 2795 sglinfo->si_asp = &kas; 2796 } 2797 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2798 paddr += offset; 2799 psize = MIN(size, (MMU_PAGESIZE - offset)); 2800 pcnt++; 2801 } else { 2802 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2803 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2804 if (sglinfo->si_asp == NULL) { 2805 sglinfo->si_asp = &kas; 2806 } 2807 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2808 paddr += offset; 2809 psize = MIN(size, (MMU_PAGESIZE - offset)); 2810 vaddr += psize; 2811 } 2812 2813 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2814 2815 if ((raddr + psize) > sglinfo->si_segmask) { 2816 upper_addr = B_TRUE; 2817 } else { 2818 lower_addr = B_TRUE; 2819 } 2820 size -= psize; 2821 2822 /* 2823 * Walk through the rest of the pages in the buffer. Track to see 2824 * if we have pages on both sides of the segment boundary. 2825 */ 2826 while (size > 0) { 2827 /* partial or full page */ 2828 psize = MIN(size, MMU_PAGESIZE); 2829 2830 if (buftype == DMA_OTYP_PAGES) { 2831 /* get the paddr from the page_t */ 2832 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2833 paddr = pfn_to_pa(pp->p_pagenum); 2834 pp = pp->p_next; 2835 } else if (pplist != NULL) { 2836 /* index into the array of page_t's to get the paddr */ 2837 ASSERT(!PP_ISFREE(pplist[pcnt])); 2838 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2839 pcnt++; 2840 } else { 2841 /* call into the VM to get the paddr */ 2842 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2843 vaddr)); 2844 vaddr += psize; 2845 } 2846 2847 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2848 2849 if ((raddr + psize) > sglinfo->si_segmask) { 2850 upper_addr = B_TRUE; 2851 } else { 2852 lower_addr = B_TRUE; 2853 } 2854 /* 2855 * if the buffer lives both above and below the segment 2856 * boundary, or the current page is the page immediately 2857 * after the segment, we will use a copy/bounce buffer for 2858 * all pages > seg. 2859 */ 2860 if ((lower_addr && upper_addr) || 2861 (raddr == (sglinfo->si_segmask + 1))) { 2862 return (B_TRUE); 2863 } 2864 2865 size -= psize; 2866 } 2867 2868 return (B_FALSE); 2869 } 2870 2871 /* 2872 * rootnex_get_sgl() 2873 * Called in bind fastpath to get the sgl. Most of this will be replaced 2874 * with a call to the vm layer when vm2.0 comes around... 2875 */ 2876 static void 2877 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2878 rootnex_sglinfo_t *sglinfo) 2879 { 2880 ddi_dma_atyp_t buftype; 2881 rootnex_addr_t raddr; 2882 uint64_t last_page; 2883 uint64_t offset; 2884 uint64_t addrhi; 2885 uint64_t addrlo; 2886 uint64_t maxseg; 2887 page_t **pplist; 2888 uint64_t paddr; 2889 uint32_t psize; 2890 uint32_t size; 2891 caddr_t vaddr; 2892 uint_t pcnt; 2893 page_t *pp; 2894 uint_t cnt; 2895 2896 pp = NULL; 2897 /* shortcuts */ 2898 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2899 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2900 maxseg = sglinfo->si_max_cookie_size; 2901 buftype = dmar_object->dmao_type; 2902 addrhi = sglinfo->si_max_addr; 2903 addrlo = sglinfo->si_min_addr; 2904 size = dmar_object->dmao_size; 2905 2906 pcnt = 0; 2907 cnt = 0; 2908 2909 2910 /* 2911 * check to see if we need to use the copy buffer for pages over 2912 * the segment attr. 2913 */ 2914 sglinfo->si_bounce_on_seg = B_FALSE; 2915 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2916 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2917 dmar_object, sglinfo); 2918 } 2919 2920 /* 2921 * if we were passed down a linked list of pages, i.e. pointer to 2922 * page_t, use this to get our physical address and buf offset. 2923 */ 2924 if (buftype == DMA_OTYP_PAGES) { 2925 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2926 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2927 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2928 MMU_PAGEOFFSET; 2929 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2930 psize = MIN(size, (MMU_PAGESIZE - offset)); 2931 pp = pp->p_next; 2932 sglinfo->si_asp = NULL; 2933 2934 /* 2935 * We weren't passed down a linked list of pages, but if we were passed 2936 * down an array of pages, use this to get our physical address and buf 2937 * offset. 2938 */ 2939 } else if (pplist != NULL) { 2940 ASSERT((buftype == DMA_OTYP_VADDR) || 2941 (buftype == DMA_OTYP_BUFVADDR)); 2942 2943 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2944 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2945 if (sglinfo->si_asp == NULL) { 2946 sglinfo->si_asp = &kas; 2947 } 2948 2949 ASSERT(!PP_ISFREE(pplist[pcnt])); 2950 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2951 paddr += offset; 2952 psize = MIN(size, (MMU_PAGESIZE - offset)); 2953 pcnt++; 2954 2955 /* 2956 * All we have is a virtual address, we'll need to call into the VM 2957 * to get the physical address. 2958 */ 2959 } else { 2960 ASSERT((buftype == DMA_OTYP_VADDR) || 2961 (buftype == DMA_OTYP_BUFVADDR)); 2962 2963 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2964 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2965 if (sglinfo->si_asp == NULL) { 2966 sglinfo->si_asp = &kas; 2967 } 2968 2969 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2970 paddr += offset; 2971 psize = MIN(size, (MMU_PAGESIZE - offset)); 2972 vaddr += psize; 2973 } 2974 2975 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2976 2977 /* 2978 * Setup the first cookie with the physical address of the page and the 2979 * size of the page (which takes into account the initial offset into 2980 * the page. 2981 */ 2982 sgl[cnt].dmac_laddress = raddr; 2983 sgl[cnt].dmac_size = psize; 2984 sgl[cnt].dmac_type = 0; 2985 2986 /* 2987 * Save away the buffer offset into the page. We'll need this later in 2988 * the copy buffer code to help figure out the page index within the 2989 * buffer and the offset into the current page. 2990 */ 2991 sglinfo->si_buf_offset = offset; 2992 2993 /* 2994 * If we are using the copy buffer for anything over the segment 2995 * boundary, and this page is over the segment boundary. 2996 * OR 2997 * if the DMA engine can't reach the physical address. 2998 */ 2999 if (((sglinfo->si_bounce_on_seg) && 3000 ((raddr + psize) > sglinfo->si_segmask)) || 3001 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3002 /* 3003 * Increase how much copy buffer we use. We always increase by 3004 * pagesize so we don't have to worry about converting offsets. 3005 * Set a flag in the cookies dmac_type to indicate that it uses 3006 * the copy buffer. If this isn't the last cookie, go to the 3007 * next cookie (since we separate each page which uses the copy 3008 * buffer in case the copy buffer is not physically contiguous. 3009 */ 3010 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3011 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3012 if ((cnt + 1) < sglinfo->si_max_pages) { 3013 cnt++; 3014 sgl[cnt].dmac_laddress = 0; 3015 sgl[cnt].dmac_size = 0; 3016 sgl[cnt].dmac_type = 0; 3017 } 3018 } 3019 3020 /* 3021 * save this page's physical address so we can figure out if the next 3022 * page is physically contiguous. Keep decrementing size until we are 3023 * done with the buffer. 3024 */ 3025 last_page = raddr & MMU_PAGEMASK; 3026 size -= psize; 3027 3028 while (size > 0) { 3029 /* Get the size for this page (i.e. partial or full page) */ 3030 psize = MIN(size, MMU_PAGESIZE); 3031 3032 if (buftype == DMA_OTYP_PAGES) { 3033 /* get the paddr from the page_t */ 3034 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3035 paddr = pfn_to_pa(pp->p_pagenum); 3036 pp = pp->p_next; 3037 } else if (pplist != NULL) { 3038 /* index into the array of page_t's to get the paddr */ 3039 ASSERT(!PP_ISFREE(pplist[pcnt])); 3040 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3041 pcnt++; 3042 } else { 3043 /* call into the VM to get the paddr */ 3044 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3045 vaddr)); 3046 vaddr += psize; 3047 } 3048 3049 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3050 3051 /* 3052 * If we are using the copy buffer for anything over the 3053 * segment boundary, and this page is over the segment 3054 * boundary. 3055 * OR 3056 * if the DMA engine can't reach the physical address. 3057 */ 3058 if (((sglinfo->si_bounce_on_seg) && 3059 ((raddr + psize) > sglinfo->si_segmask)) || 3060 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3061 3062 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3063 3064 /* 3065 * if there is something in the current cookie, go to 3066 * the next one. We only want one page in a cookie which 3067 * uses the copybuf since the copybuf doesn't have to 3068 * be physically contiguous. 3069 */ 3070 if (sgl[cnt].dmac_size != 0) { 3071 cnt++; 3072 } 3073 sgl[cnt].dmac_laddress = raddr; 3074 sgl[cnt].dmac_size = psize; 3075 #if defined(__amd64) 3076 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3077 #else 3078 /* 3079 * save the buf offset for 32-bit kernel. used in the 3080 * obsoleted interfaces. 3081 */ 3082 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3083 (dmar_object->dmao_size - size); 3084 #endif 3085 /* if this isn't the last cookie, go to the next one */ 3086 if ((cnt + 1) < sglinfo->si_max_pages) { 3087 cnt++; 3088 sgl[cnt].dmac_laddress = 0; 3089 sgl[cnt].dmac_size = 0; 3090 sgl[cnt].dmac_type = 0; 3091 } 3092 3093 /* 3094 * this page didn't need the copy buffer, if it's not physically 3095 * contiguous, or it would put us over a segment boundary, or it 3096 * puts us over the max cookie size, or the current sgl doesn't 3097 * have anything in it. 3098 */ 3099 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3100 !(raddr & sglinfo->si_segmask) || 3101 ((sgl[cnt].dmac_size + psize) > maxseg) || 3102 (sgl[cnt].dmac_size == 0)) { 3103 /* 3104 * if we're not already in a new cookie, go to the next 3105 * cookie. 3106 */ 3107 if (sgl[cnt].dmac_size != 0) { 3108 cnt++; 3109 } 3110 3111 /* save the cookie information */ 3112 sgl[cnt].dmac_laddress = raddr; 3113 sgl[cnt].dmac_size = psize; 3114 #if defined(__amd64) 3115 sgl[cnt].dmac_type = 0; 3116 #else 3117 /* 3118 * save the buf offset for 32-bit kernel. used in the 3119 * obsoleted interfaces. 3120 */ 3121 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3122 #endif 3123 3124 /* 3125 * this page didn't need the copy buffer, it is physically 3126 * contiguous with the last page, and it's <= the max cookie 3127 * size. 3128 */ 3129 } else { 3130 sgl[cnt].dmac_size += psize; 3131 3132 /* 3133 * if this exactly == the maximum cookie size, and 3134 * it isn't the last cookie, go to the next cookie. 3135 */ 3136 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3137 ((cnt + 1) < sglinfo->si_max_pages)) { 3138 cnt++; 3139 sgl[cnt].dmac_laddress = 0; 3140 sgl[cnt].dmac_size = 0; 3141 sgl[cnt].dmac_type = 0; 3142 } 3143 } 3144 3145 /* 3146 * save this page's physical address so we can figure out if the 3147 * next page is physically contiguous. Keep decrementing size 3148 * until we are done with the buffer. 3149 */ 3150 last_page = raddr; 3151 size -= psize; 3152 } 3153 3154 /* we're done, save away how many cookies the sgl has */ 3155 if (sgl[cnt].dmac_size == 0) { 3156 ASSERT(cnt < sglinfo->si_max_pages); 3157 sglinfo->si_sgl_size = cnt; 3158 } else { 3159 sglinfo->si_sgl_size = cnt + 1; 3160 } 3161 } 3162 3163 static void 3164 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3165 rootnex_sglinfo_t *sglinfo) 3166 { 3167 uint64_t offset; 3168 uint64_t maxseg; 3169 uint64_t dvaddr; 3170 struct dvmaseg *dvs; 3171 uint64_t paddr; 3172 uint32_t psize, ssize; 3173 uint32_t size; 3174 uint_t cnt; 3175 int physcontig; 3176 3177 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3178 3179 /* shortcuts */ 3180 maxseg = sglinfo->si_max_cookie_size; 3181 size = dmar_object->dmao_size; 3182 3183 cnt = 0; 3184 sglinfo->si_bounce_on_seg = B_FALSE; 3185 3186 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3187 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3188 ssize = dvs->dvs_len; 3189 paddr = dvs->dvs_start; 3190 paddr += offset; 3191 psize = MIN(ssize, (maxseg - offset)); 3192 dvaddr = paddr + psize; 3193 ssize -= psize; 3194 3195 sgl[cnt].dmac_laddress = paddr; 3196 sgl[cnt].dmac_size = psize; 3197 sgl[cnt].dmac_type = 0; 3198 3199 size -= psize; 3200 while (size > 0) { 3201 if (ssize == 0) { 3202 dvs++; 3203 ssize = dvs->dvs_len; 3204 dvaddr = dvs->dvs_start; 3205 physcontig = 0; 3206 } else 3207 physcontig = 1; 3208 3209 paddr = dvaddr; 3210 psize = MIN(ssize, maxseg); 3211 dvaddr += psize; 3212 ssize -= psize; 3213 3214 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3215 ((sgl[cnt].dmac_size + psize) > maxseg) || 3216 (sgl[cnt].dmac_size == 0)) { 3217 /* 3218 * if we're not already in a new cookie, go to the next 3219 * cookie. 3220 */ 3221 if (sgl[cnt].dmac_size != 0) { 3222 cnt++; 3223 } 3224 3225 /* save the cookie information */ 3226 sgl[cnt].dmac_laddress = paddr; 3227 sgl[cnt].dmac_size = psize; 3228 sgl[cnt].dmac_type = 0; 3229 } else { 3230 sgl[cnt].dmac_size += psize; 3231 3232 /* 3233 * if this exactly == the maximum cookie size, and 3234 * it isn't the last cookie, go to the next cookie. 3235 */ 3236 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3237 ((cnt + 1) < sglinfo->si_max_pages)) { 3238 cnt++; 3239 sgl[cnt].dmac_laddress = 0; 3240 sgl[cnt].dmac_size = 0; 3241 sgl[cnt].dmac_type = 0; 3242 } 3243 } 3244 size -= psize; 3245 } 3246 3247 /* we're done, save away how many cookies the sgl has */ 3248 if (sgl[cnt].dmac_size == 0) { 3249 sglinfo->si_sgl_size = cnt; 3250 } else { 3251 sglinfo->si_sgl_size = cnt + 1; 3252 } 3253 } 3254 3255 /* 3256 * rootnex_bind_slowpath() 3257 * Call in the bind path if the calling driver can't use the sgl without 3258 * modifying it. We either need to use the copy buffer and/or we will end up 3259 * with a partial bind. 3260 */ 3261 static int 3262 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3263 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3264 { 3265 rootnex_sglinfo_t *sinfo; 3266 rootnex_window_t *window; 3267 ddi_dma_cookie_t *cookie; 3268 size_t copybuf_used; 3269 size_t dmac_size; 3270 boolean_t partial; 3271 off_t cur_offset; 3272 page_t *cur_pp; 3273 major_t mnum; 3274 int e; 3275 int i; 3276 3277 3278 sinfo = &dma->dp_sglinfo; 3279 copybuf_used = 0; 3280 partial = B_FALSE; 3281 3282 /* 3283 * If we're using the copybuf, set the copybuf state in dma struct. 3284 * Needs to be first since it sets the copy buffer size. 3285 */ 3286 if (sinfo->si_copybuf_req != 0) { 3287 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3288 if (e != DDI_SUCCESS) { 3289 return (e); 3290 } 3291 } else { 3292 dma->dp_copybuf_size = 0; 3293 } 3294 3295 /* 3296 * Figure out if we need to do a partial mapping. If so, figure out 3297 * if we need to trim the buffers when we munge the sgl. 3298 */ 3299 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3300 (dmao->dmao_size > dma->dp_maxxfer) || 3301 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3302 dma->dp_partial_required = B_TRUE; 3303 if (attr->dma_attr_granular != 1) { 3304 dma->dp_trim_required = B_TRUE; 3305 } 3306 } else { 3307 dma->dp_partial_required = B_FALSE; 3308 dma->dp_trim_required = B_FALSE; 3309 } 3310 3311 /* If we need to do a partial bind, make sure the driver supports it */ 3312 if (dma->dp_partial_required && 3313 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3314 3315 mnum = ddi_driver_major(dma->dp_dip); 3316 /* 3317 * patchable which allows us to print one warning per major 3318 * number. 3319 */ 3320 if ((rootnex_bind_warn) && 3321 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3322 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3323 cmn_err(CE_WARN, "!%s: coding error detected, the " 3324 "driver is using ddi_dma_attr(9S) incorrectly. " 3325 "There is a small risk of data corruption in " 3326 "particular with large I/Os. The driver should be " 3327 "replaced with a corrected version for proper " 3328 "system operation. To disable this warning, add " 3329 "'set rootnex:rootnex_bind_warn=0' to " 3330 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3331 } 3332 return (DDI_DMA_TOOBIG); 3333 } 3334 3335 /* 3336 * we might need multiple windows, setup state to handle them. In this 3337 * code path, we will have at least one window. 3338 */ 3339 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3340 if (e != DDI_SUCCESS) { 3341 rootnex_teardown_copybuf(dma); 3342 return (e); 3343 } 3344 3345 window = &dma->dp_window[0]; 3346 cookie = &dma->dp_cookies[0]; 3347 cur_offset = 0; 3348 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3349 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3350 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3351 } 3352 3353 /* loop though all the cookies we got back from get_sgl() */ 3354 for (i = 0; i < sinfo->si_sgl_size; i++) { 3355 /* 3356 * If we're using the copy buffer, check this cookie and setup 3357 * its associated copy buffer state. If this cookie uses the 3358 * copy buffer, make sure we sync this window during dma_sync. 3359 */ 3360 if (dma->dp_copybuf_size > 0) { 3361 rootnex_setup_cookie(dmao, dma, cookie, 3362 cur_offset, ©buf_used, &cur_pp); 3363 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3364 window->wd_dosync = B_TRUE; 3365 } 3366 } 3367 3368 /* 3369 * save away the cookie size, since it could be modified in 3370 * the windowing code. 3371 */ 3372 dmac_size = cookie->dmac_size; 3373 3374 /* if we went over max copybuf size */ 3375 if (dma->dp_copybuf_size && 3376 (copybuf_used > dma->dp_copybuf_size)) { 3377 partial = B_TRUE; 3378 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3379 cookie, cur_offset, ©buf_used); 3380 if (e != DDI_SUCCESS) { 3381 rootnex_teardown_copybuf(dma); 3382 rootnex_teardown_windows(dma); 3383 return (e); 3384 } 3385 3386 /* 3387 * if the coookie uses the copy buffer, make sure the 3388 * new window we just moved to is set to sync. 3389 */ 3390 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3391 window->wd_dosync = B_TRUE; 3392 } 3393 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3394 dma->dp_dip); 3395 3396 /* if the cookie cnt == max sgllen, move to the next window */ 3397 } else if (window->wd_cookie_cnt >= 3398 (unsigned)attr->dma_attr_sgllen) { 3399 partial = B_TRUE; 3400 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3401 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3402 cookie, attr, cur_offset); 3403 if (e != DDI_SUCCESS) { 3404 rootnex_teardown_copybuf(dma); 3405 rootnex_teardown_windows(dma); 3406 return (e); 3407 } 3408 3409 /* 3410 * if the coookie uses the copy buffer, make sure the 3411 * new window we just moved to is set to sync. 3412 */ 3413 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3414 window->wd_dosync = B_TRUE; 3415 } 3416 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3417 dma->dp_dip); 3418 3419 /* else if we will be over maxxfer */ 3420 } else if ((window->wd_size + dmac_size) > 3421 dma->dp_maxxfer) { 3422 partial = B_TRUE; 3423 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3424 cookie); 3425 if (e != DDI_SUCCESS) { 3426 rootnex_teardown_copybuf(dma); 3427 rootnex_teardown_windows(dma); 3428 return (e); 3429 } 3430 3431 /* 3432 * if the coookie uses the copy buffer, make sure the 3433 * new window we just moved to is set to sync. 3434 */ 3435 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3436 window->wd_dosync = B_TRUE; 3437 } 3438 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3439 dma->dp_dip); 3440 3441 /* else this cookie fits in the current window */ 3442 } else { 3443 window->wd_cookie_cnt++; 3444 window->wd_size += dmac_size; 3445 } 3446 3447 /* track our offset into the buffer, go to the next cookie */ 3448 ASSERT(dmac_size <= dmao->dmao_size); 3449 ASSERT(cookie->dmac_size <= dmac_size); 3450 cur_offset += dmac_size; 3451 cookie++; 3452 } 3453 3454 /* if we ended up with a zero sized window in the end, clean it up */ 3455 if (window->wd_size == 0) { 3456 hp->dmai_nwin--; 3457 window--; 3458 } 3459 3460 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3461 3462 if (!partial) { 3463 return (DDI_DMA_MAPPED); 3464 } 3465 3466 ASSERT(dma->dp_partial_required); 3467 return (DDI_DMA_PARTIAL_MAP); 3468 } 3469 3470 /* 3471 * rootnex_setup_copybuf() 3472 * Called in bind slowpath. Figures out if we're going to use the copy 3473 * buffer, and if we do, sets up the basic state to handle it. 3474 */ 3475 static int 3476 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3477 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3478 { 3479 rootnex_sglinfo_t *sinfo; 3480 ddi_dma_attr_t lattr; 3481 size_t max_copybuf; 3482 int cansleep; 3483 int e; 3484 #if !defined(__amd64) 3485 int vmflag; 3486 #endif 3487 3488 ASSERT(!dma->dp_dvma_used); 3489 3490 sinfo = &dma->dp_sglinfo; 3491 3492 /* read this first so it's consistent through the routine */ 3493 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3494 3495 /* We need to call into the rootnex on ddi_dma_sync() */ 3496 hp->dmai_rflags &= ~DMP_NOSYNC; 3497 3498 /* make sure the copybuf size <= the max size */ 3499 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3500 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3501 3502 #if !defined(__amd64) 3503 /* 3504 * if we don't have kva space to copy to/from, allocate the KVA space 3505 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3506 * the 64-bit kernel. 3507 */ 3508 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3509 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3510 3511 /* convert the sleep flags */ 3512 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3513 vmflag = VM_SLEEP; 3514 } else { 3515 vmflag = VM_NOSLEEP; 3516 } 3517 3518 /* allocate Kernel VA space that we can bcopy to/from */ 3519 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3520 vmflag); 3521 if (dma->dp_kva == NULL) { 3522 return (DDI_DMA_NORESOURCES); 3523 } 3524 } 3525 #endif 3526 3527 /* convert the sleep flags */ 3528 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3529 cansleep = 1; 3530 } else { 3531 cansleep = 0; 3532 } 3533 3534 /* 3535 * Allocate the actual copy buffer. This needs to fit within the DMA 3536 * engine limits, so we can't use kmem_alloc... We don't need 3537 * contiguous memory (sgllen) since we will be forcing windows on 3538 * sgllen anyway. 3539 */ 3540 lattr = *attr; 3541 lattr.dma_attr_align = MMU_PAGESIZE; 3542 lattr.dma_attr_sgllen = -1; /* no limit */ 3543 /* 3544 * if we're using the copy buffer because of seg, use that for our 3545 * upper address limit. 3546 */ 3547 if (sinfo->si_bounce_on_seg) { 3548 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3549 } 3550 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3551 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3552 if (e != DDI_SUCCESS) { 3553 #if !defined(__amd64) 3554 if (dma->dp_kva != NULL) { 3555 vmem_free(heap_arena, dma->dp_kva, 3556 dma->dp_copybuf_size); 3557 } 3558 #endif 3559 return (DDI_DMA_NORESOURCES); 3560 } 3561 3562 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3563 size_t, dma->dp_copybuf_size); 3564 3565 return (DDI_SUCCESS); 3566 } 3567 3568 3569 /* 3570 * rootnex_setup_windows() 3571 * Called in bind slowpath to setup the window state. We always have windows 3572 * in the slowpath. Even if the window count = 1. 3573 */ 3574 static int 3575 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3576 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3577 { 3578 rootnex_window_t *windowp; 3579 rootnex_sglinfo_t *sinfo; 3580 size_t copy_state_size; 3581 size_t win_state_size; 3582 size_t state_available; 3583 size_t space_needed; 3584 uint_t copybuf_win; 3585 uint_t maxxfer_win; 3586 size_t space_used; 3587 uint_t sglwin; 3588 3589 3590 sinfo = &dma->dp_sglinfo; 3591 3592 dma->dp_current_win = 0; 3593 hp->dmai_nwin = 0; 3594 3595 /* If we don't need to do a partial, we only have one window */ 3596 if (!dma->dp_partial_required) { 3597 dma->dp_max_win = 1; 3598 3599 /* 3600 * we need multiple windows, need to figure out the worse case number 3601 * of windows. 3602 */ 3603 } else { 3604 /* 3605 * if we need windows because we need more copy buffer that 3606 * we allow, the worse case number of windows we could need 3607 * here would be (copybuf space required / copybuf space that 3608 * we have) plus one for remainder, and plus 2 to handle the 3609 * extra pages on the trim for the first and last pages of the 3610 * buffer (a page is the minimum window size so under the right 3611 * attr settings, you could have a window for each page). 3612 * The last page will only be hit here if the size is not a 3613 * multiple of the granularity (which theoretically shouldn't 3614 * be the case but never has been enforced, so we could have 3615 * broken things without it). 3616 */ 3617 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3618 ASSERT(dma->dp_copybuf_size > 0); 3619 copybuf_win = (sinfo->si_copybuf_req / 3620 dma->dp_copybuf_size) + 1 + 2; 3621 } else { 3622 copybuf_win = 0; 3623 } 3624 3625 /* 3626 * if we need windows because we have more cookies than the H/W 3627 * can handle, the number of windows we would need here would 3628 * be (cookie count / cookies count H/W supports minus 1[for 3629 * trim]) plus one for remainder. 3630 */ 3631 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3632 sglwin = (sinfo->si_sgl_size / 3633 (attr->dma_attr_sgllen - 1)) + 1; 3634 } else { 3635 sglwin = 0; 3636 } 3637 3638 /* 3639 * if we need windows because we're binding more memory than the 3640 * H/W can transfer at once, the number of windows we would need 3641 * here would be (xfer count / max xfer H/W supports) plus one 3642 * for remainder, and plus 2 to handle the extra pages on the 3643 * trim (see above comment about trim) 3644 */ 3645 if (dmao->dmao_size > dma->dp_maxxfer) { 3646 maxxfer_win = (dmao->dmao_size / 3647 dma->dp_maxxfer) + 1 + 2; 3648 } else { 3649 maxxfer_win = 0; 3650 } 3651 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3652 ASSERT(dma->dp_max_win > 0); 3653 } 3654 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3655 3656 /* 3657 * Get space for window and potential copy buffer state. Before we 3658 * go and allocate memory, see if we can get away with using what's 3659 * left in the pre-allocted state or the dynamically allocated sgl. 3660 */ 3661 space_used = (uintptr_t)(sinfo->si_sgl_size * 3662 sizeof (ddi_dma_cookie_t)); 3663 3664 /* if we dynamically allocated space for the cookies */ 3665 if (dma->dp_need_to_free_cookie) { 3666 /* if we have more space in the pre-allocted buffer, use it */ 3667 ASSERT(space_used <= dma->dp_cookie_size); 3668 if ((dma->dp_cookie_size - space_used) <= 3669 rootnex_state->r_prealloc_size) { 3670 state_available = rootnex_state->r_prealloc_size; 3671 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3672 3673 /* 3674 * else, we have more free space in the dynamically allocated 3675 * buffer, i.e. the buffer wasn't worse case fragmented so we 3676 * didn't need a lot of cookies. 3677 */ 3678 } else { 3679 state_available = dma->dp_cookie_size - space_used; 3680 windowp = (rootnex_window_t *) 3681 &dma->dp_cookies[sinfo->si_sgl_size]; 3682 } 3683 3684 /* we used the pre-alloced buffer */ 3685 } else { 3686 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3687 state_available = rootnex_state->r_prealloc_size - space_used; 3688 windowp = (rootnex_window_t *) 3689 &dma->dp_cookies[sinfo->si_sgl_size]; 3690 } 3691 3692 /* 3693 * figure out how much state we need to track the copy buffer. Add an 3694 * addition 8 bytes for pointer alignemnt later. 3695 */ 3696 if (dma->dp_copybuf_size > 0) { 3697 copy_state_size = sinfo->si_max_pages * 3698 sizeof (rootnex_pgmap_t); 3699 } else { 3700 copy_state_size = 0; 3701 } 3702 /* add an additional 8 bytes for pointer alignment */ 3703 space_needed = win_state_size + copy_state_size + 0x8; 3704 3705 /* if we have enough space already, use it */ 3706 if (state_available >= space_needed) { 3707 dma->dp_window = windowp; 3708 dma->dp_need_to_free_window = B_FALSE; 3709 3710 /* not enough space, need to allocate more. */ 3711 } else { 3712 dma->dp_window = kmem_alloc(space_needed, kmflag); 3713 if (dma->dp_window == NULL) { 3714 return (DDI_DMA_NORESOURCES); 3715 } 3716 dma->dp_need_to_free_window = B_TRUE; 3717 dma->dp_window_size = space_needed; 3718 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3719 dma->dp_dip, size_t, space_needed); 3720 } 3721 3722 /* 3723 * we allocate copy buffer state and window state at the same time. 3724 * setup our copy buffer state pointers. Make sure it's aligned. 3725 */ 3726 if (dma->dp_copybuf_size > 0) { 3727 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3728 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3729 3730 #if !defined(__amd64) 3731 /* 3732 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3733 * false/NULL. Should be quicker to bzero vs loop and set. 3734 */ 3735 bzero(dma->dp_pgmap, copy_state_size); 3736 #endif 3737 } else { 3738 dma->dp_pgmap = NULL; 3739 } 3740 3741 return (DDI_SUCCESS); 3742 } 3743 3744 3745 /* 3746 * rootnex_teardown_copybuf() 3747 * cleans up after rootnex_setup_copybuf() 3748 */ 3749 static void 3750 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3751 { 3752 #if !defined(__amd64) 3753 int i; 3754 3755 /* 3756 * if we allocated kernel heap VMEM space, go through all the pages and 3757 * map out any of the ones that we're mapped into the kernel heap VMEM 3758 * arena. Then free the VMEM space. 3759 */ 3760 if (dma->dp_kva != NULL) { 3761 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3762 if (dma->dp_pgmap[i].pm_mapped) { 3763 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3764 MMU_PAGESIZE, HAT_UNLOAD); 3765 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3766 } 3767 } 3768 3769 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3770 } 3771 3772 #endif 3773 3774 /* if we allocated a copy buffer, free it */ 3775 if (dma->dp_cbaddr != NULL) { 3776 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3777 } 3778 } 3779 3780 3781 /* 3782 * rootnex_teardown_windows() 3783 * cleans up after rootnex_setup_windows() 3784 */ 3785 static void 3786 rootnex_teardown_windows(rootnex_dma_t *dma) 3787 { 3788 /* 3789 * if we had to allocate window state on the last bind (because we 3790 * didn't have enough pre-allocated space in the handle), free it. 3791 */ 3792 if (dma->dp_need_to_free_window) { 3793 kmem_free(dma->dp_window, dma->dp_window_size); 3794 } 3795 } 3796 3797 3798 /* 3799 * rootnex_init_win() 3800 * Called in bind slow path during creation of a new window. Initializes 3801 * window state to default values. 3802 */ 3803 /*ARGSUSED*/ 3804 static void 3805 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3806 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3807 { 3808 hp->dmai_nwin++; 3809 window->wd_dosync = B_FALSE; 3810 window->wd_offset = cur_offset; 3811 window->wd_size = 0; 3812 window->wd_first_cookie = cookie; 3813 window->wd_cookie_cnt = 0; 3814 window->wd_trim.tr_trim_first = B_FALSE; 3815 window->wd_trim.tr_trim_last = B_FALSE; 3816 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3817 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3818 #if !defined(__amd64) 3819 window->wd_remap_copybuf = dma->dp_cb_remaping; 3820 #endif 3821 } 3822 3823 3824 /* 3825 * rootnex_setup_cookie() 3826 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3827 * the sgl uses the copy buffer, we need to go through each cookie, figure 3828 * out if it uses the copy buffer, and if it does, save away everything we'll 3829 * need during sync. 3830 */ 3831 static void 3832 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3833 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3834 page_t **cur_pp) 3835 { 3836 boolean_t copybuf_sz_power_2; 3837 rootnex_sglinfo_t *sinfo; 3838 paddr_t paddr; 3839 uint_t pidx; 3840 uint_t pcnt; 3841 off_t poff; 3842 #if defined(__amd64) 3843 pfn_t pfn; 3844 #else 3845 page_t **pplist; 3846 #endif 3847 3848 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3849 3850 sinfo = &dma->dp_sglinfo; 3851 3852 /* 3853 * Calculate the page index relative to the start of the buffer. The 3854 * index to the current page for our buffer is the offset into the 3855 * first page of the buffer plus our current offset into the buffer 3856 * itself, shifted of course... 3857 */ 3858 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3859 ASSERT(pidx < sinfo->si_max_pages); 3860 3861 /* if this cookie uses the copy buffer */ 3862 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3863 /* 3864 * NOTE: we know that since this cookie uses the copy buffer, it 3865 * is <= MMU_PAGESIZE. 3866 */ 3867 3868 /* 3869 * get the offset into the page. For the 64-bit kernel, get the 3870 * pfn which we'll use with seg kpm. 3871 */ 3872 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3873 #if defined(__amd64) 3874 /* mfn_to_pfn() is a NOP on i86pc */ 3875 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3876 #endif /* __amd64 */ 3877 3878 /* figure out if the copybuf size is a power of 2 */ 3879 if (!ISP2(dma->dp_copybuf_size)) { 3880 copybuf_sz_power_2 = B_FALSE; 3881 } else { 3882 copybuf_sz_power_2 = B_TRUE; 3883 } 3884 3885 /* This page uses the copy buffer */ 3886 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3887 3888 /* 3889 * save the copy buffer KVA that we'll use with this page. 3890 * if we still fit within the copybuf, it's a simple add. 3891 * otherwise, we need to wrap over using & or % accordingly. 3892 */ 3893 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3894 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3895 *copybuf_used; 3896 } else { 3897 if (copybuf_sz_power_2) { 3898 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3899 (uintptr_t)dma->dp_cbaddr + 3900 (*copybuf_used & 3901 (dma->dp_copybuf_size - 1))); 3902 } else { 3903 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3904 (uintptr_t)dma->dp_cbaddr + 3905 (*copybuf_used % dma->dp_copybuf_size)); 3906 } 3907 } 3908 3909 /* 3910 * over write the cookie physical address with the address of 3911 * the physical address of the copy buffer page that we will 3912 * use. 3913 */ 3914 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3915 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3916 3917 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3918 3919 /* if we have a kernel VA, it's easy, just save that address */ 3920 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3921 (sinfo->si_asp == &kas)) { 3922 /* 3923 * save away the page aligned virtual address of the 3924 * driver buffer. Offsets are handled in the sync code. 3925 */ 3926 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3927 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3928 & MMU_PAGEMASK); 3929 #if !defined(__amd64) 3930 /* 3931 * we didn't need to, and will never need to map this 3932 * page. 3933 */ 3934 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3935 #endif 3936 3937 /* we don't have a kernel VA. We need one for the bcopy. */ 3938 } else { 3939 #if defined(__amd64) 3940 /* 3941 * for the 64-bit kernel, it's easy. We use seg kpm to 3942 * get a Kernel VA for the corresponding pfn. 3943 */ 3944 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3945 #else 3946 /* 3947 * for the 32-bit kernel, this is a pain. First we'll 3948 * save away the page_t or user VA for this page. This 3949 * is needed in rootnex_dma_win() when we switch to a 3950 * new window which requires us to re-map the copy 3951 * buffer. 3952 */ 3953 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3954 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3955 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3956 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3957 } else if (pplist != NULL) { 3958 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3959 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3960 } else { 3961 dma->dp_pgmap[pidx].pm_pp = NULL; 3962 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3963 (((uintptr_t) 3964 dmar_object->dmao_obj.virt_obj.v_addr + 3965 cur_offset) & MMU_PAGEMASK); 3966 } 3967 3968 /* 3969 * save away the page aligned virtual address which was 3970 * allocated from the kernel heap arena (taking into 3971 * account if we need more copy buffer than we alloced 3972 * and use multiple windows to handle this, i.e. &,%). 3973 * NOTE: there isn't and physical memory backing up this 3974 * virtual address space currently. 3975 */ 3976 if ((*copybuf_used + MMU_PAGESIZE) <= 3977 dma->dp_copybuf_size) { 3978 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3979 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3980 MMU_PAGEMASK); 3981 } else { 3982 if (copybuf_sz_power_2) { 3983 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3984 (((uintptr_t)dma->dp_kva + 3985 (*copybuf_used & 3986 (dma->dp_copybuf_size - 1))) & 3987 MMU_PAGEMASK); 3988 } else { 3989 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3990 (((uintptr_t)dma->dp_kva + 3991 (*copybuf_used % 3992 dma->dp_copybuf_size)) & 3993 MMU_PAGEMASK); 3994 } 3995 } 3996 3997 /* 3998 * if we haven't used up the available copy buffer yet, 3999 * map the kva to the physical page. 4000 */ 4001 if (!dma->dp_cb_remaping && ((*copybuf_used + 4002 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 4003 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 4004 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 4005 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 4006 dma->dp_pgmap[pidx].pm_kaddr); 4007 } else { 4008 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4009 sinfo->si_asp, 4010 dma->dp_pgmap[pidx].pm_kaddr); 4011 } 4012 4013 /* 4014 * we've used up the available copy buffer, this page 4015 * will have to be mapped during rootnex_dma_win() when 4016 * we switch to a new window which requires a re-map 4017 * the copy buffer. (32-bit kernel only) 4018 */ 4019 } else { 4020 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4021 } 4022 #endif 4023 /* go to the next page_t */ 4024 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4025 *cur_pp = (*cur_pp)->p_next; 4026 } 4027 } 4028 4029 /* add to the copy buffer count */ 4030 *copybuf_used += MMU_PAGESIZE; 4031 4032 /* 4033 * This cookie doesn't use the copy buffer. Walk through the pages this 4034 * cookie occupies to reflect this. 4035 */ 4036 } else { 4037 /* 4038 * figure out how many pages the cookie occupies. We need to 4039 * use the original page offset of the buffer and the cookies 4040 * offset in the buffer to do this. 4041 */ 4042 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4043 pcnt = mmu_btopr(cookie->dmac_size + poff); 4044 4045 while (pcnt > 0) { 4046 #if !defined(__amd64) 4047 /* 4048 * the 32-bit kernel doesn't have seg kpm, so we need 4049 * to map in the driver buffer (if it didn't come down 4050 * with a kernel VA) on the fly. Since this page doesn't 4051 * use the copy buffer, it's not, or will it ever, have 4052 * to be mapped in. 4053 */ 4054 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4055 #endif 4056 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4057 4058 /* 4059 * we need to update pidx and cur_pp or we'll loose 4060 * track of where we are. 4061 */ 4062 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4063 *cur_pp = (*cur_pp)->p_next; 4064 } 4065 pidx++; 4066 pcnt--; 4067 } 4068 } 4069 } 4070 4071 4072 /* 4073 * rootnex_sgllen_window_boundary() 4074 * Called in the bind slow path when the next cookie causes us to exceed (in 4075 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4076 * length supported by the DMA H/W. 4077 */ 4078 static int 4079 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4080 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4081 off_t cur_offset) 4082 { 4083 off_t new_offset; 4084 size_t trim_sz; 4085 off_t coffset; 4086 4087 4088 /* 4089 * if we know we'll never have to trim, it's pretty easy. Just move to 4090 * the next window and init it. We're done. 4091 */ 4092 if (!dma->dp_trim_required) { 4093 (*windowp)++; 4094 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4095 (*windowp)->wd_cookie_cnt++; 4096 (*windowp)->wd_size = cookie->dmac_size; 4097 return (DDI_SUCCESS); 4098 } 4099 4100 /* figure out how much we need to trim from the window */ 4101 ASSERT(attr->dma_attr_granular != 0); 4102 if (dma->dp_granularity_power_2) { 4103 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4104 } else { 4105 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4106 } 4107 4108 /* The window's a whole multiple of granularity. We're done */ 4109 if (trim_sz == 0) { 4110 (*windowp)++; 4111 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4112 (*windowp)->wd_cookie_cnt++; 4113 (*windowp)->wd_size = cookie->dmac_size; 4114 return (DDI_SUCCESS); 4115 } 4116 4117 /* 4118 * The window's not a whole multiple of granularity, since we know this 4119 * is due to the sgllen, we need to go back to the last cookie and trim 4120 * that one, add the left over part of the old cookie into the new 4121 * window, and then add in the new cookie into the new window. 4122 */ 4123 4124 /* 4125 * make sure the driver isn't making us do something bad... Trimming and 4126 * sgllen == 1 don't go together. 4127 */ 4128 if (attr->dma_attr_sgllen == 1) { 4129 return (DDI_DMA_NOMAPPING); 4130 } 4131 4132 /* 4133 * first, setup the current window to account for the trim. Need to go 4134 * back to the last cookie for this. 4135 */ 4136 cookie--; 4137 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4138 (*windowp)->wd_trim.tr_last_cookie = cookie; 4139 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4140 ASSERT(cookie->dmac_size > trim_sz); 4141 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4142 (*windowp)->wd_size -= trim_sz; 4143 4144 /* save the buffer offsets for the next window */ 4145 coffset = cookie->dmac_size - trim_sz; 4146 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4147 4148 /* 4149 * set this now in case this is the first window. all other cases are 4150 * set in dma_win() 4151 */ 4152 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4153 4154 /* 4155 * initialize the next window using what's left over in the previous 4156 * cookie. 4157 */ 4158 (*windowp)++; 4159 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4160 (*windowp)->wd_cookie_cnt++; 4161 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4162 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4163 (*windowp)->wd_trim.tr_first_size = trim_sz; 4164 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4165 (*windowp)->wd_dosync = B_TRUE; 4166 } 4167 4168 /* 4169 * now go back to the current cookie and add it to the new window. set 4170 * the new window size to the what was left over from the previous 4171 * cookie and what's in the current cookie. 4172 */ 4173 cookie++; 4174 (*windowp)->wd_cookie_cnt++; 4175 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4176 4177 /* 4178 * trim plus the next cookie could put us over maxxfer (a cookie can be 4179 * a max size of maxxfer). Handle that case. 4180 */ 4181 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4182 /* 4183 * maxxfer is already a whole multiple of granularity, and this 4184 * trim will be <= the previous trim (since a cookie can't be 4185 * larger than maxxfer). Make things simple here. 4186 */ 4187 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4188 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4189 (*windowp)->wd_trim.tr_last_cookie = cookie; 4190 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4191 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4192 (*windowp)->wd_size -= trim_sz; 4193 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4194 4195 /* save the buffer offsets for the next window */ 4196 coffset = cookie->dmac_size - trim_sz; 4197 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4198 4199 /* setup the next window */ 4200 (*windowp)++; 4201 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4202 (*windowp)->wd_cookie_cnt++; 4203 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4204 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4205 coffset; 4206 (*windowp)->wd_trim.tr_first_size = trim_sz; 4207 } 4208 4209 return (DDI_SUCCESS); 4210 } 4211 4212 4213 /* 4214 * rootnex_copybuf_window_boundary() 4215 * Called in bind slowpath when we get to a window boundary because we used 4216 * up all the copy buffer that we have. 4217 */ 4218 static int 4219 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4220 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4221 size_t *copybuf_used) 4222 { 4223 rootnex_sglinfo_t *sinfo; 4224 off_t new_offset; 4225 size_t trim_sz; 4226 paddr_t paddr; 4227 off_t coffset; 4228 uint_t pidx; 4229 off_t poff; 4230 4231 pidx = 0; 4232 sinfo = &dma->dp_sglinfo; 4233 4234 /* 4235 * the copy buffer should be a whole multiple of page size. We know that 4236 * this cookie is <= MMU_PAGESIZE. 4237 */ 4238 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4239 4240 /* 4241 * from now on, all new windows in this bind need to be re-mapped during 4242 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4243 * space... 4244 */ 4245 #if !defined(__amd64) 4246 dma->dp_cb_remaping = B_TRUE; 4247 #endif 4248 4249 /* reset copybuf used */ 4250 *copybuf_used = 0; 4251 4252 /* 4253 * if we don't have to trim (since granularity is set to 1), go to the 4254 * next window and add the current cookie to it. We know the current 4255 * cookie uses the copy buffer since we're in this code path. 4256 */ 4257 if (!dma->dp_trim_required) { 4258 (*windowp)++; 4259 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4260 4261 /* Add this cookie to the new window */ 4262 (*windowp)->wd_cookie_cnt++; 4263 (*windowp)->wd_size += cookie->dmac_size; 4264 *copybuf_used += MMU_PAGESIZE; 4265 return (DDI_SUCCESS); 4266 } 4267 4268 /* 4269 * *** may need to trim, figure it out. 4270 */ 4271 4272 /* figure out how much we need to trim from the window */ 4273 if (dma->dp_granularity_power_2) { 4274 trim_sz = (*windowp)->wd_size & 4275 (hp->dmai_attr.dma_attr_granular - 1); 4276 } else { 4277 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4278 } 4279 4280 /* 4281 * if the window's a whole multiple of granularity, go to the next 4282 * window, init it, then add in the current cookie. We know the current 4283 * cookie uses the copy buffer since we're in this code path. 4284 */ 4285 if (trim_sz == 0) { 4286 (*windowp)++; 4287 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4288 4289 /* Add this cookie to the new window */ 4290 (*windowp)->wd_cookie_cnt++; 4291 (*windowp)->wd_size += cookie->dmac_size; 4292 *copybuf_used += MMU_PAGESIZE; 4293 return (DDI_SUCCESS); 4294 } 4295 4296 /* 4297 * *** We figured it out, we definitly need to trim 4298 */ 4299 4300 /* 4301 * make sure the driver isn't making us do something bad... 4302 * Trimming and sgllen == 1 don't go together. 4303 */ 4304 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4305 return (DDI_DMA_NOMAPPING); 4306 } 4307 4308 /* 4309 * first, setup the current window to account for the trim. Need to go 4310 * back to the last cookie for this. Some of the last cookie will be in 4311 * the current window, and some of the last cookie will be in the new 4312 * window. All of the current cookie will be in the new window. 4313 */ 4314 cookie--; 4315 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4316 (*windowp)->wd_trim.tr_last_cookie = cookie; 4317 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4318 ASSERT(cookie->dmac_size > trim_sz); 4319 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4320 (*windowp)->wd_size -= trim_sz; 4321 4322 /* 4323 * we're trimming the last cookie (not the current cookie). So that 4324 * last cookie may have or may not have been using the copy buffer ( 4325 * we know the cookie passed in uses the copy buffer since we're in 4326 * this code path). 4327 * 4328 * If the last cookie doesn't use the copy buffer, nothing special to 4329 * do. However, if it does uses the copy buffer, it will be both the 4330 * last page in the current window and the first page in the next 4331 * window. Since we are reusing the copy buffer (and KVA space on the 4332 * 32-bit kernel), this page will use the end of the copy buffer in the 4333 * current window, and the start of the copy buffer in the next window. 4334 * Track that info... The cookie physical address was already set to 4335 * the copy buffer physical address in setup_cookie.. 4336 */ 4337 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4338 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4339 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4340 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4341 (*windowp)->wd_trim.tr_last_pidx = pidx; 4342 (*windowp)->wd_trim.tr_last_cbaddr = 4343 dma->dp_pgmap[pidx].pm_cbaddr; 4344 #if !defined(__amd64) 4345 (*windowp)->wd_trim.tr_last_kaddr = 4346 dma->dp_pgmap[pidx].pm_kaddr; 4347 #endif 4348 } 4349 4350 /* save the buffer offsets for the next window */ 4351 coffset = cookie->dmac_size - trim_sz; 4352 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4353 4354 /* 4355 * set this now in case this is the first window. all other cases are 4356 * set in dma_win() 4357 */ 4358 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4359 4360 /* 4361 * initialize the next window using what's left over in the previous 4362 * cookie. 4363 */ 4364 (*windowp)++; 4365 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4366 (*windowp)->wd_cookie_cnt++; 4367 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4368 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4369 (*windowp)->wd_trim.tr_first_size = trim_sz; 4370 4371 /* 4372 * again, we're tracking if the last cookie uses the copy buffer. 4373 * read the comment above for more info on why we need to track 4374 * additional state. 4375 * 4376 * For the first cookie in the new window, we need reset the physical 4377 * address to DMA into to the start of the copy buffer plus any 4378 * initial page offset which may be present. 4379 */ 4380 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4381 (*windowp)->wd_dosync = B_TRUE; 4382 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4383 (*windowp)->wd_trim.tr_first_pidx = pidx; 4384 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4385 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4386 4387 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4388 poff; 4389 (*windowp)->wd_trim.tr_first_paddr = 4390 ROOTNEX_PADDR_TO_RBASE(paddr); 4391 4392 #if !defined(__amd64) 4393 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4394 #endif 4395 /* account for the cookie copybuf usage in the new window */ 4396 *copybuf_used += MMU_PAGESIZE; 4397 4398 /* 4399 * every piece of code has to have a hack, and here is this 4400 * ones :-) 4401 * 4402 * There is a complex interaction between setup_cookie and the 4403 * copybuf window boundary. The complexity had to be in either 4404 * the maxxfer window, or the copybuf window, and I chose the 4405 * copybuf code. 4406 * 4407 * So in this code path, we have taken the last cookie, 4408 * virtually broken it in half due to the trim, and it happens 4409 * to use the copybuf which further complicates life. At the 4410 * same time, we have already setup the current cookie, which 4411 * is now wrong. More background info: the current cookie uses 4412 * the copybuf, so it is only a page long max. So we need to 4413 * fix the current cookies copy buffer address, physical 4414 * address, and kva for the 32-bit kernel. We due this by 4415 * bumping them by page size (of course, we can't due this on 4416 * the physical address since the copy buffer may not be 4417 * physically contiguous). 4418 */ 4419 cookie++; 4420 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4421 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4422 4423 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4424 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4425 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4426 4427 #if !defined(__amd64) 4428 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4429 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4430 #endif 4431 } else { 4432 /* go back to the current cookie */ 4433 cookie++; 4434 } 4435 4436 /* 4437 * add the current cookie to the new window. set the new window size to 4438 * the what was left over from the previous cookie and what's in the 4439 * current cookie. 4440 */ 4441 (*windowp)->wd_cookie_cnt++; 4442 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4443 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4444 4445 /* 4446 * we know that the cookie passed in always uses the copy buffer. We 4447 * wouldn't be here if it didn't. 4448 */ 4449 *copybuf_used += MMU_PAGESIZE; 4450 4451 return (DDI_SUCCESS); 4452 } 4453 4454 4455 /* 4456 * rootnex_maxxfer_window_boundary() 4457 * Called in bind slowpath when we get to a window boundary because we will 4458 * go over maxxfer. 4459 */ 4460 static int 4461 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4462 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4463 { 4464 size_t dmac_size; 4465 off_t new_offset; 4466 size_t trim_sz; 4467 off_t coffset; 4468 4469 4470 /* 4471 * calculate how much we have to trim off of the current cookie to equal 4472 * maxxfer. We don't have to account for granularity here since our 4473 * maxxfer already takes that into account. 4474 */ 4475 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4476 ASSERT(trim_sz <= cookie->dmac_size); 4477 ASSERT(trim_sz <= dma->dp_maxxfer); 4478 4479 /* save cookie size since we need it later and we might change it */ 4480 dmac_size = cookie->dmac_size; 4481 4482 /* 4483 * if we're not trimming the entire cookie, setup the current window to 4484 * account for the trim. 4485 */ 4486 if (trim_sz < cookie->dmac_size) { 4487 (*windowp)->wd_cookie_cnt++; 4488 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4489 (*windowp)->wd_trim.tr_last_cookie = cookie; 4490 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4491 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4492 (*windowp)->wd_size = dma->dp_maxxfer; 4493 4494 /* 4495 * set the adjusted cookie size now in case this is the first 4496 * window. All other windows are taken care of in get win 4497 */ 4498 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4499 } 4500 4501 /* 4502 * coffset is the current offset within the cookie, new_offset is the 4503 * current offset with the entire buffer. 4504 */ 4505 coffset = dmac_size - trim_sz; 4506 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4507 4508 /* initialize the next window */ 4509 (*windowp)++; 4510 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4511 (*windowp)->wd_cookie_cnt++; 4512 (*windowp)->wd_size = trim_sz; 4513 if (trim_sz < dmac_size) { 4514 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4515 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4516 coffset; 4517 (*windowp)->wd_trim.tr_first_size = trim_sz; 4518 } 4519 4520 return (DDI_SUCCESS); 4521 } 4522 4523 4524 /*ARGSUSED*/ 4525 static int 4526 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4527 off_t off, size_t len, uint_t cache_flags) 4528 { 4529 rootnex_sglinfo_t *sinfo; 4530 rootnex_pgmap_t *cbpage; 4531 rootnex_window_t *win; 4532 ddi_dma_impl_t *hp; 4533 rootnex_dma_t *dma; 4534 caddr_t fromaddr; 4535 caddr_t toaddr; 4536 uint_t psize; 4537 off_t offset; 4538 uint_t pidx; 4539 size_t size; 4540 off_t poff; 4541 int e; 4542 4543 4544 hp = (ddi_dma_impl_t *)handle; 4545 dma = (rootnex_dma_t *)hp->dmai_private; 4546 sinfo = &dma->dp_sglinfo; 4547 4548 /* 4549 * if we don't have any windows, we don't need to sync. A copybuf 4550 * will cause us to have at least one window. 4551 */ 4552 if (dma->dp_window == NULL) { 4553 return (DDI_SUCCESS); 4554 } 4555 4556 /* This window may not need to be sync'd */ 4557 win = &dma->dp_window[dma->dp_current_win]; 4558 if (!win->wd_dosync) { 4559 return (DDI_SUCCESS); 4560 } 4561 4562 /* handle off and len special cases */ 4563 if ((off == 0) || (rootnex_sync_ignore_params)) { 4564 offset = win->wd_offset; 4565 } else { 4566 offset = off; 4567 } 4568 if ((len == 0) || (rootnex_sync_ignore_params)) { 4569 size = win->wd_size; 4570 } else { 4571 size = len; 4572 } 4573 4574 /* check the sync args to make sure they make a little sense */ 4575 if (rootnex_sync_check_parms) { 4576 e = rootnex_valid_sync_parms(hp, win, offset, size, 4577 cache_flags); 4578 if (e != DDI_SUCCESS) { 4579 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4580 return (DDI_FAILURE); 4581 } 4582 } 4583 4584 /* 4585 * special case the first page to handle the offset into the page. The 4586 * offset to the current page for our buffer is the offset into the 4587 * first page of the buffer plus our current offset into the buffer 4588 * itself, masked of course. 4589 */ 4590 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4591 psize = MIN((MMU_PAGESIZE - poff), size); 4592 4593 /* go through all the pages that we want to sync */ 4594 while (size > 0) { 4595 /* 4596 * Calculate the page index relative to the start of the buffer. 4597 * The index to the current page for our buffer is the offset 4598 * into the first page of the buffer plus our current offset 4599 * into the buffer itself, shifted of course... 4600 */ 4601 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4602 ASSERT(pidx < sinfo->si_max_pages); 4603 4604 /* 4605 * if this page uses the copy buffer, we need to sync it, 4606 * otherwise, go on to the next page. 4607 */ 4608 cbpage = &dma->dp_pgmap[pidx]; 4609 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4610 (cbpage->pm_uses_copybuf == B_FALSE)); 4611 if (cbpage->pm_uses_copybuf) { 4612 /* cbaddr and kaddr should be page aligned */ 4613 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4614 MMU_PAGEOFFSET) == 0); 4615 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4616 MMU_PAGEOFFSET) == 0); 4617 4618 /* 4619 * if we're copying for the device, we are going to 4620 * copy from the drivers buffer and to the rootnex 4621 * allocated copy buffer. 4622 */ 4623 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4624 fromaddr = cbpage->pm_kaddr + poff; 4625 toaddr = cbpage->pm_cbaddr + poff; 4626 ROOTNEX_DPROBE2(rootnex__sync__dev, 4627 dev_info_t *, dma->dp_dip, size_t, psize); 4628 4629 /* 4630 * if we're copying for the cpu/kernel, we are going to 4631 * copy from the rootnex allocated copy buffer to the 4632 * drivers buffer. 4633 */ 4634 } else { 4635 fromaddr = cbpage->pm_cbaddr + poff; 4636 toaddr = cbpage->pm_kaddr + poff; 4637 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4638 dev_info_t *, dma->dp_dip, size_t, psize); 4639 } 4640 4641 bcopy(fromaddr, toaddr, psize); 4642 } 4643 4644 /* 4645 * decrement size until we're done, update our offset into the 4646 * buffer, and get the next page size. 4647 */ 4648 size -= psize; 4649 offset += psize; 4650 psize = MIN(MMU_PAGESIZE, size); 4651 4652 /* page offset is zero for the rest of this loop */ 4653 poff = 0; 4654 } 4655 4656 return (DDI_SUCCESS); 4657 } 4658 4659 /* 4660 * rootnex_dma_sync() 4661 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4662 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4663 * is set, ddi_dma_sync() returns immediately passing back success. 4664 */ 4665 /*ARGSUSED*/ 4666 static int 4667 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4668 off_t off, size_t len, uint_t cache_flags) 4669 { 4670 #if defined(__amd64) && !defined(__xpv) 4671 if (IOMMU_USED(rdip)) { 4672 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4673 cache_flags)); 4674 } 4675 #endif 4676 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4677 cache_flags)); 4678 } 4679 4680 /* 4681 * rootnex_valid_sync_parms() 4682 * checks the parameters passed to sync to verify they are correct. 4683 */ 4684 static int 4685 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4686 off_t offset, size_t size, uint_t cache_flags) 4687 { 4688 off_t woffset; 4689 4690 4691 /* 4692 * the first part of the test to make sure the offset passed in is 4693 * within the window. 4694 */ 4695 if (offset < win->wd_offset) { 4696 return (DDI_FAILURE); 4697 } 4698 4699 /* 4700 * second and last part of the test to make sure the offset and length 4701 * passed in is within the window. 4702 */ 4703 woffset = offset - win->wd_offset; 4704 if ((woffset + size) > win->wd_size) { 4705 return (DDI_FAILURE); 4706 } 4707 4708 /* 4709 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4710 * be set too. 4711 */ 4712 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4713 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4714 return (DDI_SUCCESS); 4715 } 4716 4717 /* 4718 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4719 * should be set. Also DDI_DMA_READ should be set in the flags. 4720 */ 4721 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4722 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4723 (hp->dmai_rflags & DDI_DMA_READ)) { 4724 return (DDI_SUCCESS); 4725 } 4726 4727 return (DDI_FAILURE); 4728 } 4729 4730 4731 /*ARGSUSED*/ 4732 static int 4733 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4734 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4735 uint_t *ccountp) 4736 { 4737 rootnex_window_t *window; 4738 rootnex_trim_t *trim; 4739 ddi_dma_impl_t *hp; 4740 rootnex_dma_t *dma; 4741 ddi_dma_obj_t *dmao; 4742 #if !defined(__amd64) 4743 rootnex_sglinfo_t *sinfo; 4744 rootnex_pgmap_t *pmap; 4745 uint_t pidx; 4746 uint_t pcnt; 4747 off_t poff; 4748 int i; 4749 #endif 4750 4751 4752 hp = (ddi_dma_impl_t *)handle; 4753 dma = (rootnex_dma_t *)hp->dmai_private; 4754 #if !defined(__amd64) 4755 sinfo = &dma->dp_sglinfo; 4756 #endif 4757 4758 /* If we try and get a window which doesn't exist, return failure */ 4759 if (win >= hp->dmai_nwin) { 4760 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4761 return (DDI_FAILURE); 4762 } 4763 4764 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4765 4766 /* 4767 * if we don't have any windows, and they're asking for the first 4768 * window, setup the cookie pointer to the first cookie in the bind. 4769 * setup our return values, then increment the cookie since we return 4770 * the first cookie on the stack. 4771 */ 4772 if (dma->dp_window == NULL) { 4773 if (win != 0) { 4774 ROOTNEX_DPROF_INC( 4775 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4776 return (DDI_FAILURE); 4777 } 4778 hp->dmai_cookie = dma->dp_cookies; 4779 *offp = 0; 4780 *lenp = dmao->dmao_size; 4781 *ccountp = dma->dp_sglinfo.si_sgl_size; 4782 *cookiep = hp->dmai_cookie[0]; 4783 hp->dmai_cookie++; 4784 hp->dmai_ncookies = *ccountp; 4785 hp->dmai_curcookie = 1; 4786 return (DDI_SUCCESS); 4787 } 4788 4789 /* sync the old window before moving on to the new one */ 4790 window = &dma->dp_window[dma->dp_current_win]; 4791 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4792 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4793 DDI_DMA_SYNC_FORCPU); 4794 } 4795 4796 #if !defined(__amd64) 4797 /* 4798 * before we move to the next window, if we need to re-map, unmap all 4799 * the pages in this window. 4800 */ 4801 if (dma->dp_cb_remaping) { 4802 /* 4803 * If we switch to this window again, we'll need to map in 4804 * on the fly next time. 4805 */ 4806 window->wd_remap_copybuf = B_TRUE; 4807 4808 /* 4809 * calculate the page index into the buffer where this window 4810 * starts, and the number of pages this window takes up. 4811 */ 4812 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4813 MMU_PAGESHIFT; 4814 poff = (sinfo->si_buf_offset + window->wd_offset) & 4815 MMU_PAGEOFFSET; 4816 pcnt = mmu_btopr(window->wd_size + poff); 4817 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4818 4819 /* unmap pages which are currently mapped in this window */ 4820 for (i = 0; i < pcnt; i++) { 4821 if (dma->dp_pgmap[pidx].pm_mapped) { 4822 hat_unload(kas.a_hat, 4823 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4824 HAT_UNLOAD); 4825 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4826 } 4827 pidx++; 4828 } 4829 } 4830 #endif 4831 4832 /* 4833 * Move to the new window. 4834 * NOTE: current_win must be set for sync to work right 4835 */ 4836 dma->dp_current_win = win; 4837 window = &dma->dp_window[win]; 4838 4839 /* if needed, adjust the first and/or last cookies for trim */ 4840 trim = &window->wd_trim; 4841 if (trim->tr_trim_first) { 4842 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4843 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4844 #if !defined(__amd64) 4845 window->wd_first_cookie->dmac_type = 4846 (window->wd_first_cookie->dmac_type & 4847 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4848 #endif 4849 if (trim->tr_first_copybuf_win) { 4850 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4851 trim->tr_first_cbaddr; 4852 #if !defined(__amd64) 4853 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4854 trim->tr_first_kaddr; 4855 #endif 4856 } 4857 } 4858 if (trim->tr_trim_last) { 4859 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4860 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4861 if (trim->tr_last_copybuf_win) { 4862 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4863 trim->tr_last_cbaddr; 4864 #if !defined(__amd64) 4865 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4866 trim->tr_last_kaddr; 4867 #endif 4868 } 4869 } 4870 4871 /* 4872 * setup the cookie pointer to the first cookie in the window. setup 4873 * our return values, then increment the cookie since we return the 4874 * first cookie on the stack. 4875 */ 4876 hp->dmai_cookie = window->wd_first_cookie; 4877 *offp = window->wd_offset; 4878 *lenp = window->wd_size; 4879 *ccountp = window->wd_cookie_cnt; 4880 *cookiep = hp->dmai_cookie[0]; 4881 hp->dmai_ncookies = *ccountp; 4882 hp->dmai_curcookie = 1; 4883 hp->dmai_cookie++; 4884 4885 #if !defined(__amd64) 4886 /* re-map copybuf if required for this window */ 4887 if (dma->dp_cb_remaping) { 4888 /* 4889 * calculate the page index into the buffer where this 4890 * window starts. 4891 */ 4892 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4893 MMU_PAGESHIFT; 4894 ASSERT(pidx < sinfo->si_max_pages); 4895 4896 /* 4897 * the first page can get unmapped if it's shared with the 4898 * previous window. Even if the rest of this window is already 4899 * mapped in, we need to still check this one. 4900 */ 4901 pmap = &dma->dp_pgmap[pidx]; 4902 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4903 if (pmap->pm_pp != NULL) { 4904 pmap->pm_mapped = B_TRUE; 4905 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4906 } else if (pmap->pm_vaddr != NULL) { 4907 pmap->pm_mapped = B_TRUE; 4908 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4909 pmap->pm_kaddr); 4910 } 4911 } 4912 pidx++; 4913 4914 /* map in the rest of the pages if required */ 4915 if (window->wd_remap_copybuf) { 4916 window->wd_remap_copybuf = B_FALSE; 4917 4918 /* figure out many pages this window takes up */ 4919 poff = (sinfo->si_buf_offset + window->wd_offset) & 4920 MMU_PAGEOFFSET; 4921 pcnt = mmu_btopr(window->wd_size + poff); 4922 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4923 4924 /* map pages which require it */ 4925 for (i = 1; i < pcnt; i++) { 4926 pmap = &dma->dp_pgmap[pidx]; 4927 if (pmap->pm_uses_copybuf) { 4928 ASSERT(pmap->pm_mapped == B_FALSE); 4929 if (pmap->pm_pp != NULL) { 4930 pmap->pm_mapped = B_TRUE; 4931 i86_pp_map(pmap->pm_pp, 4932 pmap->pm_kaddr); 4933 } else if (pmap->pm_vaddr != NULL) { 4934 pmap->pm_mapped = B_TRUE; 4935 i86_va_map(pmap->pm_vaddr, 4936 sinfo->si_asp, 4937 pmap->pm_kaddr); 4938 } 4939 } 4940 pidx++; 4941 } 4942 } 4943 } 4944 #endif 4945 4946 /* if the new window uses the copy buffer, sync it for the device */ 4947 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4948 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4949 DDI_DMA_SYNC_FORDEV); 4950 } 4951 4952 return (DDI_SUCCESS); 4953 } 4954 4955 /* 4956 * rootnex_dma_win() 4957 * called from ddi_dma_getwin() 4958 */ 4959 /*ARGSUSED*/ 4960 static int 4961 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4962 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4963 uint_t *ccountp) 4964 { 4965 #if defined(__amd64) && !defined(__xpv) 4966 if (IOMMU_USED(rdip)) { 4967 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4968 cookiep, ccountp)); 4969 } 4970 #endif 4971 4972 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4973 cookiep, ccountp)); 4974 } 4975 4976 #if defined(__amd64) && !defined(__xpv) 4977 /*ARGSUSED*/ 4978 static int 4979 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4980 ddi_dma_handle_t handle, void *v) 4981 { 4982 ddi_dma_impl_t *hp; 4983 rootnex_dma_t *dma; 4984 4985 hp = (ddi_dma_impl_t *)handle; 4986 dma = (rootnex_dma_t *)hp->dmai_private; 4987 dma->dp_iommu_private = v; 4988 4989 return (DDI_SUCCESS); 4990 } 4991 4992 /*ARGSUSED*/ 4993 static void * 4994 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4995 ddi_dma_handle_t handle) 4996 { 4997 ddi_dma_impl_t *hp; 4998 rootnex_dma_t *dma; 4999 5000 hp = (ddi_dma_impl_t *)handle; 5001 dma = (rootnex_dma_t *)hp->dmai_private; 5002 5003 return (dma->dp_iommu_private); 5004 } 5005 #endif 5006 5007 /* 5008 * ************************ 5009 * obsoleted dma routines 5010 * ************************ 5011 */ 5012 5013 /* 5014 * rootnex_dma_mctl() 5015 * 5016 * We don't support this legacy interface any more on x86. 5017 */ 5018 /* ARGSUSED */ 5019 static int 5020 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5021 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5022 uint_t cache_flags) 5023 { 5024 /* 5025 * The only thing dma_mctl is usef for anymore is legacy SPARC 5026 * dvma and sbus-specific routines. 5027 */ 5028 return (DDI_FAILURE); 5029 } 5030 5031 /* 5032 * ********* 5033 * FMA Code 5034 * ********* 5035 */ 5036 5037 /* 5038 * rootnex_fm_init() 5039 * FMA init busop 5040 */ 5041 /* ARGSUSED */ 5042 static int 5043 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5044 ddi_iblock_cookie_t *ibc) 5045 { 5046 *ibc = rootnex_state->r_err_ibc; 5047 5048 return (ddi_system_fmcap); 5049 } 5050 5051 /* 5052 * rootnex_dma_check() 5053 * Function called after a dma fault occurred to find out whether the 5054 * fault address is associated with a driver that is able to handle faults 5055 * and recover from faults. 5056 */ 5057 /* ARGSUSED */ 5058 static int 5059 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5060 const void *not_used) 5061 { 5062 rootnex_window_t *window; 5063 uint64_t start_addr; 5064 uint64_t fault_addr; 5065 ddi_dma_impl_t *hp; 5066 rootnex_dma_t *dma; 5067 uint64_t end_addr; 5068 size_t csize; 5069 int i; 5070 int j; 5071 5072 5073 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5074 hp = (ddi_dma_impl_t *)handle; 5075 ASSERT(hp); 5076 5077 dma = (rootnex_dma_t *)hp->dmai_private; 5078 5079 /* Get the address that we need to search for */ 5080 fault_addr = *(uint64_t *)addr; 5081 5082 /* 5083 * if we don't have any windows, we can just walk through all the 5084 * cookies. 5085 */ 5086 if (dma->dp_window == NULL) { 5087 /* for each cookie */ 5088 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5089 /* 5090 * if the faulted address is within the physical address 5091 * range of the cookie, return DDI_FM_NONFATAL. 5092 */ 5093 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5094 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5095 dma->dp_cookies[i].dmac_size))) { 5096 return (DDI_FM_NONFATAL); 5097 } 5098 } 5099 5100 /* fault_addr not within this DMA handle */ 5101 return (DDI_FM_UNKNOWN); 5102 } 5103 5104 /* we have mutiple windows, walk through each window */ 5105 for (i = 0; i < hp->dmai_nwin; i++) { 5106 window = &dma->dp_window[i]; 5107 5108 /* Go through all the cookies in the window */ 5109 for (j = 0; j < window->wd_cookie_cnt; j++) { 5110 5111 start_addr = window->wd_first_cookie[j].dmac_laddress; 5112 csize = window->wd_first_cookie[j].dmac_size; 5113 5114 /* 5115 * if we are trimming the first cookie in the window, 5116 * and this is the first cookie, adjust the start 5117 * address and size of the cookie to account for the 5118 * trim. 5119 */ 5120 if (window->wd_trim.tr_trim_first && (j == 0)) { 5121 start_addr = window->wd_trim.tr_first_paddr; 5122 csize = window->wd_trim.tr_first_size; 5123 } 5124 5125 /* 5126 * if we are trimming the last cookie in the window, 5127 * and this is the last cookie, adjust the start 5128 * address and size of the cookie to account for the 5129 * trim. 5130 */ 5131 if (window->wd_trim.tr_trim_last && 5132 (j == (window->wd_cookie_cnt - 1))) { 5133 start_addr = window->wd_trim.tr_last_paddr; 5134 csize = window->wd_trim.tr_last_size; 5135 } 5136 5137 end_addr = start_addr + csize; 5138 5139 /* 5140 * if the faulted address is within the physical 5141 * address of the cookie, return DDI_FM_NONFATAL. 5142 */ 5143 if ((fault_addr >= start_addr) && 5144 (fault_addr <= end_addr)) { 5145 return (DDI_FM_NONFATAL); 5146 } 5147 } 5148 } 5149 5150 /* fault_addr not within this DMA handle */ 5151 return (DDI_FM_UNKNOWN); 5152 } 5153 5154 /*ARGSUSED*/ 5155 static int 5156 rootnex_quiesce(dev_info_t *dip) 5157 { 5158 #if defined(__amd64) && !defined(__xpv) 5159 return (immu_quiesce()); 5160 #else 5161 return (DDI_SUCCESS); 5162 #endif 5163 } 5164 5165 #if defined(__xpv) 5166 void 5167 immu_init(void) 5168 { 5169 ; 5170 } 5171 5172 void 5173 immu_startup(void) 5174 { 5175 ; 5176 } 5177 /*ARGSUSED*/ 5178 void 5179 immu_physmem_update(uint64_t addr, uint64_t size) 5180 { 5181 ; 5182 } 5183 #endif 5184