1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 */ 28 29 /* 30 * x86 root nexus driver 31 */ 32 33 #include <sys/sysmacros.h> 34 #include <sys/conf.h> 35 #include <sys/autoconf.h> 36 #include <sys/sysmacros.h> 37 #include <sys/debug.h> 38 #include <sys/psw.h> 39 #include <sys/ddidmareq.h> 40 #include <sys/promif.h> 41 #include <sys/devops.h> 42 #include <sys/kmem.h> 43 #include <sys/cmn_err.h> 44 #include <vm/seg.h> 45 #include <vm/seg_kmem.h> 46 #include <vm/seg_dev.h> 47 #include <sys/vmem.h> 48 #include <sys/mman.h> 49 #include <vm/hat.h> 50 #include <vm/as.h> 51 #include <vm/page.h> 52 #include <sys/avintr.h> 53 #include <sys/errno.h> 54 #include <sys/modctl.h> 55 #include <sys/ddi_impldefs.h> 56 #include <sys/sunddi.h> 57 #include <sys/sunndi.h> 58 #include <sys/mach_intr.h> 59 #include <sys/psm.h> 60 #include <sys/ontrap.h> 61 #include <sys/atomic.h> 62 #include <sys/sdt.h> 63 #include <sys/rootnex.h> 64 #include <vm/hat_i86.h> 65 #include <sys/ddifm.h> 66 #include <sys/ddi_isa.h> 67 #include <sys/apic.h> 68 69 #ifdef __xpv 70 #include <sys/bootinfo.h> 71 #include <sys/hypervisor.h> 72 #include <sys/bootconf.h> 73 #include <vm/kboot_mmu.h> 74 #endif 75 76 #if defined(__amd64) && !defined(__xpv) 77 #include <sys/immu.h> 78 #endif 79 80 81 /* 82 * enable/disable extra checking of function parameters. Useful for debugging 83 * drivers. 84 */ 85 #ifdef DEBUG 86 int rootnex_alloc_check_parms = 1; 87 int rootnex_bind_check_parms = 1; 88 int rootnex_bind_check_inuse = 1; 89 int rootnex_unbind_verify_buffer = 0; 90 int rootnex_sync_check_parms = 1; 91 #else 92 int rootnex_alloc_check_parms = 0; 93 int rootnex_bind_check_parms = 0; 94 int rootnex_bind_check_inuse = 0; 95 int rootnex_unbind_verify_buffer = 0; 96 int rootnex_sync_check_parms = 0; 97 #endif 98 99 boolean_t rootnex_dmar_not_setup; 100 101 /* Master Abort and Target Abort panic flag */ 102 int rootnex_fm_ma_ta_panic_flag = 0; 103 104 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 105 int rootnex_bind_fail = 1; 106 int rootnex_bind_warn = 1; 107 uint8_t *rootnex_warn_list; 108 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 109 #define ROOTNEX_BIND_WARNING (0x1 << 0) 110 111 /* 112 * revert back to old broken behavior of always sync'ing entire copy buffer. 113 * This is useful if be have a buggy driver which doesn't correctly pass in 114 * the offset and size into ddi_dma_sync(). 115 */ 116 int rootnex_sync_ignore_params = 0; 117 118 /* 119 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 120 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 121 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 122 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 123 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 124 * (< 8K). We will still need to allocate the copy buffer during bind though 125 * (if we need one). These can only be modified in /etc/system before rootnex 126 * attach. 127 */ 128 #if defined(__amd64) 129 int rootnex_prealloc_cookies = 65; 130 int rootnex_prealloc_windows = 4; 131 int rootnex_prealloc_copybuf = 2; 132 #else 133 int rootnex_prealloc_cookies = 33; 134 int rootnex_prealloc_windows = 4; 135 int rootnex_prealloc_copybuf = 2; 136 #endif 137 138 /* driver global state */ 139 static rootnex_state_t *rootnex_state; 140 141 #ifdef DEBUG 142 /* shortcut to rootnex counters */ 143 static uint64_t *rootnex_cnt; 144 #endif 145 146 /* 147 * XXX - does x86 even need these or are they left over from the SPARC days? 148 */ 149 /* statically defined integer/boolean properties for the root node */ 150 static rootnex_intprop_t rootnex_intprp[] = { 151 { "PAGESIZE", PAGESIZE }, 152 { "MMU_PAGESIZE", MMU_PAGESIZE }, 153 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 154 { DDI_RELATIVE_ADDRESSING, 1 }, 155 }; 156 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 157 158 /* 159 * If we're dom0, we're using a real device so we need to load 160 * the cookies with MFNs instead of PFNs. 161 */ 162 #ifdef __xpv 163 typedef maddr_t rootnex_addr_t; 164 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 165 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 166 #else 167 typedef paddr_t rootnex_addr_t; 168 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 169 #endif 170 171 static struct cb_ops rootnex_cb_ops = { 172 nodev, /* open */ 173 nodev, /* close */ 174 nodev, /* strategy */ 175 nodev, /* print */ 176 nodev, /* dump */ 177 nodev, /* read */ 178 nodev, /* write */ 179 nodev, /* ioctl */ 180 nodev, /* devmap */ 181 nodev, /* mmap */ 182 nodev, /* segmap */ 183 nochpoll, /* chpoll */ 184 ddi_prop_op, /* cb_prop_op */ 185 NULL, /* struct streamtab */ 186 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 187 CB_REV, /* Rev */ 188 nodev, /* cb_aread */ 189 nodev /* cb_awrite */ 190 }; 191 192 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 193 off_t offset, off_t len, caddr_t *vaddrp); 194 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 195 struct hat *hat, struct seg *seg, caddr_t addr, 196 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 197 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 198 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 199 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 201 ddi_dma_handle_t *handlep); 202 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle); 204 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 205 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 206 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 207 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle); 209 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 210 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 211 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 212 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 213 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 214 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 215 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 216 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 217 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 218 ddi_ctl_enum_t ctlop, void *arg, void *result); 219 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 220 ddi_iblock_cookie_t *ibc); 221 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 222 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 223 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 224 void *); 225 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 226 227 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 228 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 229 ddi_dma_handle_t *handlep); 230 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 231 ddi_dma_handle_t handle); 232 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 233 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 234 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 235 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 236 ddi_dma_handle_t handle); 237 #if defined(__amd64) && !defined(__xpv) 238 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 239 ddi_dma_handle_t handle); 240 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 241 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 242 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 243 ddi_dma_cookie_t *cookiep, uint_t ccount); 244 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 245 ddi_dma_handle_t handle); 246 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 247 #endif 248 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 249 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 250 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 251 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 252 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 253 254 #if defined(__amd64) && !defined(__xpv) 255 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 256 ddi_dma_handle_t handle, void *v); 257 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 258 ddi_dma_handle_t handle); 259 #endif 260 261 262 static struct bus_ops rootnex_bus_ops = { 263 BUSO_REV, 264 rootnex_map, 265 NULL, 266 NULL, 267 NULL, 268 rootnex_map_fault, 269 rootnex_dma_map, 270 rootnex_dma_allochdl, 271 rootnex_dma_freehdl, 272 rootnex_dma_bindhdl, 273 rootnex_dma_unbindhdl, 274 rootnex_dma_sync, 275 rootnex_dma_win, 276 rootnex_dma_mctl, 277 rootnex_ctlops, 278 ddi_bus_prop_op, 279 i_ddi_rootnex_get_eventcookie, 280 i_ddi_rootnex_add_eventcall, 281 i_ddi_rootnex_remove_eventcall, 282 i_ddi_rootnex_post_event, 283 0, /* bus_intr_ctl */ 284 0, /* bus_config */ 285 0, /* bus_unconfig */ 286 rootnex_fm_init, /* bus_fm_init */ 287 NULL, /* bus_fm_fini */ 288 NULL, /* bus_fm_access_enter */ 289 NULL, /* bus_fm_access_exit */ 290 NULL, /* bus_powr */ 291 rootnex_intr_ops /* bus_intr_op */ 292 }; 293 294 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 295 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 296 static int rootnex_quiesce(dev_info_t *dip); 297 298 static struct dev_ops rootnex_ops = { 299 DEVO_REV, 300 0, 301 ddi_no_info, 302 nulldev, 303 nulldev, 304 rootnex_attach, 305 rootnex_detach, 306 nulldev, 307 &rootnex_cb_ops, 308 &rootnex_bus_ops, 309 NULL, 310 rootnex_quiesce, /* quiesce */ 311 }; 312 313 static struct modldrv rootnex_modldrv = { 314 &mod_driverops, 315 "i86pc root nexus", 316 &rootnex_ops 317 }; 318 319 static struct modlinkage rootnex_modlinkage = { 320 MODREV_1, 321 (void *)&rootnex_modldrv, 322 NULL 323 }; 324 325 #if defined(__amd64) && !defined(__xpv) 326 static iommulib_nexops_t iommulib_nexops = { 327 IOMMU_NEXOPS_VERSION, 328 "Rootnex IOMMU ops Vers 1.1", 329 NULL, 330 rootnex_coredma_allochdl, 331 rootnex_coredma_freehdl, 332 rootnex_coredma_bindhdl, 333 rootnex_coredma_unbindhdl, 334 rootnex_coredma_reset_cookies, 335 rootnex_coredma_get_cookies, 336 rootnex_coredma_set_cookies, 337 rootnex_coredma_clear_cookies, 338 rootnex_coredma_get_sleep_flags, 339 rootnex_coredma_sync, 340 rootnex_coredma_win, 341 rootnex_dma_map, 342 rootnex_dma_mctl, 343 rootnex_coredma_hdl_setprivate, 344 rootnex_coredma_hdl_getprivate 345 }; 346 #endif 347 348 /* 349 * extern hacks 350 */ 351 extern struct seg_ops segdev_ops; 352 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 353 #ifdef DDI_MAP_DEBUG 354 extern int ddi_map_debug_flag; 355 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 356 #endif 357 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 358 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 359 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 360 psm_intr_op_t, int *); 361 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 362 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 363 364 /* 365 * Use device arena to use for device control register mappings. 366 * Various kernel memory walkers (debugger, dtrace) need to know 367 * to avoid this address range to prevent undesired device activity. 368 */ 369 extern void *device_arena_alloc(size_t size, int vm_flag); 370 extern void device_arena_free(void * vaddr, size_t size); 371 372 373 /* 374 * Internal functions 375 */ 376 static int rootnex_dma_init(); 377 static void rootnex_add_props(dev_info_t *); 378 static int rootnex_ctl_reportdev(dev_info_t *dip); 379 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 380 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 381 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 382 static int rootnex_map_handle(ddi_map_req_t *mp); 383 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 384 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 385 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 386 ddi_dma_attr_t *attr); 387 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 388 rootnex_sglinfo_t *sglinfo); 389 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 390 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 391 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 392 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 393 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 394 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 395 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 396 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 397 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 398 static void rootnex_teardown_windows(rootnex_dma_t *dma); 399 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 400 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 401 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 402 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 403 size_t *copybuf_used, page_t **cur_pp); 404 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 405 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 406 ddi_dma_attr_t *attr, off_t cur_offset); 407 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 408 rootnex_dma_t *dma, rootnex_window_t **windowp, 409 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 410 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 411 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 412 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 413 off_t offset, size_t size, uint_t cache_flags); 414 static int rootnex_verify_buffer(rootnex_dma_t *dma); 415 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 416 const void *comp_addr, const void *not_used); 417 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 418 rootnex_sglinfo_t *sglinfo); 419 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 420 421 /* 422 * _init() 423 * 424 */ 425 int 426 _init(void) 427 { 428 429 rootnex_state = NULL; 430 return (mod_install(&rootnex_modlinkage)); 431 } 432 433 434 /* 435 * _info() 436 * 437 */ 438 int 439 _info(struct modinfo *modinfop) 440 { 441 return (mod_info(&rootnex_modlinkage, modinfop)); 442 } 443 444 445 /* 446 * _fini() 447 * 448 */ 449 int 450 _fini(void) 451 { 452 return (EBUSY); 453 } 454 455 456 /* 457 * rootnex_attach() 458 * 459 */ 460 static int 461 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 462 { 463 int fmcap; 464 int e; 465 466 switch (cmd) { 467 case DDI_ATTACH: 468 break; 469 case DDI_RESUME: 470 #if defined(__amd64) && !defined(__xpv) 471 return (immu_unquiesce()); 472 #else 473 return (DDI_SUCCESS); 474 #endif 475 default: 476 return (DDI_FAILURE); 477 } 478 479 /* 480 * We should only have one instance of rootnex. Save it away since we 481 * don't have an easy way to get it back later. 482 */ 483 ASSERT(rootnex_state == NULL); 484 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 485 486 rootnex_state->r_dip = dip; 487 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 488 rootnex_state->r_reserved_msg_printed = B_FALSE; 489 #ifdef DEBUG 490 rootnex_cnt = &rootnex_state->r_counters[0]; 491 #endif 492 493 /* 494 * Set minimum fm capability level for i86pc platforms and then 495 * initialize error handling. Since we're the rootnex, we don't 496 * care what's returned in the fmcap field. 497 */ 498 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 499 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 500 fmcap = ddi_system_fmcap; 501 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 502 503 /* initialize DMA related state */ 504 e = rootnex_dma_init(); 505 if (e != DDI_SUCCESS) { 506 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 507 return (DDI_FAILURE); 508 } 509 510 /* Add static root node properties */ 511 rootnex_add_props(dip); 512 513 /* since we can't call ddi_report_dev() */ 514 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 515 516 /* Initialize rootnex event handle */ 517 i_ddi_rootnex_init_events(dip); 518 519 #if defined(__amd64) && !defined(__xpv) 520 e = iommulib_nexus_register(dip, &iommulib_nexops, 521 &rootnex_state->r_iommulib_handle); 522 523 ASSERT(e == DDI_SUCCESS); 524 #endif 525 526 return (DDI_SUCCESS); 527 } 528 529 530 /* 531 * rootnex_detach() 532 * 533 */ 534 /*ARGSUSED*/ 535 static int 536 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 537 { 538 switch (cmd) { 539 case DDI_SUSPEND: 540 #if defined(__amd64) && !defined(__xpv) 541 return (immu_quiesce()); 542 #else 543 return (DDI_SUCCESS); 544 #endif 545 default: 546 return (DDI_FAILURE); 547 } 548 /*NOTREACHED*/ 549 550 } 551 552 553 /* 554 * rootnex_dma_init() 555 * 556 */ 557 /*ARGSUSED*/ 558 static int 559 rootnex_dma_init() 560 { 561 size_t bufsize; 562 563 564 /* 565 * size of our cookie/window/copybuf state needed in dma bind that we 566 * pre-alloc in dma_alloc_handle 567 */ 568 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 569 rootnex_state->r_prealloc_size = 570 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 571 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 572 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 573 574 /* 575 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 576 * allocate 16 extra bytes for struct pointer alignment 577 * (p->dmai_private & dma->dp_prealloc_buffer) 578 */ 579 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 580 rootnex_state->r_prealloc_size + 0x10; 581 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 582 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 583 if (rootnex_state->r_dmahdl_cache == NULL) { 584 return (DDI_FAILURE); 585 } 586 587 /* 588 * allocate array to track which major numbers we have printed warnings 589 * for. 590 */ 591 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 592 KM_SLEEP); 593 594 return (DDI_SUCCESS); 595 } 596 597 598 /* 599 * rootnex_add_props() 600 * 601 */ 602 static void 603 rootnex_add_props(dev_info_t *dip) 604 { 605 rootnex_intprop_t *rpp; 606 int i; 607 608 /* Add static integer/boolean properties to the root node */ 609 rpp = rootnex_intprp; 610 for (i = 0; i < NROOT_INTPROPS; i++) { 611 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 612 rpp[i].prop_name, rpp[i].prop_value); 613 } 614 } 615 616 617 618 /* 619 * ************************* 620 * ctlops related routines 621 * ************************* 622 */ 623 624 /* 625 * rootnex_ctlops() 626 * 627 */ 628 /*ARGSUSED*/ 629 static int 630 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 631 void *arg, void *result) 632 { 633 int n, *ptr; 634 struct ddi_parent_private_data *pdp; 635 636 switch (ctlop) { 637 case DDI_CTLOPS_DMAPMAPC: 638 /* 639 * Return 'partial' to indicate that dma mapping 640 * has to be done in the main MMU. 641 */ 642 return (DDI_DMA_PARTIAL); 643 644 case DDI_CTLOPS_BTOP: 645 /* 646 * Convert byte count input to physical page units. 647 * (byte counts that are not a page-size multiple 648 * are rounded down) 649 */ 650 *(ulong_t *)result = btop(*(ulong_t *)arg); 651 return (DDI_SUCCESS); 652 653 case DDI_CTLOPS_PTOB: 654 /* 655 * Convert size in physical pages to bytes 656 */ 657 *(ulong_t *)result = ptob(*(ulong_t *)arg); 658 return (DDI_SUCCESS); 659 660 case DDI_CTLOPS_BTOPR: 661 /* 662 * Convert byte count input to physical page units 663 * (byte counts that are not a page-size multiple 664 * are rounded up) 665 */ 666 *(ulong_t *)result = btopr(*(ulong_t *)arg); 667 return (DDI_SUCCESS); 668 669 case DDI_CTLOPS_INITCHILD: 670 return (impl_ddi_sunbus_initchild(arg)); 671 672 case DDI_CTLOPS_UNINITCHILD: 673 impl_ddi_sunbus_removechild(arg); 674 return (DDI_SUCCESS); 675 676 case DDI_CTLOPS_REPORTDEV: 677 return (rootnex_ctl_reportdev(rdip)); 678 679 case DDI_CTLOPS_IOMIN: 680 /* 681 * Nothing to do here but reflect back.. 682 */ 683 return (DDI_SUCCESS); 684 685 case DDI_CTLOPS_REGSIZE: 686 case DDI_CTLOPS_NREGS: 687 break; 688 689 case DDI_CTLOPS_SIDDEV: 690 if (ndi_dev_is_prom_node(rdip)) 691 return (DDI_SUCCESS); 692 if (ndi_dev_is_persistent_node(rdip)) 693 return (DDI_SUCCESS); 694 return (DDI_FAILURE); 695 696 case DDI_CTLOPS_POWER: 697 return ((*pm_platform_power)((power_req_t *)arg)); 698 699 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 700 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 701 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 702 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 703 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 704 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 705 if (!rootnex_state->r_reserved_msg_printed) { 706 rootnex_state->r_reserved_msg_printed = B_TRUE; 707 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 708 "1 or more reserved/obsolete operations."); 709 } 710 return (DDI_FAILURE); 711 712 default: 713 return (DDI_FAILURE); 714 } 715 /* 716 * The rest are for "hardware" properties 717 */ 718 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 719 return (DDI_FAILURE); 720 721 if (ctlop == DDI_CTLOPS_NREGS) { 722 ptr = (int *)result; 723 *ptr = pdp->par_nreg; 724 } else { 725 off_t *size = (off_t *)result; 726 727 ptr = (int *)arg; 728 n = *ptr; 729 if (n >= pdp->par_nreg) { 730 return (DDI_FAILURE); 731 } 732 *size = (off_t)pdp->par_reg[n].regspec_size; 733 } 734 return (DDI_SUCCESS); 735 } 736 737 738 /* 739 * rootnex_ctl_reportdev() 740 * 741 */ 742 static int 743 rootnex_ctl_reportdev(dev_info_t *dev) 744 { 745 int i, n, len, f_len = 0; 746 char *buf; 747 748 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 749 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 750 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 751 len = strlen(buf); 752 753 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 754 755 struct regspec *rp = sparc_pd_getreg(dev, i); 756 757 if (i == 0) 758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 759 ": "); 760 else 761 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 762 " and "); 763 len = strlen(buf); 764 765 switch (rp->regspec_bustype) { 766 767 case BTEISA: 768 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 769 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 770 break; 771 772 case BTISA: 773 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 774 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 775 break; 776 777 default: 778 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 779 "space %x offset %x", 780 rp->regspec_bustype, rp->regspec_addr); 781 break; 782 } 783 len = strlen(buf); 784 } 785 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 786 int pri; 787 788 if (i != 0) { 789 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 790 ","); 791 len = strlen(buf); 792 } 793 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 794 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 795 " sparc ipl %d", pri); 796 len = strlen(buf); 797 } 798 #ifdef DEBUG 799 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 800 cmn_err(CE_NOTE, "next message is truncated: " 801 "printed length 1024, real length %d", f_len); 802 } 803 #endif /* DEBUG */ 804 cmn_err(CE_CONT, "?%s\n", buf); 805 kmem_free(buf, REPORTDEV_BUFSIZE); 806 return (DDI_SUCCESS); 807 } 808 809 810 /* 811 * ****************** 812 * map related code 813 * ****************** 814 */ 815 816 /* 817 * rootnex_map() 818 * 819 */ 820 static int 821 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 822 off_t len, caddr_t *vaddrp) 823 { 824 struct regspec *rp, tmp_reg; 825 ddi_map_req_t mr = *mp; /* Get private copy of request */ 826 int error; 827 828 mp = &mr; 829 830 switch (mp->map_op) { 831 case DDI_MO_MAP_LOCKED: 832 case DDI_MO_UNMAP: 833 case DDI_MO_MAP_HANDLE: 834 break; 835 default: 836 #ifdef DDI_MAP_DEBUG 837 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 838 mp->map_op); 839 #endif /* DDI_MAP_DEBUG */ 840 return (DDI_ME_UNIMPLEMENTED); 841 } 842 843 if (mp->map_flags & DDI_MF_USER_MAPPING) { 844 #ifdef DDI_MAP_DEBUG 845 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 846 #endif /* DDI_MAP_DEBUG */ 847 return (DDI_ME_UNIMPLEMENTED); 848 } 849 850 /* 851 * First, if given an rnumber, convert it to a regspec... 852 * (Presumably, this is on behalf of a child of the root node?) 853 */ 854 855 if (mp->map_type == DDI_MT_RNUMBER) { 856 857 int rnumber = mp->map_obj.rnumber; 858 #ifdef DDI_MAP_DEBUG 859 static char *out_of_range = 860 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 861 #endif /* DDI_MAP_DEBUG */ 862 863 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 864 if (rp == NULL) { 865 #ifdef DDI_MAP_DEBUG 866 cmn_err(CE_WARN, out_of_range, rnumber, 867 ddi_get_name(rdip)); 868 #endif /* DDI_MAP_DEBUG */ 869 return (DDI_ME_RNUMBER_RANGE); 870 } 871 872 /* 873 * Convert the given ddi_map_req_t from rnumber to regspec... 874 */ 875 876 mp->map_type = DDI_MT_REGSPEC; 877 mp->map_obj.rp = rp; 878 } 879 880 /* 881 * Adjust offset and length correspnding to called values... 882 * XXX: A non-zero length means override the one in the regspec 883 * XXX: (regardless of what's in the parent's range?) 884 */ 885 886 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 887 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 888 889 #ifdef DDI_MAP_DEBUG 890 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 891 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 892 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 893 len, mp->map_handlep); 894 #endif /* DDI_MAP_DEBUG */ 895 896 /* 897 * I/O or memory mapping: 898 * 899 * <bustype=0, addr=x, len=x>: memory 900 * <bustype=1, addr=x, len=x>: i/o 901 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 902 */ 903 904 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 905 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 906 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 907 ddi_get_name(rdip), rp->regspec_bustype, 908 rp->regspec_addr, rp->regspec_size); 909 return (DDI_ME_INVAL); 910 } 911 912 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 913 /* 914 * compatibility i/o mapping 915 */ 916 rp->regspec_bustype += (uint_t)offset; 917 } else { 918 /* 919 * Normal memory or i/o mapping 920 */ 921 rp->regspec_addr += (uint_t)offset; 922 } 923 924 if (len != 0) 925 rp->regspec_size = (uint_t)len; 926 927 #ifdef DDI_MAP_DEBUG 928 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 929 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 930 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 931 offset, len, mp->map_handlep); 932 #endif /* DDI_MAP_DEBUG */ 933 934 /* 935 * Apply any parent ranges at this level, if applicable. 936 * (This is where nexus specific regspec translation takes place. 937 * Use of this function is implicit agreement that translation is 938 * provided via ddi_apply_range.) 939 */ 940 941 #ifdef DDI_MAP_DEBUG 942 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 943 ddi_get_name(dip), ddi_get_name(rdip)); 944 #endif /* DDI_MAP_DEBUG */ 945 946 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 947 return (error); 948 949 switch (mp->map_op) { 950 case DDI_MO_MAP_LOCKED: 951 952 /* 953 * Set up the locked down kernel mapping to the regspec... 954 */ 955 956 return (rootnex_map_regspec(mp, vaddrp)); 957 958 case DDI_MO_UNMAP: 959 960 /* 961 * Release mapping... 962 */ 963 964 return (rootnex_unmap_regspec(mp, vaddrp)); 965 966 case DDI_MO_MAP_HANDLE: 967 968 return (rootnex_map_handle(mp)); 969 970 default: 971 return (DDI_ME_UNIMPLEMENTED); 972 } 973 } 974 975 976 /* 977 * rootnex_map_fault() 978 * 979 * fault in mappings for requestors 980 */ 981 /*ARGSUSED*/ 982 static int 983 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 984 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 985 uint_t lock) 986 { 987 988 #ifdef DDI_MAP_DEBUG 989 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 990 ddi_map_debug(" Seg <%s>\n", 991 seg->s_ops == &segdev_ops ? "segdev" : 992 seg == &kvseg ? "segkmem" : "NONE!"); 993 #endif /* DDI_MAP_DEBUG */ 994 995 /* 996 * This is all terribly broken, but it is a start 997 * 998 * XXX Note that this test means that segdev_ops 999 * must be exported from seg_dev.c. 1000 * XXX What about devices with their own segment drivers? 1001 */ 1002 if (seg->s_ops == &segdev_ops) { 1003 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1004 1005 if (hat == NULL) { 1006 /* 1007 * This is one plausible interpretation of 1008 * a null hat i.e. use the first hat on the 1009 * address space hat list which by convention is 1010 * the hat of the system MMU. At alternative 1011 * would be to panic .. this might well be better .. 1012 */ 1013 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1014 hat = seg->s_as->a_hat; 1015 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1016 } 1017 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1018 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1019 } else if (seg == &kvseg && dp == NULL) { 1020 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1021 HAT_LOAD_LOCK); 1022 } else 1023 return (DDI_FAILURE); 1024 return (DDI_SUCCESS); 1025 } 1026 1027 1028 /* 1029 * rootnex_map_regspec() 1030 * we don't support mapping of I/O cards above 4Gb 1031 */ 1032 static int 1033 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1034 { 1035 rootnex_addr_t rbase; 1036 void *cvaddr; 1037 uint_t npages, pgoffset; 1038 struct regspec *rp; 1039 ddi_acc_hdl_t *hp; 1040 ddi_acc_impl_t *ap; 1041 uint_t hat_acc_flags; 1042 paddr_t pbase; 1043 1044 rp = mp->map_obj.rp; 1045 hp = mp->map_handlep; 1046 1047 #ifdef DDI_MAP_DEBUG 1048 ddi_map_debug( 1049 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1050 rp->regspec_bustype, rp->regspec_addr, 1051 rp->regspec_size, mp->map_handlep); 1052 #endif /* DDI_MAP_DEBUG */ 1053 1054 /* 1055 * I/O or memory mapping 1056 * 1057 * <bustype=0, addr=x, len=x>: memory 1058 * <bustype=1, addr=x, len=x>: i/o 1059 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1060 */ 1061 1062 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1063 cmn_err(CE_WARN, "rootnex: invalid register spec" 1064 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1065 rp->regspec_addr, rp->regspec_size); 1066 return (DDI_FAILURE); 1067 } 1068 1069 if (rp->regspec_bustype != 0) { 1070 /* 1071 * I/O space - needs a handle. 1072 */ 1073 if (hp == NULL) { 1074 return (DDI_FAILURE); 1075 } 1076 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1077 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1078 impl_acc_hdl_init(hp); 1079 1080 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1081 #ifdef DDI_MAP_DEBUG 1082 ddi_map_debug("rootnex_map_regspec: mmap() " 1083 "to I/O space is not supported.\n"); 1084 #endif /* DDI_MAP_DEBUG */ 1085 return (DDI_ME_INVAL); 1086 } else { 1087 /* 1088 * 1275-compliant vs. compatibility i/o mapping 1089 */ 1090 *vaddrp = 1091 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1092 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1093 ((caddr_t)(uintptr_t)rp->regspec_addr); 1094 #ifdef __xpv 1095 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1096 hp->ah_pfn = xen_assign_pfn( 1097 mmu_btop((ulong_t)rp->regspec_addr & 1098 MMU_PAGEMASK)); 1099 } else { 1100 hp->ah_pfn = mmu_btop( 1101 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1102 } 1103 #else 1104 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1105 MMU_PAGEMASK); 1106 #endif 1107 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1108 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1109 } 1110 1111 #ifdef DDI_MAP_DEBUG 1112 ddi_map_debug( 1113 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1114 rp->regspec_size, *vaddrp); 1115 #endif /* DDI_MAP_DEBUG */ 1116 return (DDI_SUCCESS); 1117 } 1118 1119 /* 1120 * Memory space 1121 */ 1122 1123 if (hp != NULL) { 1124 /* 1125 * hat layer ignores 1126 * hp->ah_acc.devacc_attr_endian_flags. 1127 */ 1128 switch (hp->ah_acc.devacc_attr_dataorder) { 1129 case DDI_STRICTORDER_ACC: 1130 hat_acc_flags = HAT_STRICTORDER; 1131 break; 1132 case DDI_UNORDERED_OK_ACC: 1133 hat_acc_flags = HAT_UNORDERED_OK; 1134 break; 1135 case DDI_MERGING_OK_ACC: 1136 hat_acc_flags = HAT_MERGING_OK; 1137 break; 1138 case DDI_LOADCACHING_OK_ACC: 1139 hat_acc_flags = HAT_LOADCACHING_OK; 1140 break; 1141 case DDI_STORECACHING_OK_ACC: 1142 hat_acc_flags = HAT_STORECACHING_OK; 1143 break; 1144 } 1145 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1146 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1147 impl_acc_hdl_init(hp); 1148 hp->ah_hat_flags = hat_acc_flags; 1149 } else { 1150 hat_acc_flags = HAT_STRICTORDER; 1151 } 1152 1153 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1154 #ifdef __xpv 1155 /* 1156 * If we're dom0, we're using a real device so we need to translate 1157 * the MA to a PA. 1158 */ 1159 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1160 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1161 } else { 1162 pbase = rbase; 1163 } 1164 #else 1165 pbase = rbase; 1166 #endif 1167 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1168 1169 if (rp->regspec_size == 0) { 1170 #ifdef DDI_MAP_DEBUG 1171 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1172 #endif /* DDI_MAP_DEBUG */ 1173 return (DDI_ME_INVAL); 1174 } 1175 1176 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1177 /* extra cast to make gcc happy */ 1178 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1179 } else { 1180 npages = mmu_btopr(rp->regspec_size + pgoffset); 1181 1182 #ifdef DDI_MAP_DEBUG 1183 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1184 "physical %llx", npages, pbase); 1185 #endif /* DDI_MAP_DEBUG */ 1186 1187 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1188 if (cvaddr == NULL) 1189 return (DDI_ME_NORESOURCES); 1190 1191 /* 1192 * Now map in the pages we've allocated... 1193 */ 1194 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1195 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1196 HAT_LOAD_LOCK); 1197 *vaddrp = (caddr_t)cvaddr + pgoffset; 1198 1199 /* save away pfn and npages for FMA */ 1200 hp = mp->map_handlep; 1201 if (hp) { 1202 hp->ah_pfn = mmu_btop(pbase); 1203 hp->ah_pnum = npages; 1204 } 1205 } 1206 1207 #ifdef DDI_MAP_DEBUG 1208 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1209 #endif /* DDI_MAP_DEBUG */ 1210 return (DDI_SUCCESS); 1211 } 1212 1213 1214 /* 1215 * rootnex_unmap_regspec() 1216 * 1217 */ 1218 static int 1219 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1220 { 1221 caddr_t addr = (caddr_t)*vaddrp; 1222 uint_t npages, pgoffset; 1223 struct regspec *rp; 1224 1225 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1226 return (0); 1227 1228 rp = mp->map_obj.rp; 1229 1230 if (rp->regspec_size == 0) { 1231 #ifdef DDI_MAP_DEBUG 1232 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1233 #endif /* DDI_MAP_DEBUG */ 1234 return (DDI_ME_INVAL); 1235 } 1236 1237 /* 1238 * I/O or memory mapping: 1239 * 1240 * <bustype=0, addr=x, len=x>: memory 1241 * <bustype=1, addr=x, len=x>: i/o 1242 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1243 */ 1244 if (rp->regspec_bustype != 0) { 1245 /* 1246 * This is I/O space, which requires no particular 1247 * processing on unmap since it isn't mapped in the 1248 * first place. 1249 */ 1250 return (DDI_SUCCESS); 1251 } 1252 1253 /* 1254 * Memory space 1255 */ 1256 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1257 npages = mmu_btopr(rp->regspec_size + pgoffset); 1258 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1259 device_arena_free(addr - pgoffset, ptob(npages)); 1260 1261 /* 1262 * Destroy the pointer - the mapping has logically gone 1263 */ 1264 *vaddrp = NULL; 1265 1266 return (DDI_SUCCESS); 1267 } 1268 1269 1270 /* 1271 * rootnex_map_handle() 1272 * 1273 */ 1274 static int 1275 rootnex_map_handle(ddi_map_req_t *mp) 1276 { 1277 rootnex_addr_t rbase; 1278 ddi_acc_hdl_t *hp; 1279 uint_t pgoffset; 1280 struct regspec *rp; 1281 paddr_t pbase; 1282 1283 rp = mp->map_obj.rp; 1284 1285 #ifdef DDI_MAP_DEBUG 1286 ddi_map_debug( 1287 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1288 rp->regspec_bustype, rp->regspec_addr, 1289 rp->regspec_size, mp->map_handlep); 1290 #endif /* DDI_MAP_DEBUG */ 1291 1292 /* 1293 * I/O or memory mapping: 1294 * 1295 * <bustype=0, addr=x, len=x>: memory 1296 * <bustype=1, addr=x, len=x>: i/o 1297 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1298 */ 1299 if (rp->regspec_bustype != 0) { 1300 /* 1301 * This refers to I/O space, and we don't support "mapping" 1302 * I/O space to a user. 1303 */ 1304 return (DDI_FAILURE); 1305 } 1306 1307 /* 1308 * Set up the hat_flags for the mapping. 1309 */ 1310 hp = mp->map_handlep; 1311 1312 switch (hp->ah_acc.devacc_attr_endian_flags) { 1313 case DDI_NEVERSWAP_ACC: 1314 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1315 break; 1316 case DDI_STRUCTURE_LE_ACC: 1317 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1318 break; 1319 case DDI_STRUCTURE_BE_ACC: 1320 return (DDI_FAILURE); 1321 default: 1322 return (DDI_REGS_ACC_CONFLICT); 1323 } 1324 1325 switch (hp->ah_acc.devacc_attr_dataorder) { 1326 case DDI_STRICTORDER_ACC: 1327 break; 1328 case DDI_UNORDERED_OK_ACC: 1329 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1330 break; 1331 case DDI_MERGING_OK_ACC: 1332 hp->ah_hat_flags |= HAT_MERGING_OK; 1333 break; 1334 case DDI_LOADCACHING_OK_ACC: 1335 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1336 break; 1337 case DDI_STORECACHING_OK_ACC: 1338 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1339 break; 1340 default: 1341 return (DDI_FAILURE); 1342 } 1343 1344 rbase = (rootnex_addr_t)rp->regspec_addr & 1345 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1346 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1347 1348 if (rp->regspec_size == 0) 1349 return (DDI_ME_INVAL); 1350 1351 #ifdef __xpv 1352 /* 1353 * If we're dom0, we're using a real device so we need to translate 1354 * the MA to a PA. 1355 */ 1356 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1357 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1358 (rbase & MMU_PAGEOFFSET); 1359 } else { 1360 pbase = rbase; 1361 } 1362 #else 1363 pbase = rbase; 1364 #endif 1365 1366 hp->ah_pfn = mmu_btop(pbase); 1367 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1368 1369 return (DDI_SUCCESS); 1370 } 1371 1372 1373 1374 /* 1375 * ************************ 1376 * interrupt related code 1377 * ************************ 1378 */ 1379 1380 /* 1381 * rootnex_intr_ops() 1382 * bus_intr_op() function for interrupt support 1383 */ 1384 /* ARGSUSED */ 1385 static int 1386 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1387 ddi_intr_handle_impl_t *hdlp, void *result) 1388 { 1389 struct intrspec *ispec; 1390 1391 DDI_INTR_NEXDBG((CE_CONT, 1392 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1393 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1394 1395 /* Process the interrupt operation */ 1396 switch (intr_op) { 1397 case DDI_INTROP_GETCAP: 1398 /* First check with pcplusmp */ 1399 if (psm_intr_ops == NULL) 1400 return (DDI_FAILURE); 1401 1402 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1403 *(int *)result = 0; 1404 return (DDI_FAILURE); 1405 } 1406 break; 1407 case DDI_INTROP_SETCAP: 1408 if (psm_intr_ops == NULL) 1409 return (DDI_FAILURE); 1410 1411 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1412 return (DDI_FAILURE); 1413 break; 1414 case DDI_INTROP_ALLOC: 1415 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1416 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1417 case DDI_INTROP_FREE: 1418 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1419 return (rootnex_free_intr_fixed(rdip, hdlp)); 1420 case DDI_INTROP_GETPRI: 1421 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1422 return (DDI_FAILURE); 1423 *(int *)result = ispec->intrspec_pri; 1424 break; 1425 case DDI_INTROP_SETPRI: 1426 /* Validate the interrupt priority passed to us */ 1427 if (*(int *)result > LOCK_LEVEL) 1428 return (DDI_FAILURE); 1429 1430 /* Ensure that PSM is all initialized and ispec is ok */ 1431 if ((psm_intr_ops == NULL) || 1432 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1433 return (DDI_FAILURE); 1434 1435 /* Change the priority */ 1436 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1437 PSM_FAILURE) 1438 return (DDI_FAILURE); 1439 1440 /* update the ispec with the new priority */ 1441 ispec->intrspec_pri = *(int *)result; 1442 break; 1443 case DDI_INTROP_ADDISR: 1444 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 return (DDI_FAILURE); 1446 ispec->intrspec_func = hdlp->ih_cb_func; 1447 break; 1448 case DDI_INTROP_REMISR: 1449 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1450 return (DDI_FAILURE); 1451 ispec->intrspec_func = (uint_t (*)()) 0; 1452 break; 1453 case DDI_INTROP_ENABLE: 1454 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1455 return (DDI_FAILURE); 1456 1457 /* Call psmi to translate irq with the dip */ 1458 if (psm_intr_ops == NULL) 1459 return (DDI_FAILURE); 1460 1461 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1462 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1463 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1464 return (DDI_FAILURE); 1465 1466 /* Add the interrupt handler */ 1467 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1468 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1469 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1470 return (DDI_FAILURE); 1471 break; 1472 case DDI_INTROP_DISABLE: 1473 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1474 return (DDI_FAILURE); 1475 1476 /* Call psm_ops() to translate irq with the dip */ 1477 if (psm_intr_ops == NULL) 1478 return (DDI_FAILURE); 1479 1480 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1481 (void) (*psm_intr_ops)(rdip, hdlp, 1482 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1483 1484 /* Remove the interrupt handler */ 1485 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1486 hdlp->ih_cb_func, hdlp->ih_vector); 1487 break; 1488 case DDI_INTROP_SETMASK: 1489 if (psm_intr_ops == NULL) 1490 return (DDI_FAILURE); 1491 1492 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1493 return (DDI_FAILURE); 1494 break; 1495 case DDI_INTROP_CLRMASK: 1496 if (psm_intr_ops == NULL) 1497 return (DDI_FAILURE); 1498 1499 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1500 return (DDI_FAILURE); 1501 break; 1502 case DDI_INTROP_GETPENDING: 1503 if (psm_intr_ops == NULL) 1504 return (DDI_FAILURE); 1505 1506 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1507 result)) { 1508 *(int *)result = 0; 1509 return (DDI_FAILURE); 1510 } 1511 break; 1512 case DDI_INTROP_NAVAIL: 1513 case DDI_INTROP_NINTRS: 1514 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1515 if (*(int *)result == 0) { 1516 /* 1517 * Special case for 'pcic' driver' only. This driver 1518 * driver is a child of 'isa' and 'rootnex' drivers. 1519 * 1520 * See detailed comments on this in the function 1521 * rootnex_get_ispec(). 1522 * 1523 * Children of 'pcic' send 'NINITR' request all the 1524 * way to rootnex driver. But, the 'pdp->par_nintr' 1525 * field may not initialized. So, we fake it here 1526 * to return 1 (a la what PCMCIA nexus does). 1527 */ 1528 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1529 *(int *)result = 1; 1530 else 1531 return (DDI_FAILURE); 1532 } 1533 break; 1534 case DDI_INTROP_SUPPORTED_TYPES: 1535 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1536 break; 1537 default: 1538 return (DDI_FAILURE); 1539 } 1540 1541 return (DDI_SUCCESS); 1542 } 1543 1544 1545 /* 1546 * rootnex_get_ispec() 1547 * convert an interrupt number to an interrupt specification. 1548 * The interrupt number determines which interrupt spec will be 1549 * returned if more than one exists. 1550 * 1551 * Look into the parent private data area of the 'rdip' to find out 1552 * the interrupt specification. First check to make sure there is 1553 * one that matchs "inumber" and then return a pointer to it. 1554 * 1555 * Return NULL if one could not be found. 1556 * 1557 * NOTE: This is needed for rootnex_intr_ops() 1558 */ 1559 static struct intrspec * 1560 rootnex_get_ispec(dev_info_t *rdip, int inum) 1561 { 1562 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1563 1564 /* 1565 * Special case handling for drivers that provide their own 1566 * intrspec structures instead of relying on the DDI framework. 1567 * 1568 * A broken hardware driver in ON could potentially provide its 1569 * own intrspec structure, instead of relying on the hardware. 1570 * If these drivers are children of 'rootnex' then we need to 1571 * continue to provide backward compatibility to them here. 1572 * 1573 * Following check is a special case for 'pcic' driver which 1574 * was found to have broken hardwre andby provides its own intrspec. 1575 * 1576 * Verbatim comments from this driver are shown here: 1577 * "Don't use the ddi_add_intr since we don't have a 1578 * default intrspec in all cases." 1579 * 1580 * Since an 'ispec' may not be always created for it, 1581 * check for that and create one if so. 1582 * 1583 * NOTE: Currently 'pcic' is the only driver found to do this. 1584 */ 1585 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1586 pdp->par_nintr = 1; 1587 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1588 pdp->par_nintr, KM_SLEEP); 1589 } 1590 1591 /* Validate the interrupt number */ 1592 if (inum >= pdp->par_nintr) 1593 return (NULL); 1594 1595 /* Get the interrupt structure pointer and return that */ 1596 return ((struct intrspec *)&pdp->par_intr[inum]); 1597 } 1598 1599 /* 1600 * Allocate interrupt vector for FIXED (legacy) type. 1601 */ 1602 static int 1603 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1604 void *result) 1605 { 1606 struct intrspec *ispec; 1607 ddi_intr_handle_impl_t info_hdl; 1608 int ret; 1609 int free_phdl = 0; 1610 apic_get_type_t type_info; 1611 1612 if (psm_intr_ops == NULL) 1613 return (DDI_FAILURE); 1614 1615 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1616 return (DDI_FAILURE); 1617 1618 /* 1619 * If the PSM module is "APIX" then pass the request for it 1620 * to allocate the vector now. 1621 */ 1622 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1623 info_hdl.ih_private = &type_info; 1624 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1625 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1626 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1627 free_phdl = 1; 1628 i_ddi_alloc_intr_phdl(hdlp); 1629 } 1630 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1631 ret = (*psm_intr_ops)(rdip, hdlp, 1632 PSM_INTR_OP_ALLOC_VECTORS, result); 1633 if (free_phdl) { /* free up the phdl structure */ 1634 free_phdl = 0; 1635 i_ddi_free_intr_phdl(hdlp); 1636 hdlp->ih_private = NULL; 1637 } 1638 } else { 1639 /* 1640 * No APIX module; fall back to the old scheme where the 1641 * interrupt vector is allocated during ddi_enable_intr() call. 1642 */ 1643 hdlp->ih_pri = ispec->intrspec_pri; 1644 *(int *)result = hdlp->ih_scratch1; 1645 ret = DDI_SUCCESS; 1646 } 1647 1648 return (ret); 1649 } 1650 1651 /* 1652 * Free up interrupt vector for FIXED (legacy) type. 1653 */ 1654 static int 1655 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1656 { 1657 struct intrspec *ispec; 1658 struct ddi_parent_private_data *pdp; 1659 ddi_intr_handle_impl_t info_hdl; 1660 int ret; 1661 apic_get_type_t type_info; 1662 1663 if (psm_intr_ops == NULL) 1664 return (DDI_FAILURE); 1665 1666 /* 1667 * If the PSM module is "APIX" then pass the request for it 1668 * to free up the vector now. 1669 */ 1670 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1671 info_hdl.ih_private = &type_info; 1672 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1673 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1674 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1675 return (DDI_FAILURE); 1676 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1677 ret = (*psm_intr_ops)(rdip, hdlp, 1678 PSM_INTR_OP_FREE_VECTORS, NULL); 1679 } else { 1680 /* 1681 * No APIX module; fall back to the old scheme where 1682 * the interrupt vector was already freed during 1683 * ddi_disable_intr() call. 1684 */ 1685 ret = DDI_SUCCESS; 1686 } 1687 1688 pdp = ddi_get_parent_data(rdip); 1689 1690 /* 1691 * Special case for 'pcic' driver' only. 1692 * If an intrspec was created for it, clean it up here 1693 * See detailed comments on this in the function 1694 * rootnex_get_ispec(). 1695 */ 1696 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1697 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1698 pdp->par_nintr); 1699 /* 1700 * Set it to zero; so that 1701 * DDI framework doesn't free it again 1702 */ 1703 pdp->par_intr = NULL; 1704 pdp->par_nintr = 0; 1705 } 1706 1707 return (ret); 1708 } 1709 1710 1711 /* 1712 * ****************** 1713 * dma related code 1714 * ****************** 1715 */ 1716 1717 /*ARGSUSED*/ 1718 static int 1719 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1720 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1721 ddi_dma_handle_t *handlep) 1722 { 1723 uint64_t maxsegmentsize_ll; 1724 uint_t maxsegmentsize; 1725 ddi_dma_impl_t *hp; 1726 rootnex_dma_t *dma; 1727 uint64_t count_max; 1728 uint64_t seg; 1729 int kmflag; 1730 int e; 1731 1732 1733 /* convert our sleep flags */ 1734 if (waitfp == DDI_DMA_SLEEP) { 1735 kmflag = KM_SLEEP; 1736 } else { 1737 kmflag = KM_NOSLEEP; 1738 } 1739 1740 /* 1741 * We try to do only one memory allocation here. We'll do a little 1742 * pointer manipulation later. If the bind ends up taking more than 1743 * our prealloc's space, we'll have to allocate more memory in the 1744 * bind operation. Not great, but much better than before and the 1745 * best we can do with the current bind interfaces. 1746 */ 1747 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1748 if (hp == NULL) 1749 return (DDI_DMA_NORESOURCES); 1750 1751 /* Do our pointer manipulation now, align the structures */ 1752 hp->dmai_private = (void *)(((uintptr_t)hp + 1753 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1754 dma = (rootnex_dma_t *)hp->dmai_private; 1755 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1756 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1757 1758 /* setup the handle */ 1759 rootnex_clean_dmahdl(hp); 1760 hp->dmai_error.err_fep = NULL; 1761 hp->dmai_error.err_cf = NULL; 1762 dma->dp_dip = rdip; 1763 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1764 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1765 1766 /* 1767 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1768 * is being used. Set the upper limit to the seg value. 1769 * There will be enough DVMA space to always get addresses 1770 * that will match the constraints. 1771 */ 1772 if (IOMMU_USED(rdip) && 1773 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1774 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1775 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1776 } else 1777 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1778 1779 hp->dmai_minxfer = attr->dma_attr_minxfer; 1780 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1781 hp->dmai_rdip = rdip; 1782 hp->dmai_attr = *attr; 1783 1784 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1785 dma->dp_sglinfo.si_cancross = B_FALSE; 1786 else 1787 dma->dp_sglinfo.si_cancross = B_TRUE; 1788 1789 /* we don't need to worry about the SPL since we do a tryenter */ 1790 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1791 1792 /* 1793 * Figure out our maximum segment size. If the segment size is greater 1794 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1795 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1796 * dma_attr_count_max are size-1 type values. 1797 * 1798 * Maximum segment size is the largest physically contiguous chunk of 1799 * memory that we can return from a bind (i.e. the maximum size of a 1800 * single cookie). 1801 */ 1802 1803 /* handle the rollover cases */ 1804 seg = attr->dma_attr_seg + 1; 1805 if (seg < attr->dma_attr_seg) { 1806 seg = attr->dma_attr_seg; 1807 } 1808 count_max = attr->dma_attr_count_max + 1; 1809 if (count_max < attr->dma_attr_count_max) { 1810 count_max = attr->dma_attr_count_max; 1811 } 1812 1813 /* 1814 * granularity may or may not be a power of two. If it isn't, we can't 1815 * use a simple mask. 1816 */ 1817 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1818 dma->dp_granularity_power_2 = B_FALSE; 1819 } else { 1820 dma->dp_granularity_power_2 = B_TRUE; 1821 } 1822 1823 /* 1824 * maxxfer should be a whole multiple of granularity. If we're going to 1825 * break up a window because we're greater than maxxfer, we might as 1826 * well make sure it's maxxfer is a whole multiple so we don't have to 1827 * worry about triming the window later on for this case. 1828 */ 1829 if (attr->dma_attr_granular > 1) { 1830 if (dma->dp_granularity_power_2) { 1831 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1832 (attr->dma_attr_maxxfer & 1833 (attr->dma_attr_granular - 1)); 1834 } else { 1835 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1836 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1837 } 1838 } else { 1839 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1840 } 1841 1842 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1843 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1844 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1845 maxsegmentsize = 0xFFFFFFFF; 1846 } else { 1847 maxsegmentsize = maxsegmentsize_ll; 1848 } 1849 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1850 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1851 1852 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1853 if (rootnex_alloc_check_parms) { 1854 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1855 if (e != DDI_SUCCESS) { 1856 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1857 (void) rootnex_dma_freehdl(dip, rdip, 1858 (ddi_dma_handle_t)hp); 1859 return (e); 1860 } 1861 } 1862 1863 *handlep = (ddi_dma_handle_t)hp; 1864 1865 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1866 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1867 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1868 1869 return (DDI_SUCCESS); 1870 } 1871 1872 1873 /* 1874 * rootnex_dma_allochdl() 1875 * called from ddi_dma_alloc_handle(). 1876 */ 1877 static int 1878 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1879 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1880 { 1881 int retval = DDI_SUCCESS; 1882 #if defined(__amd64) && !defined(__xpv) 1883 1884 if (IOMMU_UNITIALIZED(rdip)) { 1885 retval = iommulib_nex_open(dip, rdip); 1886 1887 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1888 return (retval); 1889 } 1890 1891 if (IOMMU_UNUSED(rdip)) { 1892 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1893 handlep); 1894 } else { 1895 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1896 waitfp, arg, handlep); 1897 } 1898 #else 1899 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1900 handlep); 1901 #endif 1902 switch (retval) { 1903 case DDI_DMA_NORESOURCES: 1904 if (waitfp != DDI_DMA_DONTWAIT) { 1905 ddi_set_callback(waitfp, arg, 1906 &rootnex_state->r_dvma_call_list_id); 1907 } 1908 break; 1909 case DDI_SUCCESS: 1910 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1911 break; 1912 default: 1913 break; 1914 } 1915 return (retval); 1916 } 1917 1918 /*ARGSUSED*/ 1919 static int 1920 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1921 ddi_dma_handle_t handle) 1922 { 1923 ddi_dma_impl_t *hp; 1924 rootnex_dma_t *dma; 1925 1926 1927 hp = (ddi_dma_impl_t *)handle; 1928 dma = (rootnex_dma_t *)hp->dmai_private; 1929 1930 /* unbind should have been called first */ 1931 ASSERT(!dma->dp_inuse); 1932 1933 mutex_destroy(&dma->dp_mutex); 1934 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1935 1936 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1937 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1938 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1939 1940 return (DDI_SUCCESS); 1941 } 1942 1943 /* 1944 * rootnex_dma_freehdl() 1945 * called from ddi_dma_free_handle(). 1946 */ 1947 static int 1948 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1949 { 1950 int ret; 1951 1952 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1953 #if defined(__amd64) && !defined(__xpv) 1954 if (IOMMU_USED(rdip)) 1955 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1956 else 1957 #endif 1958 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1959 1960 if (rootnex_state->r_dvma_call_list_id) 1961 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1962 1963 return (ret); 1964 } 1965 1966 /*ARGSUSED*/ 1967 static int 1968 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1969 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1970 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1971 { 1972 rootnex_sglinfo_t *sinfo; 1973 ddi_dma_obj_t *dmao; 1974 #if defined(__amd64) && !defined(__xpv) 1975 struct dvmaseg *dvs; 1976 ddi_dma_cookie_t *cookie; 1977 #endif 1978 ddi_dma_attr_t *attr; 1979 ddi_dma_impl_t *hp; 1980 rootnex_dma_t *dma; 1981 int kmflag; 1982 int e; 1983 uint_t ncookies; 1984 1985 hp = (ddi_dma_impl_t *)handle; 1986 dma = (rootnex_dma_t *)hp->dmai_private; 1987 dmao = &dma->dp_dma; 1988 sinfo = &dma->dp_sglinfo; 1989 attr = &hp->dmai_attr; 1990 1991 /* convert the sleep flags */ 1992 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1993 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1994 } else { 1995 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1996 } 1997 1998 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1999 2000 /* 2001 * This is useful for debugging a driver. Not as useful in a production 2002 * system. The only time this will fail is if you have a driver bug. 2003 */ 2004 if (rootnex_bind_check_inuse) { 2005 /* 2006 * No one else should ever have this lock unless someone else 2007 * is trying to use this handle. So contention on the lock 2008 * is the same as inuse being set. 2009 */ 2010 e = mutex_tryenter(&dma->dp_mutex); 2011 if (e == 0) { 2012 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2013 return (DDI_DMA_INUSE); 2014 } 2015 if (dma->dp_inuse) { 2016 mutex_exit(&dma->dp_mutex); 2017 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2018 return (DDI_DMA_INUSE); 2019 } 2020 dma->dp_inuse = B_TRUE; 2021 mutex_exit(&dma->dp_mutex); 2022 } 2023 2024 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2025 if (rootnex_bind_check_parms) { 2026 e = rootnex_valid_bind_parms(dmareq, attr); 2027 if (e != DDI_SUCCESS) { 2028 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2029 rootnex_clean_dmahdl(hp); 2030 return (e); 2031 } 2032 } 2033 2034 /* save away the original bind info */ 2035 dma->dp_dma = dmareq->dmar_object; 2036 2037 #if defined(__amd64) && !defined(__xpv) 2038 if (IOMMU_USED(rdip)) { 2039 dmao = &dma->dp_dvma; 2040 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2041 switch (e) { 2042 case DDI_SUCCESS: 2043 if (sinfo->si_cancross || 2044 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2045 dmao->dmao_size > sinfo->si_max_cookie_size) { 2046 dma->dp_dvma_used = B_TRUE; 2047 break; 2048 } 2049 sinfo->si_sgl_size = 1; 2050 hp->dmai_rflags |= DMP_NOSYNC; 2051 2052 dma->dp_dvma_used = B_TRUE; 2053 dma->dp_need_to_free_cookie = B_FALSE; 2054 2055 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2056 cookie = hp->dmai_cookie = dma->dp_cookies = 2057 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2058 cookie->dmac_laddress = dvs->dvs_start + 2059 dmao->dmao_obj.dvma_obj.dv_off; 2060 cookie->dmac_size = dvs->dvs_len; 2061 cookie->dmac_type = 0; 2062 2063 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2064 rdip); 2065 goto fast; 2066 case DDI_ENOTSUP: 2067 break; 2068 default: 2069 rootnex_clean_dmahdl(hp); 2070 return (e); 2071 } 2072 } 2073 #endif 2074 2075 /* 2076 * Figure out a rough estimate of what maximum number of pages 2077 * this buffer could use (a high estimate of course). 2078 */ 2079 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2080 2081 if (dma->dp_dvma_used) { 2082 /* 2083 * The number of physical pages is the worst case. 2084 * 2085 * For DVMA, the worst case is the length divided 2086 * by the maximum cookie length, plus 1. Add to that 2087 * the number of segment boundaries potentially crossed, and 2088 * the additional number of DVMA segments that was returned. 2089 * 2090 * In the normal case, for modern devices, si_cancross will 2091 * be false, and dv_nseg will be 1, and the fast path will 2092 * have been taken above. 2093 */ 2094 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2095 + 1; 2096 if (sinfo->si_cancross) 2097 ncookies += 2098 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2099 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2100 2101 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2102 } 2103 2104 /* 2105 * We'll use the pre-allocated cookies for any bind that will *always* 2106 * fit (more important to be consistent, we don't want to create 2107 * additional degenerate cases). 2108 */ 2109 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2110 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2111 dma->dp_need_to_free_cookie = B_FALSE; 2112 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2113 uint_t, sinfo->si_max_pages); 2114 2115 /* 2116 * For anything larger than that, we'll go ahead and allocate the 2117 * maximum number of pages we expect to see. Hopefuly, we won't be 2118 * seeing this path in the fast path for high performance devices very 2119 * frequently. 2120 * 2121 * a ddi bind interface that allowed the driver to provide storage to 2122 * the bind interface would speed this case up. 2123 */ 2124 } else { 2125 /* 2126 * Save away how much memory we allocated. If we're doing a 2127 * nosleep, the alloc could fail... 2128 */ 2129 dma->dp_cookie_size = sinfo->si_max_pages * 2130 sizeof (ddi_dma_cookie_t); 2131 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2132 if (dma->dp_cookies == NULL) { 2133 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2134 rootnex_clean_dmahdl(hp); 2135 return (DDI_DMA_NORESOURCES); 2136 } 2137 dma->dp_need_to_free_cookie = B_TRUE; 2138 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2139 uint_t, sinfo->si_max_pages); 2140 } 2141 hp->dmai_cookie = dma->dp_cookies; 2142 2143 /* 2144 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2145 * looking at the constraints in the dma structure. It will then put 2146 * some additional state about the sgl in the dma struct (i.e. is 2147 * the sgl clean, or do we need to do some munging; how many pages 2148 * need to be copied, etc.) 2149 */ 2150 if (dma->dp_dvma_used) 2151 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2152 else 2153 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2154 2155 out: 2156 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2157 /* if we don't need a copy buffer, we don't need to sync */ 2158 if (sinfo->si_copybuf_req == 0) { 2159 hp->dmai_rflags |= DMP_NOSYNC; 2160 } 2161 2162 /* 2163 * if we don't need the copybuf and we don't need to do a partial, we 2164 * hit the fast path. All the high performance devices should be trying 2165 * to hit this path. To hit this path, a device should be able to reach 2166 * all of memory, shouldn't try to bind more than it can transfer, and 2167 * the buffer shouldn't require more cookies than the driver/device can 2168 * handle [sgllen]). 2169 * 2170 * Note that negative values of dma_attr_sgllen are supposed 2171 * to mean unlimited, but we just cast them to mean a 2172 * "ridiculous large limit". This saves some extra checks on 2173 * hot paths. 2174 */ 2175 if ((sinfo->si_copybuf_req == 0) && 2176 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2177 (dmao->dmao_size < dma->dp_maxxfer)) { 2178 fast: 2179 /* 2180 * If the driver supports FMA, insert the handle in the FMA DMA 2181 * handle cache. 2182 */ 2183 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2184 hp->dmai_error.err_cf = rootnex_dma_check; 2185 2186 /* 2187 * copy out the first cookie and ccountp, set the cookie 2188 * pointer to the second cookie. The first cookie is passed 2189 * back on the stack. Additional cookies are accessed via 2190 * ddi_dma_nextcookie() 2191 */ 2192 *cookiep = dma->dp_cookies[0]; 2193 *ccountp = sinfo->si_sgl_size; 2194 hp->dmai_cookie++; 2195 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2196 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2197 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2198 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2199 uint_t, dmao->dmao_size, uint_t, *ccountp); 2200 2201 2202 return (DDI_DMA_MAPPED); 2203 } 2204 2205 /* 2206 * go to the slow path, we may need to alloc more memory, create 2207 * multiple windows, and munge up a sgl to make the device happy. 2208 */ 2209 2210 /* 2211 * With the IOMMU mapobject method used, we should never hit 2212 * the slow path. If we do, something is seriously wrong. 2213 * Clean up and return an error. 2214 */ 2215 2216 #if defined(__amd64) && !defined(__xpv) 2217 2218 if (dma->dp_dvma_used) { 2219 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2220 &dma->dp_dvma); 2221 e = DDI_DMA_NOMAPPING; 2222 } else { 2223 #endif 2224 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2225 kmflag); 2226 #if defined(__amd64) && !defined(__xpv) 2227 } 2228 #endif 2229 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2230 if (dma->dp_need_to_free_cookie) { 2231 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2232 } 2233 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2234 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2235 return (e); 2236 } 2237 2238 /* 2239 * If the driver supports FMA, insert the handle in the FMA DMA handle 2240 * cache. 2241 */ 2242 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2243 hp->dmai_error.err_cf = rootnex_dma_check; 2244 2245 /* if the first window uses the copy buffer, sync it for the device */ 2246 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2247 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2248 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2249 DDI_DMA_SYNC_FORDEV); 2250 } 2251 2252 /* 2253 * copy out the first cookie and ccountp, set the cookie pointer to the 2254 * second cookie. Make sure the partial flag is set/cleared correctly. 2255 * If we have a partial map (i.e. multiple windows), the number of 2256 * cookies we return is the number of cookies in the first window. 2257 */ 2258 if (e == DDI_DMA_MAPPED) { 2259 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2260 *ccountp = sinfo->si_sgl_size; 2261 hp->dmai_nwin = 1; 2262 } else { 2263 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2264 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2265 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2266 } 2267 *cookiep = dma->dp_cookies[0]; 2268 hp->dmai_cookie++; 2269 2270 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2271 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2272 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2273 dmao->dmao_size, uint_t, *ccountp); 2274 return (e); 2275 } 2276 2277 /* 2278 * rootnex_dma_bindhdl() 2279 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2280 */ 2281 static int 2282 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2283 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2284 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2285 { 2286 int ret; 2287 #if defined(__amd64) && !defined(__xpv) 2288 if (IOMMU_USED(rdip)) 2289 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2290 cookiep, ccountp); 2291 else 2292 #endif 2293 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2294 cookiep, ccountp); 2295 2296 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2297 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2298 &rootnex_state->r_dvma_call_list_id); 2299 } 2300 2301 return (ret); 2302 } 2303 2304 2305 2306 /*ARGSUSED*/ 2307 static int 2308 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2309 ddi_dma_handle_t handle) 2310 { 2311 ddi_dma_impl_t *hp; 2312 rootnex_dma_t *dma; 2313 int e; 2314 2315 hp = (ddi_dma_impl_t *)handle; 2316 dma = (rootnex_dma_t *)hp->dmai_private; 2317 2318 /* make sure the buffer wasn't free'd before calling unbind */ 2319 if (rootnex_unbind_verify_buffer) { 2320 e = rootnex_verify_buffer(dma); 2321 if (e != DDI_SUCCESS) { 2322 ASSERT(0); 2323 return (DDI_FAILURE); 2324 } 2325 } 2326 2327 /* sync the current window before unbinding the buffer */ 2328 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2329 (hp->dmai_rflags & DDI_DMA_READ)) { 2330 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2331 DDI_DMA_SYNC_FORCPU); 2332 } 2333 2334 /* 2335 * cleanup and copy buffer or window state. if we didn't use the copy 2336 * buffer or windows, there won't be much to do :-) 2337 */ 2338 rootnex_teardown_copybuf(dma); 2339 rootnex_teardown_windows(dma); 2340 2341 #if defined(__amd64) && !defined(__xpv) 2342 if (IOMMU_USED(rdip)) 2343 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2344 &dma->dp_dvma); 2345 #endif 2346 2347 /* 2348 * If we had to allocate space to for the worse case sgl (it didn't 2349 * fit into our pre-allocate buffer), free that up now 2350 */ 2351 if (dma->dp_need_to_free_cookie) { 2352 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2353 } 2354 2355 /* 2356 * clean up the handle so it's ready for the next bind (i.e. if the 2357 * handle is reused). 2358 */ 2359 rootnex_clean_dmahdl(hp); 2360 hp->dmai_error.err_cf = NULL; 2361 2362 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2363 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2364 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2365 2366 return (DDI_SUCCESS); 2367 } 2368 2369 /* 2370 * rootnex_dma_unbindhdl() 2371 * called from ddi_dma_unbind_handle() 2372 */ 2373 /*ARGSUSED*/ 2374 static int 2375 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2376 ddi_dma_handle_t handle) 2377 { 2378 int ret; 2379 2380 #if defined(__amd64) && !defined(__xpv) 2381 if (IOMMU_USED(rdip)) 2382 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2383 else 2384 #endif 2385 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2386 2387 if (rootnex_state->r_dvma_call_list_id) 2388 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2389 2390 return (ret); 2391 } 2392 2393 #if defined(__amd64) && !defined(__xpv) 2394 2395 static int 2396 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2397 { 2398 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2399 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2400 2401 if (dma->dp_sleep_flags != KM_SLEEP && 2402 dma->dp_sleep_flags != KM_NOSLEEP) 2403 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2404 return (dma->dp_sleep_flags); 2405 } 2406 /*ARGSUSED*/ 2407 static void 2408 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2409 { 2410 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2411 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2412 rootnex_window_t *window; 2413 2414 if (dma->dp_window) { 2415 window = &dma->dp_window[dma->dp_current_win]; 2416 hp->dmai_cookie = window->wd_first_cookie; 2417 } else { 2418 hp->dmai_cookie = dma->dp_cookies; 2419 } 2420 hp->dmai_cookie++; 2421 } 2422 2423 /*ARGSUSED*/ 2424 static int 2425 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2426 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2427 { 2428 int i; 2429 int km_flags; 2430 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2431 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2432 rootnex_window_t *window; 2433 ddi_dma_cookie_t *cp; 2434 ddi_dma_cookie_t *cookie; 2435 2436 ASSERT(*cookiepp == NULL); 2437 ASSERT(*ccountp == 0); 2438 2439 if (dma->dp_window) { 2440 window = &dma->dp_window[dma->dp_current_win]; 2441 cp = window->wd_first_cookie; 2442 *ccountp = window->wd_cookie_cnt; 2443 } else { 2444 cp = dma->dp_cookies; 2445 *ccountp = dma->dp_sglinfo.si_sgl_size; 2446 } 2447 2448 km_flags = rootnex_coredma_get_sleep_flags(handle); 2449 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2450 if (cookie == NULL) { 2451 return (DDI_DMA_NORESOURCES); 2452 } 2453 2454 for (i = 0; i < *ccountp; i++) { 2455 cookie[i].dmac_notused = cp[i].dmac_notused; 2456 cookie[i].dmac_type = cp[i].dmac_type; 2457 cookie[i].dmac_address = cp[i].dmac_address; 2458 cookie[i].dmac_size = cp[i].dmac_size; 2459 } 2460 2461 *cookiepp = cookie; 2462 2463 return (DDI_SUCCESS); 2464 } 2465 2466 /*ARGSUSED*/ 2467 static int 2468 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2469 ddi_dma_cookie_t *cookiep, uint_t ccount) 2470 { 2471 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2472 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2473 rootnex_window_t *window; 2474 ddi_dma_cookie_t *cur_cookiep; 2475 2476 ASSERT(cookiep); 2477 ASSERT(ccount != 0); 2478 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2479 2480 if (dma->dp_window) { 2481 window = &dma->dp_window[dma->dp_current_win]; 2482 dma->dp_saved_cookies = window->wd_first_cookie; 2483 window->wd_first_cookie = cookiep; 2484 ASSERT(ccount == window->wd_cookie_cnt); 2485 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2486 + window->wd_first_cookie; 2487 } else { 2488 dma->dp_saved_cookies = dma->dp_cookies; 2489 dma->dp_cookies = cookiep; 2490 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2491 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2492 + dma->dp_cookies; 2493 } 2494 2495 dma->dp_need_to_switch_cookies = B_TRUE; 2496 hp->dmai_cookie = cur_cookiep; 2497 2498 return (DDI_SUCCESS); 2499 } 2500 2501 /*ARGSUSED*/ 2502 static int 2503 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2504 { 2505 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2506 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2507 rootnex_window_t *window; 2508 ddi_dma_cookie_t *cur_cookiep; 2509 ddi_dma_cookie_t *cookie_array; 2510 uint_t ccount; 2511 2512 /* check if cookies have not been switched */ 2513 if (dma->dp_need_to_switch_cookies == B_FALSE) 2514 return (DDI_SUCCESS); 2515 2516 ASSERT(dma->dp_saved_cookies); 2517 2518 if (dma->dp_window) { 2519 window = &dma->dp_window[dma->dp_current_win]; 2520 cookie_array = window->wd_first_cookie; 2521 window->wd_first_cookie = dma->dp_saved_cookies; 2522 dma->dp_saved_cookies = NULL; 2523 ccount = window->wd_cookie_cnt; 2524 cur_cookiep = (hp->dmai_cookie - cookie_array) 2525 + window->wd_first_cookie; 2526 } else { 2527 cookie_array = dma->dp_cookies; 2528 dma->dp_cookies = dma->dp_saved_cookies; 2529 dma->dp_saved_cookies = NULL; 2530 ccount = dma->dp_sglinfo.si_sgl_size; 2531 cur_cookiep = (hp->dmai_cookie - cookie_array) 2532 + dma->dp_cookies; 2533 } 2534 2535 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2536 2537 hp->dmai_cookie = cur_cookiep; 2538 2539 dma->dp_need_to_switch_cookies = B_FALSE; 2540 2541 return (DDI_SUCCESS); 2542 } 2543 2544 #endif 2545 2546 static struct as * 2547 rootnex_get_as(ddi_dma_obj_t *dmao) 2548 { 2549 struct as *asp; 2550 2551 switch (dmao->dmao_type) { 2552 case DMA_OTYP_VADDR: 2553 case DMA_OTYP_BUFVADDR: 2554 asp = dmao->dmao_obj.virt_obj.v_as; 2555 if (asp == NULL) 2556 asp = &kas; 2557 break; 2558 default: 2559 asp = NULL; 2560 break; 2561 } 2562 return (asp); 2563 } 2564 2565 /* 2566 * rootnex_verify_buffer() 2567 * verify buffer wasn't free'd 2568 */ 2569 static int 2570 rootnex_verify_buffer(rootnex_dma_t *dma) 2571 { 2572 page_t **pplist; 2573 caddr_t vaddr; 2574 uint_t pcnt; 2575 uint_t poff; 2576 page_t *pp; 2577 char b; 2578 int i; 2579 2580 /* Figure out how many pages this buffer occupies */ 2581 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2582 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2583 } else { 2584 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2585 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2586 } 2587 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2588 2589 switch (dma->dp_dma.dmao_type) { 2590 case DMA_OTYP_PAGES: 2591 /* 2592 * for a linked list of pp's walk through them to make sure 2593 * they're locked and not free. 2594 */ 2595 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2596 for (i = 0; i < pcnt; i++) { 2597 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2598 return (DDI_FAILURE); 2599 } 2600 pp = pp->p_next; 2601 } 2602 break; 2603 2604 case DMA_OTYP_VADDR: 2605 case DMA_OTYP_BUFVADDR: 2606 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2607 /* 2608 * for an array of pp's walk through them to make sure they're 2609 * not free. It's possible that they may not be locked. 2610 */ 2611 if (pplist) { 2612 for (i = 0; i < pcnt; i++) { 2613 if (PP_ISFREE(pplist[i])) { 2614 return (DDI_FAILURE); 2615 } 2616 } 2617 2618 /* For a virtual address, try to peek at each page */ 2619 } else { 2620 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2621 for (i = 0; i < pcnt; i++) { 2622 if (ddi_peek8(NULL, vaddr, &b) == 2623 DDI_FAILURE) 2624 return (DDI_FAILURE); 2625 vaddr += MMU_PAGESIZE; 2626 } 2627 } 2628 } 2629 break; 2630 2631 default: 2632 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2633 break; 2634 } 2635 2636 return (DDI_SUCCESS); 2637 } 2638 2639 2640 /* 2641 * rootnex_clean_dmahdl() 2642 * Clean the dma handle. This should be called on a handle alloc and an 2643 * unbind handle. Set the handle state to the default settings. 2644 */ 2645 static void 2646 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2647 { 2648 rootnex_dma_t *dma; 2649 2650 2651 dma = (rootnex_dma_t *)hp->dmai_private; 2652 2653 hp->dmai_nwin = 0; 2654 dma->dp_current_cookie = 0; 2655 dma->dp_copybuf_size = 0; 2656 dma->dp_window = NULL; 2657 dma->dp_cbaddr = NULL; 2658 dma->dp_inuse = B_FALSE; 2659 dma->dp_dvma_used = B_FALSE; 2660 dma->dp_need_to_free_cookie = B_FALSE; 2661 dma->dp_need_to_switch_cookies = B_FALSE; 2662 dma->dp_saved_cookies = NULL; 2663 dma->dp_sleep_flags = KM_PANIC; 2664 dma->dp_need_to_free_window = B_FALSE; 2665 dma->dp_partial_required = B_FALSE; 2666 dma->dp_trim_required = B_FALSE; 2667 dma->dp_sglinfo.si_copybuf_req = 0; 2668 #if !defined(__amd64) 2669 dma->dp_cb_remaping = B_FALSE; 2670 dma->dp_kva = NULL; 2671 #endif 2672 2673 /* FMA related initialization */ 2674 hp->dmai_fault = 0; 2675 hp->dmai_fault_check = NULL; 2676 hp->dmai_fault_notify = NULL; 2677 hp->dmai_error.err_ena = 0; 2678 hp->dmai_error.err_status = DDI_FM_OK; 2679 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2680 hp->dmai_error.err_ontrap = NULL; 2681 } 2682 2683 2684 /* 2685 * rootnex_valid_alloc_parms() 2686 * Called in ddi_dma_alloc_handle path to validate its parameters. 2687 */ 2688 static int 2689 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2690 { 2691 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2692 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2693 (attr->dma_attr_granular > MMU_PAGESIZE) || 2694 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2695 return (DDI_DMA_BADATTR); 2696 } 2697 2698 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2699 return (DDI_DMA_BADATTR); 2700 } 2701 2702 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2703 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2704 attr->dma_attr_sgllen == 0) { 2705 return (DDI_DMA_BADATTR); 2706 } 2707 2708 /* We should be able to DMA into every byte offset in a page */ 2709 if (maxsegmentsize < MMU_PAGESIZE) { 2710 return (DDI_DMA_BADATTR); 2711 } 2712 2713 /* if we're bouncing on seg, seg must be <= addr_hi */ 2714 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2715 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2716 return (DDI_DMA_BADATTR); 2717 } 2718 return (DDI_SUCCESS); 2719 } 2720 2721 /* 2722 * rootnex_valid_bind_parms() 2723 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2724 */ 2725 /* ARGSUSED */ 2726 static int 2727 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2728 { 2729 #if !defined(__amd64) 2730 /* 2731 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2732 * we can track the offset for the obsoleted interfaces. 2733 */ 2734 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2735 return (DDI_DMA_TOOBIG); 2736 } 2737 #endif 2738 2739 return (DDI_SUCCESS); 2740 } 2741 2742 2743 /* 2744 * rootnex_need_bounce_seg() 2745 * check to see if the buffer lives on both side of the seg. 2746 */ 2747 static boolean_t 2748 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2749 { 2750 ddi_dma_atyp_t buftype; 2751 rootnex_addr_t raddr; 2752 boolean_t lower_addr; 2753 boolean_t upper_addr; 2754 uint64_t offset; 2755 page_t **pplist; 2756 uint64_t paddr; 2757 uint32_t psize; 2758 uint32_t size; 2759 caddr_t vaddr; 2760 uint_t pcnt; 2761 page_t *pp; 2762 2763 2764 /* shortcuts */ 2765 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2766 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2767 buftype = dmar_object->dmao_type; 2768 size = dmar_object->dmao_size; 2769 2770 lower_addr = B_FALSE; 2771 upper_addr = B_FALSE; 2772 pcnt = 0; 2773 2774 /* 2775 * Process the first page to handle the initial offset of the buffer. 2776 * We'll use the base address we get later when we loop through all 2777 * the pages. 2778 */ 2779 if (buftype == DMA_OTYP_PAGES) { 2780 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2781 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2782 MMU_PAGEOFFSET; 2783 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2784 psize = MIN(size, (MMU_PAGESIZE - offset)); 2785 pp = pp->p_next; 2786 sglinfo->si_asp = NULL; 2787 } else if (pplist != NULL) { 2788 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2789 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2790 if (sglinfo->si_asp == NULL) { 2791 sglinfo->si_asp = &kas; 2792 } 2793 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2794 paddr += offset; 2795 psize = MIN(size, (MMU_PAGESIZE - offset)); 2796 pcnt++; 2797 } else { 2798 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2799 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2800 if (sglinfo->si_asp == NULL) { 2801 sglinfo->si_asp = &kas; 2802 } 2803 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2804 paddr += offset; 2805 psize = MIN(size, (MMU_PAGESIZE - offset)); 2806 vaddr += psize; 2807 } 2808 2809 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2810 2811 if ((raddr + psize) > sglinfo->si_segmask) { 2812 upper_addr = B_TRUE; 2813 } else { 2814 lower_addr = B_TRUE; 2815 } 2816 size -= psize; 2817 2818 /* 2819 * Walk through the rest of the pages in the buffer. Track to see 2820 * if we have pages on both sides of the segment boundary. 2821 */ 2822 while (size > 0) { 2823 /* partial or full page */ 2824 psize = MIN(size, MMU_PAGESIZE); 2825 2826 if (buftype == DMA_OTYP_PAGES) { 2827 /* get the paddr from the page_t */ 2828 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2829 paddr = pfn_to_pa(pp->p_pagenum); 2830 pp = pp->p_next; 2831 } else if (pplist != NULL) { 2832 /* index into the array of page_t's to get the paddr */ 2833 ASSERT(!PP_ISFREE(pplist[pcnt])); 2834 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2835 pcnt++; 2836 } else { 2837 /* call into the VM to get the paddr */ 2838 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2839 vaddr)); 2840 vaddr += psize; 2841 } 2842 2843 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2844 2845 if ((raddr + psize) > sglinfo->si_segmask) { 2846 upper_addr = B_TRUE; 2847 } else { 2848 lower_addr = B_TRUE; 2849 } 2850 /* 2851 * if the buffer lives both above and below the segment 2852 * boundary, or the current page is the page immediately 2853 * after the segment, we will use a copy/bounce buffer for 2854 * all pages > seg. 2855 */ 2856 if ((lower_addr && upper_addr) || 2857 (raddr == (sglinfo->si_segmask + 1))) { 2858 return (B_TRUE); 2859 } 2860 2861 size -= psize; 2862 } 2863 2864 return (B_FALSE); 2865 } 2866 2867 /* 2868 * rootnex_get_sgl() 2869 * Called in bind fastpath to get the sgl. Most of this will be replaced 2870 * with a call to the vm layer when vm2.0 comes around... 2871 */ 2872 static void 2873 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2874 rootnex_sglinfo_t *sglinfo) 2875 { 2876 ddi_dma_atyp_t buftype; 2877 rootnex_addr_t raddr; 2878 uint64_t last_page; 2879 uint64_t offset; 2880 uint64_t addrhi; 2881 uint64_t addrlo; 2882 uint64_t maxseg; 2883 page_t **pplist; 2884 uint64_t paddr; 2885 uint32_t psize; 2886 uint32_t size; 2887 caddr_t vaddr; 2888 uint_t pcnt; 2889 page_t *pp; 2890 uint_t cnt; 2891 2892 2893 /* shortcuts */ 2894 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2895 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2896 maxseg = sglinfo->si_max_cookie_size; 2897 buftype = dmar_object->dmao_type; 2898 addrhi = sglinfo->si_max_addr; 2899 addrlo = sglinfo->si_min_addr; 2900 size = dmar_object->dmao_size; 2901 2902 pcnt = 0; 2903 cnt = 0; 2904 2905 2906 /* 2907 * check to see if we need to use the copy buffer for pages over 2908 * the segment attr. 2909 */ 2910 sglinfo->si_bounce_on_seg = B_FALSE; 2911 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2912 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2913 dmar_object, sglinfo); 2914 } 2915 2916 /* 2917 * if we were passed down a linked list of pages, i.e. pointer to 2918 * page_t, use this to get our physical address and buf offset. 2919 */ 2920 if (buftype == DMA_OTYP_PAGES) { 2921 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2922 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2923 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2924 MMU_PAGEOFFSET; 2925 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2926 psize = MIN(size, (MMU_PAGESIZE - offset)); 2927 pp = pp->p_next; 2928 sglinfo->si_asp = NULL; 2929 2930 /* 2931 * We weren't passed down a linked list of pages, but if we were passed 2932 * down an array of pages, use this to get our physical address and buf 2933 * offset. 2934 */ 2935 } else if (pplist != NULL) { 2936 ASSERT((buftype == DMA_OTYP_VADDR) || 2937 (buftype == DMA_OTYP_BUFVADDR)); 2938 2939 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2940 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2941 if (sglinfo->si_asp == NULL) { 2942 sglinfo->si_asp = &kas; 2943 } 2944 2945 ASSERT(!PP_ISFREE(pplist[pcnt])); 2946 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2947 paddr += offset; 2948 psize = MIN(size, (MMU_PAGESIZE - offset)); 2949 pcnt++; 2950 2951 /* 2952 * All we have is a virtual address, we'll need to call into the VM 2953 * to get the physical address. 2954 */ 2955 } else { 2956 ASSERT((buftype == DMA_OTYP_VADDR) || 2957 (buftype == DMA_OTYP_BUFVADDR)); 2958 2959 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2960 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2961 if (sglinfo->si_asp == NULL) { 2962 sglinfo->si_asp = &kas; 2963 } 2964 2965 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2966 paddr += offset; 2967 psize = MIN(size, (MMU_PAGESIZE - offset)); 2968 vaddr += psize; 2969 } 2970 2971 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2972 2973 /* 2974 * Setup the first cookie with the physical address of the page and the 2975 * size of the page (which takes into account the initial offset into 2976 * the page. 2977 */ 2978 sgl[cnt].dmac_laddress = raddr; 2979 sgl[cnt].dmac_size = psize; 2980 sgl[cnt].dmac_type = 0; 2981 2982 /* 2983 * Save away the buffer offset into the page. We'll need this later in 2984 * the copy buffer code to help figure out the page index within the 2985 * buffer and the offset into the current page. 2986 */ 2987 sglinfo->si_buf_offset = offset; 2988 2989 /* 2990 * If we are using the copy buffer for anything over the segment 2991 * boundary, and this page is over the segment boundary. 2992 * OR 2993 * if the DMA engine can't reach the physical address. 2994 */ 2995 if (((sglinfo->si_bounce_on_seg) && 2996 ((raddr + psize) > sglinfo->si_segmask)) || 2997 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2998 /* 2999 * Increase how much copy buffer we use. We always increase by 3000 * pagesize so we don't have to worry about converting offsets. 3001 * Set a flag in the cookies dmac_type to indicate that it uses 3002 * the copy buffer. If this isn't the last cookie, go to the 3003 * next cookie (since we separate each page which uses the copy 3004 * buffer in case the copy buffer is not physically contiguous. 3005 */ 3006 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3007 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3008 if ((cnt + 1) < sglinfo->si_max_pages) { 3009 cnt++; 3010 sgl[cnt].dmac_laddress = 0; 3011 sgl[cnt].dmac_size = 0; 3012 sgl[cnt].dmac_type = 0; 3013 } 3014 } 3015 3016 /* 3017 * save this page's physical address so we can figure out if the next 3018 * page is physically contiguous. Keep decrementing size until we are 3019 * done with the buffer. 3020 */ 3021 last_page = raddr & MMU_PAGEMASK; 3022 size -= psize; 3023 3024 while (size > 0) { 3025 /* Get the size for this page (i.e. partial or full page) */ 3026 psize = MIN(size, MMU_PAGESIZE); 3027 3028 if (buftype == DMA_OTYP_PAGES) { 3029 /* get the paddr from the page_t */ 3030 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3031 paddr = pfn_to_pa(pp->p_pagenum); 3032 pp = pp->p_next; 3033 } else if (pplist != NULL) { 3034 /* index into the array of page_t's to get the paddr */ 3035 ASSERT(!PP_ISFREE(pplist[pcnt])); 3036 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3037 pcnt++; 3038 } else { 3039 /* call into the VM to get the paddr */ 3040 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3041 vaddr)); 3042 vaddr += psize; 3043 } 3044 3045 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3046 3047 /* 3048 * If we are using the copy buffer for anything over the 3049 * segment boundary, and this page is over the segment 3050 * boundary. 3051 * OR 3052 * if the DMA engine can't reach the physical address. 3053 */ 3054 if (((sglinfo->si_bounce_on_seg) && 3055 ((raddr + psize) > sglinfo->si_segmask)) || 3056 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3057 3058 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3059 3060 /* 3061 * if there is something in the current cookie, go to 3062 * the next one. We only want one page in a cookie which 3063 * uses the copybuf since the copybuf doesn't have to 3064 * be physically contiguous. 3065 */ 3066 if (sgl[cnt].dmac_size != 0) { 3067 cnt++; 3068 } 3069 sgl[cnt].dmac_laddress = raddr; 3070 sgl[cnt].dmac_size = psize; 3071 #if defined(__amd64) 3072 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3073 #else 3074 /* 3075 * save the buf offset for 32-bit kernel. used in the 3076 * obsoleted interfaces. 3077 */ 3078 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3079 (dmar_object->dmao_size - size); 3080 #endif 3081 /* if this isn't the last cookie, go to the next one */ 3082 if ((cnt + 1) < sglinfo->si_max_pages) { 3083 cnt++; 3084 sgl[cnt].dmac_laddress = 0; 3085 sgl[cnt].dmac_size = 0; 3086 sgl[cnt].dmac_type = 0; 3087 } 3088 3089 /* 3090 * this page didn't need the copy buffer, if it's not physically 3091 * contiguous, or it would put us over a segment boundary, or it 3092 * puts us over the max cookie size, or the current sgl doesn't 3093 * have anything in it. 3094 */ 3095 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3096 !(raddr & sglinfo->si_segmask) || 3097 ((sgl[cnt].dmac_size + psize) > maxseg) || 3098 (sgl[cnt].dmac_size == 0)) { 3099 /* 3100 * if we're not already in a new cookie, go to the next 3101 * cookie. 3102 */ 3103 if (sgl[cnt].dmac_size != 0) { 3104 cnt++; 3105 } 3106 3107 /* save the cookie information */ 3108 sgl[cnt].dmac_laddress = raddr; 3109 sgl[cnt].dmac_size = psize; 3110 #if defined(__amd64) 3111 sgl[cnt].dmac_type = 0; 3112 #else 3113 /* 3114 * save the buf offset for 32-bit kernel. used in the 3115 * obsoleted interfaces. 3116 */ 3117 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3118 #endif 3119 3120 /* 3121 * this page didn't need the copy buffer, it is physically 3122 * contiguous with the last page, and it's <= the max cookie 3123 * size. 3124 */ 3125 } else { 3126 sgl[cnt].dmac_size += psize; 3127 3128 /* 3129 * if this exactly == the maximum cookie size, and 3130 * it isn't the last cookie, go to the next cookie. 3131 */ 3132 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3133 ((cnt + 1) < sglinfo->si_max_pages)) { 3134 cnt++; 3135 sgl[cnt].dmac_laddress = 0; 3136 sgl[cnt].dmac_size = 0; 3137 sgl[cnt].dmac_type = 0; 3138 } 3139 } 3140 3141 /* 3142 * save this page's physical address so we can figure out if the 3143 * next page is physically contiguous. Keep decrementing size 3144 * until we are done with the buffer. 3145 */ 3146 last_page = raddr; 3147 size -= psize; 3148 } 3149 3150 /* we're done, save away how many cookies the sgl has */ 3151 if (sgl[cnt].dmac_size == 0) { 3152 ASSERT(cnt < sglinfo->si_max_pages); 3153 sglinfo->si_sgl_size = cnt; 3154 } else { 3155 sglinfo->si_sgl_size = cnt + 1; 3156 } 3157 } 3158 3159 static void 3160 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3161 rootnex_sglinfo_t *sglinfo) 3162 { 3163 uint64_t offset; 3164 uint64_t maxseg; 3165 uint64_t dvaddr; 3166 struct dvmaseg *dvs; 3167 uint64_t paddr; 3168 uint32_t psize, ssize; 3169 uint32_t size; 3170 uint_t cnt; 3171 int physcontig; 3172 3173 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3174 3175 /* shortcuts */ 3176 maxseg = sglinfo->si_max_cookie_size; 3177 size = dmar_object->dmao_size; 3178 3179 cnt = 0; 3180 sglinfo->si_bounce_on_seg = B_FALSE; 3181 3182 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3183 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3184 ssize = dvs->dvs_len; 3185 paddr = dvs->dvs_start; 3186 paddr += offset; 3187 psize = MIN(ssize, (maxseg - offset)); 3188 dvaddr = paddr + psize; 3189 ssize -= psize; 3190 3191 sgl[cnt].dmac_laddress = paddr; 3192 sgl[cnt].dmac_size = psize; 3193 sgl[cnt].dmac_type = 0; 3194 3195 size -= psize; 3196 while (size > 0) { 3197 if (ssize == 0) { 3198 dvs++; 3199 ssize = dvs->dvs_len; 3200 dvaddr = dvs->dvs_start; 3201 physcontig = 0; 3202 } else 3203 physcontig = 1; 3204 3205 paddr = dvaddr; 3206 psize = MIN(ssize, maxseg); 3207 dvaddr += psize; 3208 ssize -= psize; 3209 3210 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3211 ((sgl[cnt].dmac_size + psize) > maxseg) || 3212 (sgl[cnt].dmac_size == 0)) { 3213 /* 3214 * if we're not already in a new cookie, go to the next 3215 * cookie. 3216 */ 3217 if (sgl[cnt].dmac_size != 0) { 3218 cnt++; 3219 } 3220 3221 /* save the cookie information */ 3222 sgl[cnt].dmac_laddress = paddr; 3223 sgl[cnt].dmac_size = psize; 3224 sgl[cnt].dmac_type = 0; 3225 } else { 3226 sgl[cnt].dmac_size += psize; 3227 3228 /* 3229 * if this exactly == the maximum cookie size, and 3230 * it isn't the last cookie, go to the next cookie. 3231 */ 3232 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3233 ((cnt + 1) < sglinfo->si_max_pages)) { 3234 cnt++; 3235 sgl[cnt].dmac_laddress = 0; 3236 sgl[cnt].dmac_size = 0; 3237 sgl[cnt].dmac_type = 0; 3238 } 3239 } 3240 size -= psize; 3241 } 3242 3243 /* we're done, save away how many cookies the sgl has */ 3244 if (sgl[cnt].dmac_size == 0) { 3245 sglinfo->si_sgl_size = cnt; 3246 } else { 3247 sglinfo->si_sgl_size = cnt + 1; 3248 } 3249 } 3250 3251 /* 3252 * rootnex_bind_slowpath() 3253 * Call in the bind path if the calling driver can't use the sgl without 3254 * modifying it. We either need to use the copy buffer and/or we will end up 3255 * with a partial bind. 3256 */ 3257 static int 3258 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3259 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3260 { 3261 rootnex_sglinfo_t *sinfo; 3262 rootnex_window_t *window; 3263 ddi_dma_cookie_t *cookie; 3264 size_t copybuf_used; 3265 size_t dmac_size; 3266 boolean_t partial; 3267 off_t cur_offset; 3268 page_t *cur_pp; 3269 major_t mnum; 3270 int e; 3271 int i; 3272 3273 3274 sinfo = &dma->dp_sglinfo; 3275 copybuf_used = 0; 3276 partial = B_FALSE; 3277 3278 /* 3279 * If we're using the copybuf, set the copybuf state in dma struct. 3280 * Needs to be first since it sets the copy buffer size. 3281 */ 3282 if (sinfo->si_copybuf_req != 0) { 3283 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3284 if (e != DDI_SUCCESS) { 3285 return (e); 3286 } 3287 } else { 3288 dma->dp_copybuf_size = 0; 3289 } 3290 3291 /* 3292 * Figure out if we need to do a partial mapping. If so, figure out 3293 * if we need to trim the buffers when we munge the sgl. 3294 */ 3295 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3296 (dmao->dmao_size > dma->dp_maxxfer) || 3297 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3298 dma->dp_partial_required = B_TRUE; 3299 if (attr->dma_attr_granular != 1) { 3300 dma->dp_trim_required = B_TRUE; 3301 } 3302 } else { 3303 dma->dp_partial_required = B_FALSE; 3304 dma->dp_trim_required = B_FALSE; 3305 } 3306 3307 /* If we need to do a partial bind, make sure the driver supports it */ 3308 if (dma->dp_partial_required && 3309 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3310 3311 mnum = ddi_driver_major(dma->dp_dip); 3312 /* 3313 * patchable which allows us to print one warning per major 3314 * number. 3315 */ 3316 if ((rootnex_bind_warn) && 3317 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3318 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3319 cmn_err(CE_WARN, "!%s: coding error detected, the " 3320 "driver is using ddi_dma_attr(9S) incorrectly. " 3321 "There is a small risk of data corruption in " 3322 "particular with large I/Os. The driver should be " 3323 "replaced with a corrected version for proper " 3324 "system operation. To disable this warning, add " 3325 "'set rootnex:rootnex_bind_warn=0' to " 3326 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3327 } 3328 return (DDI_DMA_TOOBIG); 3329 } 3330 3331 /* 3332 * we might need multiple windows, setup state to handle them. In this 3333 * code path, we will have at least one window. 3334 */ 3335 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3336 if (e != DDI_SUCCESS) { 3337 rootnex_teardown_copybuf(dma); 3338 return (e); 3339 } 3340 3341 window = &dma->dp_window[0]; 3342 cookie = &dma->dp_cookies[0]; 3343 cur_offset = 0; 3344 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3345 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3346 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3347 } 3348 3349 /* loop though all the cookies we got back from get_sgl() */ 3350 for (i = 0; i < sinfo->si_sgl_size; i++) { 3351 /* 3352 * If we're using the copy buffer, check this cookie and setup 3353 * its associated copy buffer state. If this cookie uses the 3354 * copy buffer, make sure we sync this window during dma_sync. 3355 */ 3356 if (dma->dp_copybuf_size > 0) { 3357 rootnex_setup_cookie(dmao, dma, cookie, 3358 cur_offset, ©buf_used, &cur_pp); 3359 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3360 window->wd_dosync = B_TRUE; 3361 } 3362 } 3363 3364 /* 3365 * save away the cookie size, since it could be modified in 3366 * the windowing code. 3367 */ 3368 dmac_size = cookie->dmac_size; 3369 3370 /* if we went over max copybuf size */ 3371 if (dma->dp_copybuf_size && 3372 (copybuf_used > dma->dp_copybuf_size)) { 3373 partial = B_TRUE; 3374 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3375 cookie, cur_offset, ©buf_used); 3376 if (e != DDI_SUCCESS) { 3377 rootnex_teardown_copybuf(dma); 3378 rootnex_teardown_windows(dma); 3379 return (e); 3380 } 3381 3382 /* 3383 * if the coookie uses the copy buffer, make sure the 3384 * new window we just moved to is set to sync. 3385 */ 3386 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3387 window->wd_dosync = B_TRUE; 3388 } 3389 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3390 dma->dp_dip); 3391 3392 /* if the cookie cnt == max sgllen, move to the next window */ 3393 } else if (window->wd_cookie_cnt >= 3394 (unsigned)attr->dma_attr_sgllen) { 3395 partial = B_TRUE; 3396 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3397 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3398 cookie, attr, cur_offset); 3399 if (e != DDI_SUCCESS) { 3400 rootnex_teardown_copybuf(dma); 3401 rootnex_teardown_windows(dma); 3402 return (e); 3403 } 3404 3405 /* 3406 * if the coookie uses the copy buffer, make sure the 3407 * new window we just moved to is set to sync. 3408 */ 3409 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3410 window->wd_dosync = B_TRUE; 3411 } 3412 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3413 dma->dp_dip); 3414 3415 /* else if we will be over maxxfer */ 3416 } else if ((window->wd_size + dmac_size) > 3417 dma->dp_maxxfer) { 3418 partial = B_TRUE; 3419 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3420 cookie); 3421 if (e != DDI_SUCCESS) { 3422 rootnex_teardown_copybuf(dma); 3423 rootnex_teardown_windows(dma); 3424 return (e); 3425 } 3426 3427 /* 3428 * if the coookie uses the copy buffer, make sure the 3429 * new window we just moved to is set to sync. 3430 */ 3431 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3432 window->wd_dosync = B_TRUE; 3433 } 3434 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3435 dma->dp_dip); 3436 3437 /* else this cookie fits in the current window */ 3438 } else { 3439 window->wd_cookie_cnt++; 3440 window->wd_size += dmac_size; 3441 } 3442 3443 /* track our offset into the buffer, go to the next cookie */ 3444 ASSERT(dmac_size <= dmao->dmao_size); 3445 ASSERT(cookie->dmac_size <= dmac_size); 3446 cur_offset += dmac_size; 3447 cookie++; 3448 } 3449 3450 /* if we ended up with a zero sized window in the end, clean it up */ 3451 if (window->wd_size == 0) { 3452 hp->dmai_nwin--; 3453 window--; 3454 } 3455 3456 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3457 3458 if (!partial) { 3459 return (DDI_DMA_MAPPED); 3460 } 3461 3462 ASSERT(dma->dp_partial_required); 3463 return (DDI_DMA_PARTIAL_MAP); 3464 } 3465 3466 /* 3467 * rootnex_setup_copybuf() 3468 * Called in bind slowpath. Figures out if we're going to use the copy 3469 * buffer, and if we do, sets up the basic state to handle it. 3470 */ 3471 static int 3472 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3473 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3474 { 3475 rootnex_sglinfo_t *sinfo; 3476 ddi_dma_attr_t lattr; 3477 size_t max_copybuf; 3478 int cansleep; 3479 int e; 3480 #if !defined(__amd64) 3481 int vmflag; 3482 #endif 3483 3484 ASSERT(!dma->dp_dvma_used); 3485 3486 sinfo = &dma->dp_sglinfo; 3487 3488 /* read this first so it's consistent through the routine */ 3489 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3490 3491 /* We need to call into the rootnex on ddi_dma_sync() */ 3492 hp->dmai_rflags &= ~DMP_NOSYNC; 3493 3494 /* make sure the copybuf size <= the max size */ 3495 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3496 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3497 3498 #if !defined(__amd64) 3499 /* 3500 * if we don't have kva space to copy to/from, allocate the KVA space 3501 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3502 * the 64-bit kernel. 3503 */ 3504 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3505 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3506 3507 /* convert the sleep flags */ 3508 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3509 vmflag = VM_SLEEP; 3510 } else { 3511 vmflag = VM_NOSLEEP; 3512 } 3513 3514 /* allocate Kernel VA space that we can bcopy to/from */ 3515 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3516 vmflag); 3517 if (dma->dp_kva == NULL) { 3518 return (DDI_DMA_NORESOURCES); 3519 } 3520 } 3521 #endif 3522 3523 /* convert the sleep flags */ 3524 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3525 cansleep = 1; 3526 } else { 3527 cansleep = 0; 3528 } 3529 3530 /* 3531 * Allocate the actual copy buffer. This needs to fit within the DMA 3532 * engine limits, so we can't use kmem_alloc... We don't need 3533 * contiguous memory (sgllen) since we will be forcing windows on 3534 * sgllen anyway. 3535 */ 3536 lattr = *attr; 3537 lattr.dma_attr_align = MMU_PAGESIZE; 3538 lattr.dma_attr_sgllen = -1; /* no limit */ 3539 /* 3540 * if we're using the copy buffer because of seg, use that for our 3541 * upper address limit. 3542 */ 3543 if (sinfo->si_bounce_on_seg) { 3544 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3545 } 3546 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3547 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3548 if (e != DDI_SUCCESS) { 3549 #if !defined(__amd64) 3550 if (dma->dp_kva != NULL) { 3551 vmem_free(heap_arena, dma->dp_kva, 3552 dma->dp_copybuf_size); 3553 } 3554 #endif 3555 return (DDI_DMA_NORESOURCES); 3556 } 3557 3558 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3559 size_t, dma->dp_copybuf_size); 3560 3561 return (DDI_SUCCESS); 3562 } 3563 3564 3565 /* 3566 * rootnex_setup_windows() 3567 * Called in bind slowpath to setup the window state. We always have windows 3568 * in the slowpath. Even if the window count = 1. 3569 */ 3570 static int 3571 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3572 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3573 { 3574 rootnex_window_t *windowp; 3575 rootnex_sglinfo_t *sinfo; 3576 size_t copy_state_size; 3577 size_t win_state_size; 3578 size_t state_available; 3579 size_t space_needed; 3580 uint_t copybuf_win; 3581 uint_t maxxfer_win; 3582 size_t space_used; 3583 uint_t sglwin; 3584 3585 3586 sinfo = &dma->dp_sglinfo; 3587 3588 dma->dp_current_win = 0; 3589 hp->dmai_nwin = 0; 3590 3591 /* If we don't need to do a partial, we only have one window */ 3592 if (!dma->dp_partial_required) { 3593 dma->dp_max_win = 1; 3594 3595 /* 3596 * we need multiple windows, need to figure out the worse case number 3597 * of windows. 3598 */ 3599 } else { 3600 /* 3601 * if we need windows because we need more copy buffer that 3602 * we allow, the worse case number of windows we could need 3603 * here would be (copybuf space required / copybuf space that 3604 * we have) plus one for remainder, and plus 2 to handle the 3605 * extra pages on the trim for the first and last pages of the 3606 * buffer (a page is the minimum window size so under the right 3607 * attr settings, you could have a window for each page). 3608 * The last page will only be hit here if the size is not a 3609 * multiple of the granularity (which theoretically shouldn't 3610 * be the case but never has been enforced, so we could have 3611 * broken things without it). 3612 */ 3613 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3614 ASSERT(dma->dp_copybuf_size > 0); 3615 copybuf_win = (sinfo->si_copybuf_req / 3616 dma->dp_copybuf_size) + 1 + 2; 3617 } else { 3618 copybuf_win = 0; 3619 } 3620 3621 /* 3622 * if we need windows because we have more cookies than the H/W 3623 * can handle, the number of windows we would need here would 3624 * be (cookie count / cookies count H/W supports minus 1[for 3625 * trim]) plus one for remainder. 3626 */ 3627 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3628 sglwin = (sinfo->si_sgl_size / 3629 (attr->dma_attr_sgllen - 1)) + 1; 3630 } else { 3631 sglwin = 0; 3632 } 3633 3634 /* 3635 * if we need windows because we're binding more memory than the 3636 * H/W can transfer at once, the number of windows we would need 3637 * here would be (xfer count / max xfer H/W supports) plus one 3638 * for remainder, and plus 2 to handle the extra pages on the 3639 * trim (see above comment about trim) 3640 */ 3641 if (dmao->dmao_size > dma->dp_maxxfer) { 3642 maxxfer_win = (dmao->dmao_size / 3643 dma->dp_maxxfer) + 1 + 2; 3644 } else { 3645 maxxfer_win = 0; 3646 } 3647 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3648 ASSERT(dma->dp_max_win > 0); 3649 } 3650 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3651 3652 /* 3653 * Get space for window and potential copy buffer state. Before we 3654 * go and allocate memory, see if we can get away with using what's 3655 * left in the pre-allocted state or the dynamically allocated sgl. 3656 */ 3657 space_used = (uintptr_t)(sinfo->si_sgl_size * 3658 sizeof (ddi_dma_cookie_t)); 3659 3660 /* if we dynamically allocated space for the cookies */ 3661 if (dma->dp_need_to_free_cookie) { 3662 /* if we have more space in the pre-allocted buffer, use it */ 3663 ASSERT(space_used <= dma->dp_cookie_size); 3664 if ((dma->dp_cookie_size - space_used) <= 3665 rootnex_state->r_prealloc_size) { 3666 state_available = rootnex_state->r_prealloc_size; 3667 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3668 3669 /* 3670 * else, we have more free space in the dynamically allocated 3671 * buffer, i.e. the buffer wasn't worse case fragmented so we 3672 * didn't need a lot of cookies. 3673 */ 3674 } else { 3675 state_available = dma->dp_cookie_size - space_used; 3676 windowp = (rootnex_window_t *) 3677 &dma->dp_cookies[sinfo->si_sgl_size]; 3678 } 3679 3680 /* we used the pre-alloced buffer */ 3681 } else { 3682 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3683 state_available = rootnex_state->r_prealloc_size - space_used; 3684 windowp = (rootnex_window_t *) 3685 &dma->dp_cookies[sinfo->si_sgl_size]; 3686 } 3687 3688 /* 3689 * figure out how much state we need to track the copy buffer. Add an 3690 * addition 8 bytes for pointer alignemnt later. 3691 */ 3692 if (dma->dp_copybuf_size > 0) { 3693 copy_state_size = sinfo->si_max_pages * 3694 sizeof (rootnex_pgmap_t); 3695 } else { 3696 copy_state_size = 0; 3697 } 3698 /* add an additional 8 bytes for pointer alignment */ 3699 space_needed = win_state_size + copy_state_size + 0x8; 3700 3701 /* if we have enough space already, use it */ 3702 if (state_available >= space_needed) { 3703 dma->dp_window = windowp; 3704 dma->dp_need_to_free_window = B_FALSE; 3705 3706 /* not enough space, need to allocate more. */ 3707 } else { 3708 dma->dp_window = kmem_alloc(space_needed, kmflag); 3709 if (dma->dp_window == NULL) { 3710 return (DDI_DMA_NORESOURCES); 3711 } 3712 dma->dp_need_to_free_window = B_TRUE; 3713 dma->dp_window_size = space_needed; 3714 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3715 dma->dp_dip, size_t, space_needed); 3716 } 3717 3718 /* 3719 * we allocate copy buffer state and window state at the same time. 3720 * setup our copy buffer state pointers. Make sure it's aligned. 3721 */ 3722 if (dma->dp_copybuf_size > 0) { 3723 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3724 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3725 3726 #if !defined(__amd64) 3727 /* 3728 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3729 * false/NULL. Should be quicker to bzero vs loop and set. 3730 */ 3731 bzero(dma->dp_pgmap, copy_state_size); 3732 #endif 3733 } else { 3734 dma->dp_pgmap = NULL; 3735 } 3736 3737 return (DDI_SUCCESS); 3738 } 3739 3740 3741 /* 3742 * rootnex_teardown_copybuf() 3743 * cleans up after rootnex_setup_copybuf() 3744 */ 3745 static void 3746 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3747 { 3748 #if !defined(__amd64) 3749 int i; 3750 3751 /* 3752 * if we allocated kernel heap VMEM space, go through all the pages and 3753 * map out any of the ones that we're mapped into the kernel heap VMEM 3754 * arena. Then free the VMEM space. 3755 */ 3756 if (dma->dp_kva != NULL) { 3757 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3758 if (dma->dp_pgmap[i].pm_mapped) { 3759 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3760 MMU_PAGESIZE, HAT_UNLOAD); 3761 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3762 } 3763 } 3764 3765 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3766 } 3767 3768 #endif 3769 3770 /* if we allocated a copy buffer, free it */ 3771 if (dma->dp_cbaddr != NULL) { 3772 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3773 } 3774 } 3775 3776 3777 /* 3778 * rootnex_teardown_windows() 3779 * cleans up after rootnex_setup_windows() 3780 */ 3781 static void 3782 rootnex_teardown_windows(rootnex_dma_t *dma) 3783 { 3784 /* 3785 * if we had to allocate window state on the last bind (because we 3786 * didn't have enough pre-allocated space in the handle), free it. 3787 */ 3788 if (dma->dp_need_to_free_window) { 3789 kmem_free(dma->dp_window, dma->dp_window_size); 3790 } 3791 } 3792 3793 3794 /* 3795 * rootnex_init_win() 3796 * Called in bind slow path during creation of a new window. Initializes 3797 * window state to default values. 3798 */ 3799 /*ARGSUSED*/ 3800 static void 3801 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3802 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3803 { 3804 hp->dmai_nwin++; 3805 window->wd_dosync = B_FALSE; 3806 window->wd_offset = cur_offset; 3807 window->wd_size = 0; 3808 window->wd_first_cookie = cookie; 3809 window->wd_cookie_cnt = 0; 3810 window->wd_trim.tr_trim_first = B_FALSE; 3811 window->wd_trim.tr_trim_last = B_FALSE; 3812 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3813 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3814 #if !defined(__amd64) 3815 window->wd_remap_copybuf = dma->dp_cb_remaping; 3816 #endif 3817 } 3818 3819 3820 /* 3821 * rootnex_setup_cookie() 3822 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3823 * the sgl uses the copy buffer, we need to go through each cookie, figure 3824 * out if it uses the copy buffer, and if it does, save away everything we'll 3825 * need during sync. 3826 */ 3827 static void 3828 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3829 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3830 page_t **cur_pp) 3831 { 3832 boolean_t copybuf_sz_power_2; 3833 rootnex_sglinfo_t *sinfo; 3834 paddr_t paddr; 3835 uint_t pidx; 3836 uint_t pcnt; 3837 off_t poff; 3838 #if defined(__amd64) 3839 pfn_t pfn; 3840 #else 3841 page_t **pplist; 3842 #endif 3843 3844 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3845 3846 sinfo = &dma->dp_sglinfo; 3847 3848 /* 3849 * Calculate the page index relative to the start of the buffer. The 3850 * index to the current page for our buffer is the offset into the 3851 * first page of the buffer plus our current offset into the buffer 3852 * itself, shifted of course... 3853 */ 3854 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3855 ASSERT(pidx < sinfo->si_max_pages); 3856 3857 /* if this cookie uses the copy buffer */ 3858 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3859 /* 3860 * NOTE: we know that since this cookie uses the copy buffer, it 3861 * is <= MMU_PAGESIZE. 3862 */ 3863 3864 /* 3865 * get the offset into the page. For the 64-bit kernel, get the 3866 * pfn which we'll use with seg kpm. 3867 */ 3868 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3869 #if defined(__amd64) 3870 /* mfn_to_pfn() is a NOP on i86pc */ 3871 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3872 #endif /* __amd64 */ 3873 3874 /* figure out if the copybuf size is a power of 2 */ 3875 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3876 copybuf_sz_power_2 = B_FALSE; 3877 } else { 3878 copybuf_sz_power_2 = B_TRUE; 3879 } 3880 3881 /* This page uses the copy buffer */ 3882 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3883 3884 /* 3885 * save the copy buffer KVA that we'll use with this page. 3886 * if we still fit within the copybuf, it's a simple add. 3887 * otherwise, we need to wrap over using & or % accordingly. 3888 */ 3889 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3890 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3891 *copybuf_used; 3892 } else { 3893 if (copybuf_sz_power_2) { 3894 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3895 (uintptr_t)dma->dp_cbaddr + 3896 (*copybuf_used & 3897 (dma->dp_copybuf_size - 1))); 3898 } else { 3899 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3900 (uintptr_t)dma->dp_cbaddr + 3901 (*copybuf_used % dma->dp_copybuf_size)); 3902 } 3903 } 3904 3905 /* 3906 * over write the cookie physical address with the address of 3907 * the physical address of the copy buffer page that we will 3908 * use. 3909 */ 3910 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3911 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3912 3913 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3914 3915 /* if we have a kernel VA, it's easy, just save that address */ 3916 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3917 (sinfo->si_asp == &kas)) { 3918 /* 3919 * save away the page aligned virtual address of the 3920 * driver buffer. Offsets are handled in the sync code. 3921 */ 3922 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3923 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3924 & MMU_PAGEMASK); 3925 #if !defined(__amd64) 3926 /* 3927 * we didn't need to, and will never need to map this 3928 * page. 3929 */ 3930 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3931 #endif 3932 3933 /* we don't have a kernel VA. We need one for the bcopy. */ 3934 } else { 3935 #if defined(__amd64) 3936 /* 3937 * for the 64-bit kernel, it's easy. We use seg kpm to 3938 * get a Kernel VA for the corresponding pfn. 3939 */ 3940 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3941 #else 3942 /* 3943 * for the 32-bit kernel, this is a pain. First we'll 3944 * save away the page_t or user VA for this page. This 3945 * is needed in rootnex_dma_win() when we switch to a 3946 * new window which requires us to re-map the copy 3947 * buffer. 3948 */ 3949 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3950 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3951 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3952 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3953 } else if (pplist != NULL) { 3954 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3955 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3956 } else { 3957 dma->dp_pgmap[pidx].pm_pp = NULL; 3958 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3959 (((uintptr_t) 3960 dmar_object->dmao_obj.virt_obj.v_addr + 3961 cur_offset) & MMU_PAGEMASK); 3962 } 3963 3964 /* 3965 * save away the page aligned virtual address which was 3966 * allocated from the kernel heap arena (taking into 3967 * account if we need more copy buffer than we alloced 3968 * and use multiple windows to handle this, i.e. &,%). 3969 * NOTE: there isn't and physical memory backing up this 3970 * virtual address space currently. 3971 */ 3972 if ((*copybuf_used + MMU_PAGESIZE) <= 3973 dma->dp_copybuf_size) { 3974 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3975 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3976 MMU_PAGEMASK); 3977 } else { 3978 if (copybuf_sz_power_2) { 3979 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3980 (((uintptr_t)dma->dp_kva + 3981 (*copybuf_used & 3982 (dma->dp_copybuf_size - 1))) & 3983 MMU_PAGEMASK); 3984 } else { 3985 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3986 (((uintptr_t)dma->dp_kva + 3987 (*copybuf_used % 3988 dma->dp_copybuf_size)) & 3989 MMU_PAGEMASK); 3990 } 3991 } 3992 3993 /* 3994 * if we haven't used up the available copy buffer yet, 3995 * map the kva to the physical page. 3996 */ 3997 if (!dma->dp_cb_remaping && ((*copybuf_used + 3998 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3999 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 4000 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 4001 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 4002 dma->dp_pgmap[pidx].pm_kaddr); 4003 } else { 4004 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4005 sinfo->si_asp, 4006 dma->dp_pgmap[pidx].pm_kaddr); 4007 } 4008 4009 /* 4010 * we've used up the available copy buffer, this page 4011 * will have to be mapped during rootnex_dma_win() when 4012 * we switch to a new window which requires a re-map 4013 * the copy buffer. (32-bit kernel only) 4014 */ 4015 } else { 4016 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4017 } 4018 #endif 4019 /* go to the next page_t */ 4020 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4021 *cur_pp = (*cur_pp)->p_next; 4022 } 4023 } 4024 4025 /* add to the copy buffer count */ 4026 *copybuf_used += MMU_PAGESIZE; 4027 4028 /* 4029 * This cookie doesn't use the copy buffer. Walk through the pages this 4030 * cookie occupies to reflect this. 4031 */ 4032 } else { 4033 /* 4034 * figure out how many pages the cookie occupies. We need to 4035 * use the original page offset of the buffer and the cookies 4036 * offset in the buffer to do this. 4037 */ 4038 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4039 pcnt = mmu_btopr(cookie->dmac_size + poff); 4040 4041 while (pcnt > 0) { 4042 #if !defined(__amd64) 4043 /* 4044 * the 32-bit kernel doesn't have seg kpm, so we need 4045 * to map in the driver buffer (if it didn't come down 4046 * with a kernel VA) on the fly. Since this page doesn't 4047 * use the copy buffer, it's not, or will it ever, have 4048 * to be mapped in. 4049 */ 4050 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4051 #endif 4052 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4053 4054 /* 4055 * we need to update pidx and cur_pp or we'll loose 4056 * track of where we are. 4057 */ 4058 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4059 *cur_pp = (*cur_pp)->p_next; 4060 } 4061 pidx++; 4062 pcnt--; 4063 } 4064 } 4065 } 4066 4067 4068 /* 4069 * rootnex_sgllen_window_boundary() 4070 * Called in the bind slow path when the next cookie causes us to exceed (in 4071 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4072 * length supported by the DMA H/W. 4073 */ 4074 static int 4075 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4076 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4077 off_t cur_offset) 4078 { 4079 off_t new_offset; 4080 size_t trim_sz; 4081 off_t coffset; 4082 4083 4084 /* 4085 * if we know we'll never have to trim, it's pretty easy. Just move to 4086 * the next window and init it. We're done. 4087 */ 4088 if (!dma->dp_trim_required) { 4089 (*windowp)++; 4090 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4091 (*windowp)->wd_cookie_cnt++; 4092 (*windowp)->wd_size = cookie->dmac_size; 4093 return (DDI_SUCCESS); 4094 } 4095 4096 /* figure out how much we need to trim from the window */ 4097 ASSERT(attr->dma_attr_granular != 0); 4098 if (dma->dp_granularity_power_2) { 4099 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4100 } else { 4101 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4102 } 4103 4104 /* The window's a whole multiple of granularity. We're done */ 4105 if (trim_sz == 0) { 4106 (*windowp)++; 4107 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4108 (*windowp)->wd_cookie_cnt++; 4109 (*windowp)->wd_size = cookie->dmac_size; 4110 return (DDI_SUCCESS); 4111 } 4112 4113 /* 4114 * The window's not a whole multiple of granularity, since we know this 4115 * is due to the sgllen, we need to go back to the last cookie and trim 4116 * that one, add the left over part of the old cookie into the new 4117 * window, and then add in the new cookie into the new window. 4118 */ 4119 4120 /* 4121 * make sure the driver isn't making us do something bad... Trimming and 4122 * sgllen == 1 don't go together. 4123 */ 4124 if (attr->dma_attr_sgllen == 1) { 4125 return (DDI_DMA_NOMAPPING); 4126 } 4127 4128 /* 4129 * first, setup the current window to account for the trim. Need to go 4130 * back to the last cookie for this. 4131 */ 4132 cookie--; 4133 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4134 (*windowp)->wd_trim.tr_last_cookie = cookie; 4135 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4136 ASSERT(cookie->dmac_size > trim_sz); 4137 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4138 (*windowp)->wd_size -= trim_sz; 4139 4140 /* save the buffer offsets for the next window */ 4141 coffset = cookie->dmac_size - trim_sz; 4142 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4143 4144 /* 4145 * set this now in case this is the first window. all other cases are 4146 * set in dma_win() 4147 */ 4148 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4149 4150 /* 4151 * initialize the next window using what's left over in the previous 4152 * cookie. 4153 */ 4154 (*windowp)++; 4155 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4156 (*windowp)->wd_cookie_cnt++; 4157 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4158 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4159 (*windowp)->wd_trim.tr_first_size = trim_sz; 4160 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4161 (*windowp)->wd_dosync = B_TRUE; 4162 } 4163 4164 /* 4165 * now go back to the current cookie and add it to the new window. set 4166 * the new window size to the what was left over from the previous 4167 * cookie and what's in the current cookie. 4168 */ 4169 cookie++; 4170 (*windowp)->wd_cookie_cnt++; 4171 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4172 4173 /* 4174 * trim plus the next cookie could put us over maxxfer (a cookie can be 4175 * a max size of maxxfer). Handle that case. 4176 */ 4177 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4178 /* 4179 * maxxfer is already a whole multiple of granularity, and this 4180 * trim will be <= the previous trim (since a cookie can't be 4181 * larger than maxxfer). Make things simple here. 4182 */ 4183 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4184 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4185 (*windowp)->wd_trim.tr_last_cookie = cookie; 4186 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4187 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4188 (*windowp)->wd_size -= trim_sz; 4189 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4190 4191 /* save the buffer offsets for the next window */ 4192 coffset = cookie->dmac_size - trim_sz; 4193 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4194 4195 /* setup the next window */ 4196 (*windowp)++; 4197 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4198 (*windowp)->wd_cookie_cnt++; 4199 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4200 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4201 coffset; 4202 (*windowp)->wd_trim.tr_first_size = trim_sz; 4203 } 4204 4205 return (DDI_SUCCESS); 4206 } 4207 4208 4209 /* 4210 * rootnex_copybuf_window_boundary() 4211 * Called in bind slowpath when we get to a window boundary because we used 4212 * up all the copy buffer that we have. 4213 */ 4214 static int 4215 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4216 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4217 size_t *copybuf_used) 4218 { 4219 rootnex_sglinfo_t *sinfo; 4220 off_t new_offset; 4221 size_t trim_sz; 4222 paddr_t paddr; 4223 off_t coffset; 4224 uint_t pidx; 4225 off_t poff; 4226 4227 4228 sinfo = &dma->dp_sglinfo; 4229 4230 /* 4231 * the copy buffer should be a whole multiple of page size. We know that 4232 * this cookie is <= MMU_PAGESIZE. 4233 */ 4234 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4235 4236 /* 4237 * from now on, all new windows in this bind need to be re-mapped during 4238 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4239 * space... 4240 */ 4241 #if !defined(__amd64) 4242 dma->dp_cb_remaping = B_TRUE; 4243 #endif 4244 4245 /* reset copybuf used */ 4246 *copybuf_used = 0; 4247 4248 /* 4249 * if we don't have to trim (since granularity is set to 1), go to the 4250 * next window and add the current cookie to it. We know the current 4251 * cookie uses the copy buffer since we're in this code path. 4252 */ 4253 if (!dma->dp_trim_required) { 4254 (*windowp)++; 4255 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4256 4257 /* Add this cookie to the new window */ 4258 (*windowp)->wd_cookie_cnt++; 4259 (*windowp)->wd_size += cookie->dmac_size; 4260 *copybuf_used += MMU_PAGESIZE; 4261 return (DDI_SUCCESS); 4262 } 4263 4264 /* 4265 * *** may need to trim, figure it out. 4266 */ 4267 4268 /* figure out how much we need to trim from the window */ 4269 if (dma->dp_granularity_power_2) { 4270 trim_sz = (*windowp)->wd_size & 4271 (hp->dmai_attr.dma_attr_granular - 1); 4272 } else { 4273 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4274 } 4275 4276 /* 4277 * if the window's a whole multiple of granularity, go to the next 4278 * window, init it, then add in the current cookie. We know the current 4279 * cookie uses the copy buffer since we're in this code path. 4280 */ 4281 if (trim_sz == 0) { 4282 (*windowp)++; 4283 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4284 4285 /* Add this cookie to the new window */ 4286 (*windowp)->wd_cookie_cnt++; 4287 (*windowp)->wd_size += cookie->dmac_size; 4288 *copybuf_used += MMU_PAGESIZE; 4289 return (DDI_SUCCESS); 4290 } 4291 4292 /* 4293 * *** We figured it out, we definitly need to trim 4294 */ 4295 4296 /* 4297 * make sure the driver isn't making us do something bad... 4298 * Trimming and sgllen == 1 don't go together. 4299 */ 4300 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4301 return (DDI_DMA_NOMAPPING); 4302 } 4303 4304 /* 4305 * first, setup the current window to account for the trim. Need to go 4306 * back to the last cookie for this. Some of the last cookie will be in 4307 * the current window, and some of the last cookie will be in the new 4308 * window. All of the current cookie will be in the new window. 4309 */ 4310 cookie--; 4311 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4312 (*windowp)->wd_trim.tr_last_cookie = cookie; 4313 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4314 ASSERT(cookie->dmac_size > trim_sz); 4315 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4316 (*windowp)->wd_size -= trim_sz; 4317 4318 /* 4319 * we're trimming the last cookie (not the current cookie). So that 4320 * last cookie may have or may not have been using the copy buffer ( 4321 * we know the cookie passed in uses the copy buffer since we're in 4322 * this code path). 4323 * 4324 * If the last cookie doesn't use the copy buffer, nothing special to 4325 * do. However, if it does uses the copy buffer, it will be both the 4326 * last page in the current window and the first page in the next 4327 * window. Since we are reusing the copy buffer (and KVA space on the 4328 * 32-bit kernel), this page will use the end of the copy buffer in the 4329 * current window, and the start of the copy buffer in the next window. 4330 * Track that info... The cookie physical address was already set to 4331 * the copy buffer physical address in setup_cookie.. 4332 */ 4333 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4334 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4335 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4336 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4337 (*windowp)->wd_trim.tr_last_pidx = pidx; 4338 (*windowp)->wd_trim.tr_last_cbaddr = 4339 dma->dp_pgmap[pidx].pm_cbaddr; 4340 #if !defined(__amd64) 4341 (*windowp)->wd_trim.tr_last_kaddr = 4342 dma->dp_pgmap[pidx].pm_kaddr; 4343 #endif 4344 } 4345 4346 /* save the buffer offsets for the next window */ 4347 coffset = cookie->dmac_size - trim_sz; 4348 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4349 4350 /* 4351 * set this now in case this is the first window. all other cases are 4352 * set in dma_win() 4353 */ 4354 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4355 4356 /* 4357 * initialize the next window using what's left over in the previous 4358 * cookie. 4359 */ 4360 (*windowp)++; 4361 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4362 (*windowp)->wd_cookie_cnt++; 4363 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4364 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4365 (*windowp)->wd_trim.tr_first_size = trim_sz; 4366 4367 /* 4368 * again, we're tracking if the last cookie uses the copy buffer. 4369 * read the comment above for more info on why we need to track 4370 * additional state. 4371 * 4372 * For the first cookie in the new window, we need reset the physical 4373 * address to DMA into to the start of the copy buffer plus any 4374 * initial page offset which may be present. 4375 */ 4376 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4377 (*windowp)->wd_dosync = B_TRUE; 4378 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4379 (*windowp)->wd_trim.tr_first_pidx = pidx; 4380 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4381 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4382 4383 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4384 poff; 4385 (*windowp)->wd_trim.tr_first_paddr = 4386 ROOTNEX_PADDR_TO_RBASE(paddr); 4387 4388 #if !defined(__amd64) 4389 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4390 #endif 4391 /* account for the cookie copybuf usage in the new window */ 4392 *copybuf_used += MMU_PAGESIZE; 4393 4394 /* 4395 * every piece of code has to have a hack, and here is this 4396 * ones :-) 4397 * 4398 * There is a complex interaction between setup_cookie and the 4399 * copybuf window boundary. The complexity had to be in either 4400 * the maxxfer window, or the copybuf window, and I chose the 4401 * copybuf code. 4402 * 4403 * So in this code path, we have taken the last cookie, 4404 * virtually broken it in half due to the trim, and it happens 4405 * to use the copybuf which further complicates life. At the 4406 * same time, we have already setup the current cookie, which 4407 * is now wrong. More background info: the current cookie uses 4408 * the copybuf, so it is only a page long max. So we need to 4409 * fix the current cookies copy buffer address, physical 4410 * address, and kva for the 32-bit kernel. We due this by 4411 * bumping them by page size (of course, we can't due this on 4412 * the physical address since the copy buffer may not be 4413 * physically contiguous). 4414 */ 4415 cookie++; 4416 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4417 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4418 4419 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4420 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4421 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4422 4423 #if !defined(__amd64) 4424 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4425 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4426 #endif 4427 } else { 4428 /* go back to the current cookie */ 4429 cookie++; 4430 } 4431 4432 /* 4433 * add the current cookie to the new window. set the new window size to 4434 * the what was left over from the previous cookie and what's in the 4435 * current cookie. 4436 */ 4437 (*windowp)->wd_cookie_cnt++; 4438 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4439 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4440 4441 /* 4442 * we know that the cookie passed in always uses the copy buffer. We 4443 * wouldn't be here if it didn't. 4444 */ 4445 *copybuf_used += MMU_PAGESIZE; 4446 4447 return (DDI_SUCCESS); 4448 } 4449 4450 4451 /* 4452 * rootnex_maxxfer_window_boundary() 4453 * Called in bind slowpath when we get to a window boundary because we will 4454 * go over maxxfer. 4455 */ 4456 static int 4457 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4458 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4459 { 4460 size_t dmac_size; 4461 off_t new_offset; 4462 size_t trim_sz; 4463 off_t coffset; 4464 4465 4466 /* 4467 * calculate how much we have to trim off of the current cookie to equal 4468 * maxxfer. We don't have to account for granularity here since our 4469 * maxxfer already takes that into account. 4470 */ 4471 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4472 ASSERT(trim_sz <= cookie->dmac_size); 4473 ASSERT(trim_sz <= dma->dp_maxxfer); 4474 4475 /* save cookie size since we need it later and we might change it */ 4476 dmac_size = cookie->dmac_size; 4477 4478 /* 4479 * if we're not trimming the entire cookie, setup the current window to 4480 * account for the trim. 4481 */ 4482 if (trim_sz < cookie->dmac_size) { 4483 (*windowp)->wd_cookie_cnt++; 4484 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4485 (*windowp)->wd_trim.tr_last_cookie = cookie; 4486 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4487 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4488 (*windowp)->wd_size = dma->dp_maxxfer; 4489 4490 /* 4491 * set the adjusted cookie size now in case this is the first 4492 * window. All other windows are taken care of in get win 4493 */ 4494 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4495 } 4496 4497 /* 4498 * coffset is the current offset within the cookie, new_offset is the 4499 * current offset with the entire buffer. 4500 */ 4501 coffset = dmac_size - trim_sz; 4502 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4503 4504 /* initialize the next window */ 4505 (*windowp)++; 4506 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4507 (*windowp)->wd_cookie_cnt++; 4508 (*windowp)->wd_size = trim_sz; 4509 if (trim_sz < dmac_size) { 4510 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4511 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4512 coffset; 4513 (*windowp)->wd_trim.tr_first_size = trim_sz; 4514 } 4515 4516 return (DDI_SUCCESS); 4517 } 4518 4519 4520 /*ARGSUSED*/ 4521 static int 4522 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4523 off_t off, size_t len, uint_t cache_flags) 4524 { 4525 rootnex_sglinfo_t *sinfo; 4526 rootnex_pgmap_t *cbpage; 4527 rootnex_window_t *win; 4528 ddi_dma_impl_t *hp; 4529 rootnex_dma_t *dma; 4530 caddr_t fromaddr; 4531 caddr_t toaddr; 4532 uint_t psize; 4533 off_t offset; 4534 uint_t pidx; 4535 size_t size; 4536 off_t poff; 4537 int e; 4538 4539 4540 hp = (ddi_dma_impl_t *)handle; 4541 dma = (rootnex_dma_t *)hp->dmai_private; 4542 sinfo = &dma->dp_sglinfo; 4543 4544 /* 4545 * if we don't have any windows, we don't need to sync. A copybuf 4546 * will cause us to have at least one window. 4547 */ 4548 if (dma->dp_window == NULL) { 4549 return (DDI_SUCCESS); 4550 } 4551 4552 /* This window may not need to be sync'd */ 4553 win = &dma->dp_window[dma->dp_current_win]; 4554 if (!win->wd_dosync) { 4555 return (DDI_SUCCESS); 4556 } 4557 4558 /* handle off and len special cases */ 4559 if ((off == 0) || (rootnex_sync_ignore_params)) { 4560 offset = win->wd_offset; 4561 } else { 4562 offset = off; 4563 } 4564 if ((len == 0) || (rootnex_sync_ignore_params)) { 4565 size = win->wd_size; 4566 } else { 4567 size = len; 4568 } 4569 4570 /* check the sync args to make sure they make a little sense */ 4571 if (rootnex_sync_check_parms) { 4572 e = rootnex_valid_sync_parms(hp, win, offset, size, 4573 cache_flags); 4574 if (e != DDI_SUCCESS) { 4575 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4576 return (DDI_FAILURE); 4577 } 4578 } 4579 4580 /* 4581 * special case the first page to handle the offset into the page. The 4582 * offset to the current page for our buffer is the offset into the 4583 * first page of the buffer plus our current offset into the buffer 4584 * itself, masked of course. 4585 */ 4586 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4587 psize = MIN((MMU_PAGESIZE - poff), size); 4588 4589 /* go through all the pages that we want to sync */ 4590 while (size > 0) { 4591 /* 4592 * Calculate the page index relative to the start of the buffer. 4593 * The index to the current page for our buffer is the offset 4594 * into the first page of the buffer plus our current offset 4595 * into the buffer itself, shifted of course... 4596 */ 4597 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4598 ASSERT(pidx < sinfo->si_max_pages); 4599 4600 /* 4601 * if this page uses the copy buffer, we need to sync it, 4602 * otherwise, go on to the next page. 4603 */ 4604 cbpage = &dma->dp_pgmap[pidx]; 4605 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4606 (cbpage->pm_uses_copybuf == B_FALSE)); 4607 if (cbpage->pm_uses_copybuf) { 4608 /* cbaddr and kaddr should be page aligned */ 4609 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4610 MMU_PAGEOFFSET) == 0); 4611 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4612 MMU_PAGEOFFSET) == 0); 4613 4614 /* 4615 * if we're copying for the device, we are going to 4616 * copy from the drivers buffer and to the rootnex 4617 * allocated copy buffer. 4618 */ 4619 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4620 fromaddr = cbpage->pm_kaddr + poff; 4621 toaddr = cbpage->pm_cbaddr + poff; 4622 ROOTNEX_DPROBE2(rootnex__sync__dev, 4623 dev_info_t *, dma->dp_dip, size_t, psize); 4624 4625 /* 4626 * if we're copying for the cpu/kernel, we are going to 4627 * copy from the rootnex allocated copy buffer to the 4628 * drivers buffer. 4629 */ 4630 } else { 4631 fromaddr = cbpage->pm_cbaddr + poff; 4632 toaddr = cbpage->pm_kaddr + poff; 4633 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4634 dev_info_t *, dma->dp_dip, size_t, psize); 4635 } 4636 4637 bcopy(fromaddr, toaddr, psize); 4638 } 4639 4640 /* 4641 * decrement size until we're done, update our offset into the 4642 * buffer, and get the next page size. 4643 */ 4644 size -= psize; 4645 offset += psize; 4646 psize = MIN(MMU_PAGESIZE, size); 4647 4648 /* page offset is zero for the rest of this loop */ 4649 poff = 0; 4650 } 4651 4652 return (DDI_SUCCESS); 4653 } 4654 4655 /* 4656 * rootnex_dma_sync() 4657 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4658 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4659 * is set, ddi_dma_sync() returns immediately passing back success. 4660 */ 4661 /*ARGSUSED*/ 4662 static int 4663 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4664 off_t off, size_t len, uint_t cache_flags) 4665 { 4666 #if defined(__amd64) && !defined(__xpv) 4667 if (IOMMU_USED(rdip)) { 4668 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4669 cache_flags)); 4670 } 4671 #endif 4672 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4673 cache_flags)); 4674 } 4675 4676 /* 4677 * rootnex_valid_sync_parms() 4678 * checks the parameters passed to sync to verify they are correct. 4679 */ 4680 static int 4681 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4682 off_t offset, size_t size, uint_t cache_flags) 4683 { 4684 off_t woffset; 4685 4686 4687 /* 4688 * the first part of the test to make sure the offset passed in is 4689 * within the window. 4690 */ 4691 if (offset < win->wd_offset) { 4692 return (DDI_FAILURE); 4693 } 4694 4695 /* 4696 * second and last part of the test to make sure the offset and length 4697 * passed in is within the window. 4698 */ 4699 woffset = offset - win->wd_offset; 4700 if ((woffset + size) > win->wd_size) { 4701 return (DDI_FAILURE); 4702 } 4703 4704 /* 4705 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4706 * be set too. 4707 */ 4708 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4709 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4710 return (DDI_SUCCESS); 4711 } 4712 4713 /* 4714 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4715 * should be set. Also DDI_DMA_READ should be set in the flags. 4716 */ 4717 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4718 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4719 (hp->dmai_rflags & DDI_DMA_READ)) { 4720 return (DDI_SUCCESS); 4721 } 4722 4723 return (DDI_FAILURE); 4724 } 4725 4726 4727 /*ARGSUSED*/ 4728 static int 4729 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4730 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4731 uint_t *ccountp) 4732 { 4733 rootnex_window_t *window; 4734 rootnex_trim_t *trim; 4735 ddi_dma_impl_t *hp; 4736 rootnex_dma_t *dma; 4737 ddi_dma_obj_t *dmao; 4738 #if !defined(__amd64) 4739 rootnex_sglinfo_t *sinfo; 4740 rootnex_pgmap_t *pmap; 4741 uint_t pidx; 4742 uint_t pcnt; 4743 off_t poff; 4744 int i; 4745 #endif 4746 4747 4748 hp = (ddi_dma_impl_t *)handle; 4749 dma = (rootnex_dma_t *)hp->dmai_private; 4750 #if !defined(__amd64) 4751 sinfo = &dma->dp_sglinfo; 4752 #endif 4753 4754 /* If we try and get a window which doesn't exist, return failure */ 4755 if (win >= hp->dmai_nwin) { 4756 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4757 return (DDI_FAILURE); 4758 } 4759 4760 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4761 4762 /* 4763 * if we don't have any windows, and they're asking for the first 4764 * window, setup the cookie pointer to the first cookie in the bind. 4765 * setup our return values, then increment the cookie since we return 4766 * the first cookie on the stack. 4767 */ 4768 if (dma->dp_window == NULL) { 4769 if (win != 0) { 4770 ROOTNEX_DPROF_INC( 4771 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4772 return (DDI_FAILURE); 4773 } 4774 hp->dmai_cookie = dma->dp_cookies; 4775 *offp = 0; 4776 *lenp = dmao->dmao_size; 4777 *ccountp = dma->dp_sglinfo.si_sgl_size; 4778 *cookiep = hp->dmai_cookie[0]; 4779 hp->dmai_cookie++; 4780 return (DDI_SUCCESS); 4781 } 4782 4783 /* sync the old window before moving on to the new one */ 4784 window = &dma->dp_window[dma->dp_current_win]; 4785 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4786 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4787 DDI_DMA_SYNC_FORCPU); 4788 } 4789 4790 #if !defined(__amd64) 4791 /* 4792 * before we move to the next window, if we need to re-map, unmap all 4793 * the pages in this window. 4794 */ 4795 if (dma->dp_cb_remaping) { 4796 /* 4797 * If we switch to this window again, we'll need to map in 4798 * on the fly next time. 4799 */ 4800 window->wd_remap_copybuf = B_TRUE; 4801 4802 /* 4803 * calculate the page index into the buffer where this window 4804 * starts, and the number of pages this window takes up. 4805 */ 4806 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4807 MMU_PAGESHIFT; 4808 poff = (sinfo->si_buf_offset + window->wd_offset) & 4809 MMU_PAGEOFFSET; 4810 pcnt = mmu_btopr(window->wd_size + poff); 4811 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4812 4813 /* unmap pages which are currently mapped in this window */ 4814 for (i = 0; i < pcnt; i++) { 4815 if (dma->dp_pgmap[pidx].pm_mapped) { 4816 hat_unload(kas.a_hat, 4817 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4818 HAT_UNLOAD); 4819 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4820 } 4821 pidx++; 4822 } 4823 } 4824 #endif 4825 4826 /* 4827 * Move to the new window. 4828 * NOTE: current_win must be set for sync to work right 4829 */ 4830 dma->dp_current_win = win; 4831 window = &dma->dp_window[win]; 4832 4833 /* if needed, adjust the first and/or last cookies for trim */ 4834 trim = &window->wd_trim; 4835 if (trim->tr_trim_first) { 4836 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4837 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4838 #if !defined(__amd64) 4839 window->wd_first_cookie->dmac_type = 4840 (window->wd_first_cookie->dmac_type & 4841 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4842 #endif 4843 if (trim->tr_first_copybuf_win) { 4844 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4845 trim->tr_first_cbaddr; 4846 #if !defined(__amd64) 4847 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4848 trim->tr_first_kaddr; 4849 #endif 4850 } 4851 } 4852 if (trim->tr_trim_last) { 4853 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4854 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4855 if (trim->tr_last_copybuf_win) { 4856 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4857 trim->tr_last_cbaddr; 4858 #if !defined(__amd64) 4859 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4860 trim->tr_last_kaddr; 4861 #endif 4862 } 4863 } 4864 4865 /* 4866 * setup the cookie pointer to the first cookie in the window. setup 4867 * our return values, then increment the cookie since we return the 4868 * first cookie on the stack. 4869 */ 4870 hp->dmai_cookie = window->wd_first_cookie; 4871 *offp = window->wd_offset; 4872 *lenp = window->wd_size; 4873 *ccountp = window->wd_cookie_cnt; 4874 *cookiep = hp->dmai_cookie[0]; 4875 hp->dmai_cookie++; 4876 4877 #if !defined(__amd64) 4878 /* re-map copybuf if required for this window */ 4879 if (dma->dp_cb_remaping) { 4880 /* 4881 * calculate the page index into the buffer where this 4882 * window starts. 4883 */ 4884 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4885 MMU_PAGESHIFT; 4886 ASSERT(pidx < sinfo->si_max_pages); 4887 4888 /* 4889 * the first page can get unmapped if it's shared with the 4890 * previous window. Even if the rest of this window is already 4891 * mapped in, we need to still check this one. 4892 */ 4893 pmap = &dma->dp_pgmap[pidx]; 4894 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4895 if (pmap->pm_pp != NULL) { 4896 pmap->pm_mapped = B_TRUE; 4897 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4898 } else if (pmap->pm_vaddr != NULL) { 4899 pmap->pm_mapped = B_TRUE; 4900 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4901 pmap->pm_kaddr); 4902 } 4903 } 4904 pidx++; 4905 4906 /* map in the rest of the pages if required */ 4907 if (window->wd_remap_copybuf) { 4908 window->wd_remap_copybuf = B_FALSE; 4909 4910 /* figure out many pages this window takes up */ 4911 poff = (sinfo->si_buf_offset + window->wd_offset) & 4912 MMU_PAGEOFFSET; 4913 pcnt = mmu_btopr(window->wd_size + poff); 4914 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4915 4916 /* map pages which require it */ 4917 for (i = 1; i < pcnt; i++) { 4918 pmap = &dma->dp_pgmap[pidx]; 4919 if (pmap->pm_uses_copybuf) { 4920 ASSERT(pmap->pm_mapped == B_FALSE); 4921 if (pmap->pm_pp != NULL) { 4922 pmap->pm_mapped = B_TRUE; 4923 i86_pp_map(pmap->pm_pp, 4924 pmap->pm_kaddr); 4925 } else if (pmap->pm_vaddr != NULL) { 4926 pmap->pm_mapped = B_TRUE; 4927 i86_va_map(pmap->pm_vaddr, 4928 sinfo->si_asp, 4929 pmap->pm_kaddr); 4930 } 4931 } 4932 pidx++; 4933 } 4934 } 4935 } 4936 #endif 4937 4938 /* if the new window uses the copy buffer, sync it for the device */ 4939 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4940 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4941 DDI_DMA_SYNC_FORDEV); 4942 } 4943 4944 return (DDI_SUCCESS); 4945 } 4946 4947 /* 4948 * rootnex_dma_win() 4949 * called from ddi_dma_getwin() 4950 */ 4951 /*ARGSUSED*/ 4952 static int 4953 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4954 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4955 uint_t *ccountp) 4956 { 4957 #if defined(__amd64) && !defined(__xpv) 4958 if (IOMMU_USED(rdip)) { 4959 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4960 cookiep, ccountp)); 4961 } 4962 #endif 4963 4964 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4965 cookiep, ccountp)); 4966 } 4967 4968 #if defined(__amd64) && !defined(__xpv) 4969 /*ARGSUSED*/ 4970 static int 4971 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4972 ddi_dma_handle_t handle, void *v) 4973 { 4974 ddi_dma_impl_t *hp; 4975 rootnex_dma_t *dma; 4976 4977 hp = (ddi_dma_impl_t *)handle; 4978 dma = (rootnex_dma_t *)hp->dmai_private; 4979 dma->dp_iommu_private = v; 4980 4981 return (DDI_SUCCESS); 4982 } 4983 4984 /*ARGSUSED*/ 4985 static void * 4986 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4987 ddi_dma_handle_t handle) 4988 { 4989 ddi_dma_impl_t *hp; 4990 rootnex_dma_t *dma; 4991 4992 hp = (ddi_dma_impl_t *)handle; 4993 dma = (rootnex_dma_t *)hp->dmai_private; 4994 4995 return (dma->dp_iommu_private); 4996 } 4997 #endif 4998 4999 /* 5000 * ************************ 5001 * obsoleted dma routines 5002 * ************************ 5003 */ 5004 5005 /* 5006 * rootnex_dma_map() 5007 * called from ddi_dma_setup() 5008 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 5009 */ 5010 /* ARGSUSED */ 5011 static int 5012 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 5013 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 5014 { 5015 #if defined(__amd64) 5016 /* 5017 * this interface is not supported in 64-bit x86 kernel. See comment in 5018 * rootnex_dma_mctl() 5019 */ 5020 return (DDI_DMA_NORESOURCES); 5021 5022 #else /* 32-bit x86 kernel */ 5023 ddi_dma_handle_t *lhandlep; 5024 ddi_dma_handle_t lhandle; 5025 ddi_dma_cookie_t cookie; 5026 ddi_dma_attr_t dma_attr; 5027 ddi_dma_lim_t *dma_lim; 5028 uint_t ccnt; 5029 int e; 5030 5031 5032 /* 5033 * if the driver is just testing to see if it's possible to do the bind, 5034 * we'll use local state. Otherwise, use the handle pointer passed in. 5035 */ 5036 if (handlep == NULL) { 5037 lhandlep = &lhandle; 5038 } else { 5039 lhandlep = handlep; 5040 } 5041 5042 /* convert the limit structure to a dma_attr one */ 5043 dma_lim = dmareq->dmar_limits; 5044 dma_attr.dma_attr_version = DMA_ATTR_V0; 5045 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 5046 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 5047 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 5048 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 5049 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 5050 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 5051 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 5052 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 5053 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 5054 dma_attr.dma_attr_align = MMU_PAGESIZE; 5055 dma_attr.dma_attr_flags = 0; 5056 5057 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 5058 dmareq->dmar_arg, lhandlep); 5059 if (e != DDI_SUCCESS) { 5060 return (e); 5061 } 5062 5063 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 5064 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 5065 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5066 return (e); 5067 } 5068 5069 /* 5070 * if the driver is just testing to see if it's possible to do the bind, 5071 * free up the local state and return the result. 5072 */ 5073 if (handlep == NULL) { 5074 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 5075 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5076 if (e == DDI_DMA_MAPPED) { 5077 return (DDI_DMA_MAPOK); 5078 } else { 5079 return (DDI_DMA_NOMAPPING); 5080 } 5081 } 5082 5083 return (e); 5084 #endif /* defined(__amd64) */ 5085 } 5086 5087 /* 5088 * rootnex_dma_mctl() 5089 * 5090 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 5091 */ 5092 /* ARGSUSED */ 5093 static int 5094 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5095 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5096 uint_t cache_flags) 5097 { 5098 #if defined(__amd64) 5099 /* 5100 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 5101 * common implementation in genunix, so they no longer have x86 5102 * specific functionality which called into dma_ctl. 5103 * 5104 * The rest of the obsoleted interfaces were never supported in the 5105 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 5106 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 5107 * implementation issues. 5108 * 5109 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 5110 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 5111 * reflect that now too... 5112 * 5113 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 5114 * not going to put this functionality into the 64-bit x86 kernel now. 5115 * It wasn't ported to the 64-bit kernel for s10, no reason to change 5116 * that in a future release. 5117 */ 5118 return (DDI_FAILURE); 5119 5120 #else /* 32-bit x86 kernel */ 5121 ddi_dma_cookie_t lcookie; 5122 ddi_dma_cookie_t *cookie; 5123 rootnex_window_t *window; 5124 ddi_dma_impl_t *hp; 5125 rootnex_dma_t *dma; 5126 uint_t nwin; 5127 uint_t ccnt; 5128 size_t len; 5129 off_t off; 5130 int e; 5131 5132 5133 /* 5134 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 5135 * hacky since were optimizing for the current interfaces and so we can 5136 * cleanup the mess in genunix. Hopefully we will remove the this 5137 * obsoleted routines someday soon. 5138 */ 5139 5140 switch (request) { 5141 5142 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 5143 hp = (ddi_dma_impl_t *)handle; 5144 cookie = (ddi_dma_cookie_t *)objpp; 5145 5146 /* 5147 * convert segment to cookie. We don't distinguish between the 5148 * two :-) 5149 */ 5150 *cookie = *hp->dmai_cookie; 5151 *lenp = cookie->dmac_size; 5152 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 5153 return (DDI_SUCCESS); 5154 5155 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 5156 hp = (ddi_dma_impl_t *)handle; 5157 dma = (rootnex_dma_t *)hp->dmai_private; 5158 5159 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 5160 return (DDI_DMA_STALE); 5161 } 5162 5163 /* handle the case where we don't have any windows */ 5164 if (dma->dp_window == NULL) { 5165 /* 5166 * if seg == NULL, and we don't have any windows, 5167 * return the first cookie in the sgl. 5168 */ 5169 if (*lenp == NULL) { 5170 dma->dp_current_cookie = 0; 5171 hp->dmai_cookie = dma->dp_cookies; 5172 *objpp = (caddr_t)handle; 5173 return (DDI_SUCCESS); 5174 5175 /* if we have more cookies, go to the next cookie */ 5176 } else { 5177 if ((dma->dp_current_cookie + 1) >= 5178 dma->dp_sglinfo.si_sgl_size) { 5179 return (DDI_DMA_DONE); 5180 } 5181 dma->dp_current_cookie++; 5182 hp->dmai_cookie++; 5183 return (DDI_SUCCESS); 5184 } 5185 } 5186 5187 /* We have one or more windows */ 5188 window = &dma->dp_window[dma->dp_current_win]; 5189 5190 /* 5191 * if seg == NULL, return the first cookie in the current 5192 * window 5193 */ 5194 if (*lenp == NULL) { 5195 dma->dp_current_cookie = 0; 5196 hp->dmai_cookie = window->wd_first_cookie; 5197 5198 /* 5199 * go to the next cookie in the window then see if we done with 5200 * this window. 5201 */ 5202 } else { 5203 if ((dma->dp_current_cookie + 1) >= 5204 window->wd_cookie_cnt) { 5205 return (DDI_DMA_DONE); 5206 } 5207 dma->dp_current_cookie++; 5208 hp->dmai_cookie++; 5209 } 5210 *objpp = (caddr_t)handle; 5211 return (DDI_SUCCESS); 5212 5213 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 5214 hp = (ddi_dma_impl_t *)handle; 5215 dma = (rootnex_dma_t *)hp->dmai_private; 5216 5217 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 5218 return (DDI_DMA_STALE); 5219 } 5220 5221 /* if win == NULL, return the first window in the bind */ 5222 if (*offp == NULL) { 5223 nwin = 0; 5224 5225 /* 5226 * else, go to the next window then see if we're done with all 5227 * the windows. 5228 */ 5229 } else { 5230 nwin = dma->dp_current_win + 1; 5231 if (nwin >= hp->dmai_nwin) { 5232 return (DDI_DMA_DONE); 5233 } 5234 } 5235 5236 /* switch to the next window */ 5237 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 5238 &lcookie, &ccnt); 5239 ASSERT(e == DDI_SUCCESS); 5240 if (e != DDI_SUCCESS) { 5241 return (DDI_DMA_STALE); 5242 } 5243 5244 /* reset the cookie back to the first cookie in the window */ 5245 if (dma->dp_window != NULL) { 5246 window = &dma->dp_window[dma->dp_current_win]; 5247 hp->dmai_cookie = window->wd_first_cookie; 5248 } else { 5249 hp->dmai_cookie = dma->dp_cookies; 5250 } 5251 5252 *objpp = (caddr_t)handle; 5253 return (DDI_SUCCESS); 5254 5255 case DDI_DMA_FREE: /* ddi_dma_free() */ 5256 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 5257 (void) rootnex_dma_freehdl(dip, rdip, handle); 5258 if (rootnex_state->r_dvma_call_list_id) { 5259 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 5260 } 5261 return (DDI_SUCCESS); 5262 5263 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 5264 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 5265 /* should never get here, handled in genunix */ 5266 ASSERT(0); 5267 return (DDI_FAILURE); 5268 5269 case DDI_DMA_KVADDR: 5270 case DDI_DMA_GETERR: 5271 case DDI_DMA_COFF: 5272 return (DDI_FAILURE); 5273 } 5274 5275 return (DDI_FAILURE); 5276 #endif /* defined(__amd64) */ 5277 } 5278 5279 /* 5280 * ********* 5281 * FMA Code 5282 * ********* 5283 */ 5284 5285 /* 5286 * rootnex_fm_init() 5287 * FMA init busop 5288 */ 5289 /* ARGSUSED */ 5290 static int 5291 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5292 ddi_iblock_cookie_t *ibc) 5293 { 5294 *ibc = rootnex_state->r_err_ibc; 5295 5296 return (ddi_system_fmcap); 5297 } 5298 5299 /* 5300 * rootnex_dma_check() 5301 * Function called after a dma fault occurred to find out whether the 5302 * fault address is associated with a driver that is able to handle faults 5303 * and recover from faults. 5304 */ 5305 /* ARGSUSED */ 5306 static int 5307 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5308 const void *not_used) 5309 { 5310 rootnex_window_t *window; 5311 uint64_t start_addr; 5312 uint64_t fault_addr; 5313 ddi_dma_impl_t *hp; 5314 rootnex_dma_t *dma; 5315 uint64_t end_addr; 5316 size_t csize; 5317 int i; 5318 int j; 5319 5320 5321 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5322 hp = (ddi_dma_impl_t *)handle; 5323 ASSERT(hp); 5324 5325 dma = (rootnex_dma_t *)hp->dmai_private; 5326 5327 /* Get the address that we need to search for */ 5328 fault_addr = *(uint64_t *)addr; 5329 5330 /* 5331 * if we don't have any windows, we can just walk through all the 5332 * cookies. 5333 */ 5334 if (dma->dp_window == NULL) { 5335 /* for each cookie */ 5336 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5337 /* 5338 * if the faulted address is within the physical address 5339 * range of the cookie, return DDI_FM_NONFATAL. 5340 */ 5341 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5342 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5343 dma->dp_cookies[i].dmac_size))) { 5344 return (DDI_FM_NONFATAL); 5345 } 5346 } 5347 5348 /* fault_addr not within this DMA handle */ 5349 return (DDI_FM_UNKNOWN); 5350 } 5351 5352 /* we have mutiple windows, walk through each window */ 5353 for (i = 0; i < hp->dmai_nwin; i++) { 5354 window = &dma->dp_window[i]; 5355 5356 /* Go through all the cookies in the window */ 5357 for (j = 0; j < window->wd_cookie_cnt; j++) { 5358 5359 start_addr = window->wd_first_cookie[j].dmac_laddress; 5360 csize = window->wd_first_cookie[j].dmac_size; 5361 5362 /* 5363 * if we are trimming the first cookie in the window, 5364 * and this is the first cookie, adjust the start 5365 * address and size of the cookie to account for the 5366 * trim. 5367 */ 5368 if (window->wd_trim.tr_trim_first && (j == 0)) { 5369 start_addr = window->wd_trim.tr_first_paddr; 5370 csize = window->wd_trim.tr_first_size; 5371 } 5372 5373 /* 5374 * if we are trimming the last cookie in the window, 5375 * and this is the last cookie, adjust the start 5376 * address and size of the cookie to account for the 5377 * trim. 5378 */ 5379 if (window->wd_trim.tr_trim_last && 5380 (j == (window->wd_cookie_cnt - 1))) { 5381 start_addr = window->wd_trim.tr_last_paddr; 5382 csize = window->wd_trim.tr_last_size; 5383 } 5384 5385 end_addr = start_addr + csize; 5386 5387 /* 5388 * if the faulted address is within the physical 5389 * address of the cookie, return DDI_FM_NONFATAL. 5390 */ 5391 if ((fault_addr >= start_addr) && 5392 (fault_addr <= end_addr)) { 5393 return (DDI_FM_NONFATAL); 5394 } 5395 } 5396 } 5397 5398 /* fault_addr not within this DMA handle */ 5399 return (DDI_FM_UNKNOWN); 5400 } 5401 5402 /*ARGSUSED*/ 5403 static int 5404 rootnex_quiesce(dev_info_t *dip) 5405 { 5406 #if defined(__amd64) && !defined(__xpv) 5407 return (immu_quiesce()); 5408 #else 5409 return (DDI_SUCCESS); 5410 #endif 5411 } 5412 5413 #if defined(__xpv) 5414 void 5415 immu_init(void) 5416 { 5417 ; 5418 } 5419 5420 void 5421 immu_startup(void) 5422 { 5423 ; 5424 } 5425 /*ARGSUSED*/ 5426 void 5427 immu_physmem_update(uint64_t addr, uint64_t size) 5428 { 5429 ; 5430 } 5431 #endif 5432