1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 */ 27 28 /* 29 * x86 root nexus driver 30 */ 31 32 #include <sys/sysmacros.h> 33 #include <sys/conf.h> 34 #include <sys/autoconf.h> 35 #include <sys/sysmacros.h> 36 #include <sys/debug.h> 37 #include <sys/psw.h> 38 #include <sys/ddidmareq.h> 39 #include <sys/promif.h> 40 #include <sys/devops.h> 41 #include <sys/kmem.h> 42 #include <sys/cmn_err.h> 43 #include <vm/seg.h> 44 #include <vm/seg_kmem.h> 45 #include <vm/seg_dev.h> 46 #include <sys/vmem.h> 47 #include <sys/mman.h> 48 #include <vm/hat.h> 49 #include <vm/as.h> 50 #include <vm/page.h> 51 #include <sys/avintr.h> 52 #include <sys/errno.h> 53 #include <sys/modctl.h> 54 #include <sys/ddi_impldefs.h> 55 #include <sys/sunddi.h> 56 #include <sys/sunndi.h> 57 #include <sys/mach_intr.h> 58 #include <sys/psm.h> 59 #include <sys/ontrap.h> 60 #include <sys/atomic.h> 61 #include <sys/sdt.h> 62 #include <sys/rootnex.h> 63 #include <vm/hat_i86.h> 64 #include <sys/ddifm.h> 65 #include <sys/ddi_isa.h> 66 #include <sys/apic.h> 67 68 #ifdef __xpv 69 #include <sys/bootinfo.h> 70 #include <sys/hypervisor.h> 71 #include <sys/bootconf.h> 72 #include <vm/kboot_mmu.h> 73 #endif 74 75 #if defined(__amd64) && !defined(__xpv) 76 #include <sys/immu.h> 77 #endif 78 79 80 /* 81 * enable/disable extra checking of function parameters. Useful for debugging 82 * drivers. 83 */ 84 #ifdef DEBUG 85 int rootnex_alloc_check_parms = 1; 86 int rootnex_bind_check_parms = 1; 87 int rootnex_bind_check_inuse = 1; 88 int rootnex_unbind_verify_buffer = 0; 89 int rootnex_sync_check_parms = 1; 90 #else 91 int rootnex_alloc_check_parms = 0; 92 int rootnex_bind_check_parms = 0; 93 int rootnex_bind_check_inuse = 0; 94 int rootnex_unbind_verify_buffer = 0; 95 int rootnex_sync_check_parms = 0; 96 #endif 97 98 boolean_t rootnex_dmar_not_setup; 99 100 /* Master Abort and Target Abort panic flag */ 101 int rootnex_fm_ma_ta_panic_flag = 0; 102 103 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 104 int rootnex_bind_fail = 1; 105 int rootnex_bind_warn = 1; 106 uint8_t *rootnex_warn_list; 107 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 108 #define ROOTNEX_BIND_WARNING (0x1 << 0) 109 110 /* 111 * revert back to old broken behavior of always sync'ing entire copy buffer. 112 * This is useful if be have a buggy driver which doesn't correctly pass in 113 * the offset and size into ddi_dma_sync(). 114 */ 115 int rootnex_sync_ignore_params = 0; 116 117 /* 118 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 119 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 120 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 121 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 122 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 123 * (< 8K). We will still need to allocate the copy buffer during bind though 124 * (if we need one). These can only be modified in /etc/system before rootnex 125 * attach. 126 */ 127 #if defined(__amd64) 128 int rootnex_prealloc_cookies = 65; 129 int rootnex_prealloc_windows = 4; 130 int rootnex_prealloc_copybuf = 2; 131 #else 132 int rootnex_prealloc_cookies = 33; 133 int rootnex_prealloc_windows = 4; 134 int rootnex_prealloc_copybuf = 2; 135 #endif 136 137 /* driver global state */ 138 static rootnex_state_t *rootnex_state; 139 140 #ifdef DEBUG 141 /* shortcut to rootnex counters */ 142 static uint64_t *rootnex_cnt; 143 #endif 144 145 /* 146 * XXX - does x86 even need these or are they left over from the SPARC days? 147 */ 148 /* statically defined integer/boolean properties for the root node */ 149 static rootnex_intprop_t rootnex_intprp[] = { 150 { "PAGESIZE", PAGESIZE }, 151 { "MMU_PAGESIZE", MMU_PAGESIZE }, 152 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 153 { DDI_RELATIVE_ADDRESSING, 1 }, 154 }; 155 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 156 157 /* 158 * If we're dom0, we're using a real device so we need to load 159 * the cookies with MFNs instead of PFNs. 160 */ 161 #ifdef __xpv 162 typedef maddr_t rootnex_addr_t; 163 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 164 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 165 #else 166 typedef paddr_t rootnex_addr_t; 167 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 168 #endif 169 170 #if !defined(__xpv) 171 char _depends_on[] = "misc/iommulib misc/acpica"; 172 #endif 173 174 static struct cb_ops rootnex_cb_ops = { 175 nodev, /* open */ 176 nodev, /* close */ 177 nodev, /* strategy */ 178 nodev, /* print */ 179 nodev, /* dump */ 180 nodev, /* read */ 181 nodev, /* write */ 182 nodev, /* ioctl */ 183 nodev, /* devmap */ 184 nodev, /* mmap */ 185 nodev, /* segmap */ 186 nochpoll, /* chpoll */ 187 ddi_prop_op, /* cb_prop_op */ 188 NULL, /* struct streamtab */ 189 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 190 CB_REV, /* Rev */ 191 nodev, /* cb_aread */ 192 nodev /* cb_awrite */ 193 }; 194 195 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 196 off_t offset, off_t len, caddr_t *vaddrp); 197 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 198 struct hat *hat, struct seg *seg, caddr_t addr, 199 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 200 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 201 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 202 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 204 ddi_dma_handle_t *handlep); 205 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 206 ddi_dma_handle_t handle); 207 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 208 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 209 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 210 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 211 ddi_dma_handle_t handle); 212 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 213 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 214 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 215 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 216 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 217 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 218 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 219 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 220 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 221 ddi_ctl_enum_t ctlop, void *arg, void *result); 222 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 223 ddi_iblock_cookie_t *ibc); 224 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 225 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 226 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 227 void *); 228 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 229 230 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 231 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 232 ddi_dma_handle_t *handlep); 233 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 234 ddi_dma_handle_t handle); 235 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 236 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 237 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 238 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 239 ddi_dma_handle_t handle); 240 #if defined(__amd64) && !defined(__xpv) 241 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 242 ddi_dma_handle_t handle); 243 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 244 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 245 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 246 ddi_dma_cookie_t *cookiep, uint_t ccount); 247 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 248 ddi_dma_handle_t handle); 249 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 250 #endif 251 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 252 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 253 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 254 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 255 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 256 257 #if defined(__amd64) && !defined(__xpv) 258 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 259 ddi_dma_handle_t handle, void *v); 260 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 261 ddi_dma_handle_t handle); 262 #endif 263 264 265 static struct bus_ops rootnex_bus_ops = { 266 BUSO_REV, 267 rootnex_map, 268 NULL, 269 NULL, 270 NULL, 271 rootnex_map_fault, 272 rootnex_dma_map, 273 rootnex_dma_allochdl, 274 rootnex_dma_freehdl, 275 rootnex_dma_bindhdl, 276 rootnex_dma_unbindhdl, 277 rootnex_dma_sync, 278 rootnex_dma_win, 279 rootnex_dma_mctl, 280 rootnex_ctlops, 281 ddi_bus_prop_op, 282 i_ddi_rootnex_get_eventcookie, 283 i_ddi_rootnex_add_eventcall, 284 i_ddi_rootnex_remove_eventcall, 285 i_ddi_rootnex_post_event, 286 0, /* bus_intr_ctl */ 287 0, /* bus_config */ 288 0, /* bus_unconfig */ 289 rootnex_fm_init, /* bus_fm_init */ 290 NULL, /* bus_fm_fini */ 291 NULL, /* bus_fm_access_enter */ 292 NULL, /* bus_fm_access_exit */ 293 NULL, /* bus_powr */ 294 rootnex_intr_ops /* bus_intr_op */ 295 }; 296 297 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 298 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 299 static int rootnex_quiesce(dev_info_t *dip); 300 301 static struct dev_ops rootnex_ops = { 302 DEVO_REV, 303 0, 304 ddi_no_info, 305 nulldev, 306 nulldev, 307 rootnex_attach, 308 rootnex_detach, 309 nulldev, 310 &rootnex_cb_ops, 311 &rootnex_bus_ops, 312 NULL, 313 rootnex_quiesce, /* quiesce */ 314 }; 315 316 static struct modldrv rootnex_modldrv = { 317 &mod_driverops, 318 "i86pc root nexus", 319 &rootnex_ops 320 }; 321 322 static struct modlinkage rootnex_modlinkage = { 323 MODREV_1, 324 (void *)&rootnex_modldrv, 325 NULL 326 }; 327 328 #if defined(__amd64) && !defined(__xpv) 329 static iommulib_nexops_t iommulib_nexops = { 330 IOMMU_NEXOPS_VERSION, 331 "Rootnex IOMMU ops Vers 1.1", 332 NULL, 333 rootnex_coredma_allochdl, 334 rootnex_coredma_freehdl, 335 rootnex_coredma_bindhdl, 336 rootnex_coredma_unbindhdl, 337 rootnex_coredma_reset_cookies, 338 rootnex_coredma_get_cookies, 339 rootnex_coredma_set_cookies, 340 rootnex_coredma_clear_cookies, 341 rootnex_coredma_get_sleep_flags, 342 rootnex_coredma_sync, 343 rootnex_coredma_win, 344 rootnex_dma_map, 345 rootnex_dma_mctl, 346 rootnex_coredma_hdl_setprivate, 347 rootnex_coredma_hdl_getprivate 348 }; 349 #endif 350 351 /* 352 * extern hacks 353 */ 354 extern struct seg_ops segdev_ops; 355 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 356 #ifdef DDI_MAP_DEBUG 357 extern int ddi_map_debug_flag; 358 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 359 #endif 360 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 361 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 362 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 363 psm_intr_op_t, int *); 364 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 365 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 366 367 /* 368 * Use device arena to use for device control register mappings. 369 * Various kernel memory walkers (debugger, dtrace) need to know 370 * to avoid this address range to prevent undesired device activity. 371 */ 372 extern void *device_arena_alloc(size_t size, int vm_flag); 373 extern void device_arena_free(void * vaddr, size_t size); 374 375 376 /* 377 * Internal functions 378 */ 379 static int rootnex_dma_init(); 380 static void rootnex_add_props(dev_info_t *); 381 static int rootnex_ctl_reportdev(dev_info_t *dip); 382 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 383 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 384 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 385 static int rootnex_map_handle(ddi_map_req_t *mp); 386 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 387 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 388 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 389 ddi_dma_attr_t *attr); 390 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 391 rootnex_sglinfo_t *sglinfo); 392 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 393 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 394 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 395 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 396 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 397 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 398 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 399 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 400 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 401 static void rootnex_teardown_windows(rootnex_dma_t *dma); 402 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 403 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 404 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 405 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 406 size_t *copybuf_used, page_t **cur_pp); 407 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 408 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 409 ddi_dma_attr_t *attr, off_t cur_offset); 410 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 411 rootnex_dma_t *dma, rootnex_window_t **windowp, 412 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 413 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 414 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 415 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 416 off_t offset, size_t size, uint_t cache_flags); 417 static int rootnex_verify_buffer(rootnex_dma_t *dma); 418 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 419 const void *comp_addr, const void *not_used); 420 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 421 rootnex_sglinfo_t *sglinfo); 422 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 423 424 /* 425 * _init() 426 * 427 */ 428 int 429 _init(void) 430 { 431 432 rootnex_state = NULL; 433 return (mod_install(&rootnex_modlinkage)); 434 } 435 436 437 /* 438 * _info() 439 * 440 */ 441 int 442 _info(struct modinfo *modinfop) 443 { 444 return (mod_info(&rootnex_modlinkage, modinfop)); 445 } 446 447 448 /* 449 * _fini() 450 * 451 */ 452 int 453 _fini(void) 454 { 455 return (EBUSY); 456 } 457 458 459 /* 460 * rootnex_attach() 461 * 462 */ 463 static int 464 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 465 { 466 int fmcap; 467 int e; 468 469 switch (cmd) { 470 case DDI_ATTACH: 471 break; 472 case DDI_RESUME: 473 #if defined(__amd64) && !defined(__xpv) 474 return (immu_unquiesce()); 475 #else 476 return (DDI_SUCCESS); 477 #endif 478 default: 479 return (DDI_FAILURE); 480 } 481 482 /* 483 * We should only have one instance of rootnex. Save it away since we 484 * don't have an easy way to get it back later. 485 */ 486 ASSERT(rootnex_state == NULL); 487 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 488 489 rootnex_state->r_dip = dip; 490 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 491 rootnex_state->r_reserved_msg_printed = B_FALSE; 492 #ifdef DEBUG 493 rootnex_cnt = &rootnex_state->r_counters[0]; 494 #endif 495 496 /* 497 * Set minimum fm capability level for i86pc platforms and then 498 * initialize error handling. Since we're the rootnex, we don't 499 * care what's returned in the fmcap field. 500 */ 501 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 502 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 503 fmcap = ddi_system_fmcap; 504 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 505 506 /* initialize DMA related state */ 507 e = rootnex_dma_init(); 508 if (e != DDI_SUCCESS) { 509 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 510 return (DDI_FAILURE); 511 } 512 513 /* Add static root node properties */ 514 rootnex_add_props(dip); 515 516 /* since we can't call ddi_report_dev() */ 517 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 518 519 /* Initialize rootnex event handle */ 520 i_ddi_rootnex_init_events(dip); 521 522 #if defined(__amd64) && !defined(__xpv) 523 e = iommulib_nexus_register(dip, &iommulib_nexops, 524 &rootnex_state->r_iommulib_handle); 525 526 ASSERT(e == DDI_SUCCESS); 527 #endif 528 529 return (DDI_SUCCESS); 530 } 531 532 533 /* 534 * rootnex_detach() 535 * 536 */ 537 /*ARGSUSED*/ 538 static int 539 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 540 { 541 switch (cmd) { 542 case DDI_SUSPEND: 543 #if defined(__amd64) && !defined(__xpv) 544 return (immu_quiesce()); 545 #else 546 return (DDI_SUCCESS); 547 #endif 548 default: 549 return (DDI_FAILURE); 550 } 551 /*NOTREACHED*/ 552 553 } 554 555 556 /* 557 * rootnex_dma_init() 558 * 559 */ 560 /*ARGSUSED*/ 561 static int 562 rootnex_dma_init() 563 { 564 size_t bufsize; 565 566 567 /* 568 * size of our cookie/window/copybuf state needed in dma bind that we 569 * pre-alloc in dma_alloc_handle 570 */ 571 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 572 rootnex_state->r_prealloc_size = 573 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 574 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 575 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 576 577 /* 578 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 579 * allocate 16 extra bytes for struct pointer alignment 580 * (p->dmai_private & dma->dp_prealloc_buffer) 581 */ 582 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 583 rootnex_state->r_prealloc_size + 0x10; 584 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 585 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 586 if (rootnex_state->r_dmahdl_cache == NULL) { 587 return (DDI_FAILURE); 588 } 589 590 /* 591 * allocate array to track which major numbers we have printed warnings 592 * for. 593 */ 594 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 595 KM_SLEEP); 596 597 return (DDI_SUCCESS); 598 } 599 600 601 /* 602 * rootnex_add_props() 603 * 604 */ 605 static void 606 rootnex_add_props(dev_info_t *dip) 607 { 608 rootnex_intprop_t *rpp; 609 int i; 610 611 /* Add static integer/boolean properties to the root node */ 612 rpp = rootnex_intprp; 613 for (i = 0; i < NROOT_INTPROPS; i++) { 614 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 615 rpp[i].prop_name, rpp[i].prop_value); 616 } 617 } 618 619 620 621 /* 622 * ************************* 623 * ctlops related routines 624 * ************************* 625 */ 626 627 /* 628 * rootnex_ctlops() 629 * 630 */ 631 /*ARGSUSED*/ 632 static int 633 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 634 void *arg, void *result) 635 { 636 int n, *ptr; 637 struct ddi_parent_private_data *pdp; 638 639 switch (ctlop) { 640 case DDI_CTLOPS_DMAPMAPC: 641 /* 642 * Return 'partial' to indicate that dma mapping 643 * has to be done in the main MMU. 644 */ 645 return (DDI_DMA_PARTIAL); 646 647 case DDI_CTLOPS_BTOP: 648 /* 649 * Convert byte count input to physical page units. 650 * (byte counts that are not a page-size multiple 651 * are rounded down) 652 */ 653 *(ulong_t *)result = btop(*(ulong_t *)arg); 654 return (DDI_SUCCESS); 655 656 case DDI_CTLOPS_PTOB: 657 /* 658 * Convert size in physical pages to bytes 659 */ 660 *(ulong_t *)result = ptob(*(ulong_t *)arg); 661 return (DDI_SUCCESS); 662 663 case DDI_CTLOPS_BTOPR: 664 /* 665 * Convert byte count input to physical page units 666 * (byte counts that are not a page-size multiple 667 * are rounded up) 668 */ 669 *(ulong_t *)result = btopr(*(ulong_t *)arg); 670 return (DDI_SUCCESS); 671 672 case DDI_CTLOPS_INITCHILD: 673 return (impl_ddi_sunbus_initchild(arg)); 674 675 case DDI_CTLOPS_UNINITCHILD: 676 impl_ddi_sunbus_removechild(arg); 677 return (DDI_SUCCESS); 678 679 case DDI_CTLOPS_REPORTDEV: 680 return (rootnex_ctl_reportdev(rdip)); 681 682 case DDI_CTLOPS_IOMIN: 683 /* 684 * Nothing to do here but reflect back.. 685 */ 686 return (DDI_SUCCESS); 687 688 case DDI_CTLOPS_REGSIZE: 689 case DDI_CTLOPS_NREGS: 690 break; 691 692 case DDI_CTLOPS_SIDDEV: 693 if (ndi_dev_is_prom_node(rdip)) 694 return (DDI_SUCCESS); 695 if (ndi_dev_is_persistent_node(rdip)) 696 return (DDI_SUCCESS); 697 return (DDI_FAILURE); 698 699 case DDI_CTLOPS_POWER: 700 return ((*pm_platform_power)((power_req_t *)arg)); 701 702 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 703 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 704 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 705 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 706 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 707 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 708 if (!rootnex_state->r_reserved_msg_printed) { 709 rootnex_state->r_reserved_msg_printed = B_TRUE; 710 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 711 "1 or more reserved/obsolete operations."); 712 } 713 return (DDI_FAILURE); 714 715 default: 716 return (DDI_FAILURE); 717 } 718 /* 719 * The rest are for "hardware" properties 720 */ 721 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 722 return (DDI_FAILURE); 723 724 if (ctlop == DDI_CTLOPS_NREGS) { 725 ptr = (int *)result; 726 *ptr = pdp->par_nreg; 727 } else { 728 off_t *size = (off_t *)result; 729 730 ptr = (int *)arg; 731 n = *ptr; 732 if (n >= pdp->par_nreg) { 733 return (DDI_FAILURE); 734 } 735 *size = (off_t)pdp->par_reg[n].regspec_size; 736 } 737 return (DDI_SUCCESS); 738 } 739 740 741 /* 742 * rootnex_ctl_reportdev() 743 * 744 */ 745 static int 746 rootnex_ctl_reportdev(dev_info_t *dev) 747 { 748 int i, n, len, f_len = 0; 749 char *buf; 750 751 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 752 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 753 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 754 len = strlen(buf); 755 756 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 757 758 struct regspec *rp = sparc_pd_getreg(dev, i); 759 760 if (i == 0) 761 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 762 ": "); 763 else 764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 765 " and "); 766 len = strlen(buf); 767 768 switch (rp->regspec_bustype) { 769 770 case BTEISA: 771 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 772 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 773 break; 774 775 case BTISA: 776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 777 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 778 break; 779 780 default: 781 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 782 "space %x offset %x", 783 rp->regspec_bustype, rp->regspec_addr); 784 break; 785 } 786 len = strlen(buf); 787 } 788 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 789 int pri; 790 791 if (i != 0) { 792 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 793 ","); 794 len = strlen(buf); 795 } 796 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 797 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 798 " sparc ipl %d", pri); 799 len = strlen(buf); 800 } 801 #ifdef DEBUG 802 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 803 cmn_err(CE_NOTE, "next message is truncated: " 804 "printed length 1024, real length %d", f_len); 805 } 806 #endif /* DEBUG */ 807 cmn_err(CE_CONT, "?%s\n", buf); 808 kmem_free(buf, REPORTDEV_BUFSIZE); 809 return (DDI_SUCCESS); 810 } 811 812 813 /* 814 * ****************** 815 * map related code 816 * ****************** 817 */ 818 819 /* 820 * rootnex_map() 821 * 822 */ 823 static int 824 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 825 off_t len, caddr_t *vaddrp) 826 { 827 struct regspec *rp, tmp_reg; 828 ddi_map_req_t mr = *mp; /* Get private copy of request */ 829 int error; 830 831 mp = &mr; 832 833 switch (mp->map_op) { 834 case DDI_MO_MAP_LOCKED: 835 case DDI_MO_UNMAP: 836 case DDI_MO_MAP_HANDLE: 837 break; 838 default: 839 #ifdef DDI_MAP_DEBUG 840 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 841 mp->map_op); 842 #endif /* DDI_MAP_DEBUG */ 843 return (DDI_ME_UNIMPLEMENTED); 844 } 845 846 if (mp->map_flags & DDI_MF_USER_MAPPING) { 847 #ifdef DDI_MAP_DEBUG 848 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 849 #endif /* DDI_MAP_DEBUG */ 850 return (DDI_ME_UNIMPLEMENTED); 851 } 852 853 /* 854 * First, if given an rnumber, convert it to a regspec... 855 * (Presumably, this is on behalf of a child of the root node?) 856 */ 857 858 if (mp->map_type == DDI_MT_RNUMBER) { 859 860 int rnumber = mp->map_obj.rnumber; 861 #ifdef DDI_MAP_DEBUG 862 static char *out_of_range = 863 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 864 #endif /* DDI_MAP_DEBUG */ 865 866 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 867 if (rp == NULL) { 868 #ifdef DDI_MAP_DEBUG 869 cmn_err(CE_WARN, out_of_range, rnumber, 870 ddi_get_name(rdip)); 871 #endif /* DDI_MAP_DEBUG */ 872 return (DDI_ME_RNUMBER_RANGE); 873 } 874 875 /* 876 * Convert the given ddi_map_req_t from rnumber to regspec... 877 */ 878 879 mp->map_type = DDI_MT_REGSPEC; 880 mp->map_obj.rp = rp; 881 } 882 883 /* 884 * Adjust offset and length correspnding to called values... 885 * XXX: A non-zero length means override the one in the regspec 886 * XXX: (regardless of what's in the parent's range?) 887 */ 888 889 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 890 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 891 892 #ifdef DDI_MAP_DEBUG 893 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 894 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 895 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 896 len, mp->map_handlep); 897 #endif /* DDI_MAP_DEBUG */ 898 899 /* 900 * I/O or memory mapping: 901 * 902 * <bustype=0, addr=x, len=x>: memory 903 * <bustype=1, addr=x, len=x>: i/o 904 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 905 */ 906 907 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 908 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 909 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 910 ddi_get_name(rdip), rp->regspec_bustype, 911 rp->regspec_addr, rp->regspec_size); 912 return (DDI_ME_INVAL); 913 } 914 915 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 916 /* 917 * compatibility i/o mapping 918 */ 919 rp->regspec_bustype += (uint_t)offset; 920 } else { 921 /* 922 * Normal memory or i/o mapping 923 */ 924 rp->regspec_addr += (uint_t)offset; 925 } 926 927 if (len != 0) 928 rp->regspec_size = (uint_t)len; 929 930 #ifdef DDI_MAP_DEBUG 931 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 932 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 933 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 934 offset, len, mp->map_handlep); 935 #endif /* DDI_MAP_DEBUG */ 936 937 /* 938 * Apply any parent ranges at this level, if applicable. 939 * (This is where nexus specific regspec translation takes place. 940 * Use of this function is implicit agreement that translation is 941 * provided via ddi_apply_range.) 942 */ 943 944 #ifdef DDI_MAP_DEBUG 945 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 946 ddi_get_name(dip), ddi_get_name(rdip)); 947 #endif /* DDI_MAP_DEBUG */ 948 949 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 950 return (error); 951 952 switch (mp->map_op) { 953 case DDI_MO_MAP_LOCKED: 954 955 /* 956 * Set up the locked down kernel mapping to the regspec... 957 */ 958 959 return (rootnex_map_regspec(mp, vaddrp)); 960 961 case DDI_MO_UNMAP: 962 963 /* 964 * Release mapping... 965 */ 966 967 return (rootnex_unmap_regspec(mp, vaddrp)); 968 969 case DDI_MO_MAP_HANDLE: 970 971 return (rootnex_map_handle(mp)); 972 973 default: 974 return (DDI_ME_UNIMPLEMENTED); 975 } 976 } 977 978 979 /* 980 * rootnex_map_fault() 981 * 982 * fault in mappings for requestors 983 */ 984 /*ARGSUSED*/ 985 static int 986 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 987 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 988 uint_t lock) 989 { 990 991 #ifdef DDI_MAP_DEBUG 992 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 993 ddi_map_debug(" Seg <%s>\n", 994 seg->s_ops == &segdev_ops ? "segdev" : 995 seg == &kvseg ? "segkmem" : "NONE!"); 996 #endif /* DDI_MAP_DEBUG */ 997 998 /* 999 * This is all terribly broken, but it is a start 1000 * 1001 * XXX Note that this test means that segdev_ops 1002 * must be exported from seg_dev.c. 1003 * XXX What about devices with their own segment drivers? 1004 */ 1005 if (seg->s_ops == &segdev_ops) { 1006 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 1007 1008 if (hat == NULL) { 1009 /* 1010 * This is one plausible interpretation of 1011 * a null hat i.e. use the first hat on the 1012 * address space hat list which by convention is 1013 * the hat of the system MMU. At alternative 1014 * would be to panic .. this might well be better .. 1015 */ 1016 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 1017 hat = seg->s_as->a_hat; 1018 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1019 } 1020 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1021 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1022 } else if (seg == &kvseg && dp == NULL) { 1023 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1024 HAT_LOAD_LOCK); 1025 } else 1026 return (DDI_FAILURE); 1027 return (DDI_SUCCESS); 1028 } 1029 1030 1031 /* 1032 * rootnex_map_regspec() 1033 * we don't support mapping of I/O cards above 4Gb 1034 */ 1035 static int 1036 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1037 { 1038 rootnex_addr_t rbase; 1039 void *cvaddr; 1040 uint_t npages, pgoffset; 1041 struct regspec *rp; 1042 ddi_acc_hdl_t *hp; 1043 ddi_acc_impl_t *ap; 1044 uint_t hat_acc_flags; 1045 paddr_t pbase; 1046 1047 rp = mp->map_obj.rp; 1048 hp = mp->map_handlep; 1049 1050 #ifdef DDI_MAP_DEBUG 1051 ddi_map_debug( 1052 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1053 rp->regspec_bustype, rp->regspec_addr, 1054 rp->regspec_size, mp->map_handlep); 1055 #endif /* DDI_MAP_DEBUG */ 1056 1057 /* 1058 * I/O or memory mapping 1059 * 1060 * <bustype=0, addr=x, len=x>: memory 1061 * <bustype=1, addr=x, len=x>: i/o 1062 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1063 */ 1064 1065 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1066 cmn_err(CE_WARN, "rootnex: invalid register spec" 1067 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1068 rp->regspec_addr, rp->regspec_size); 1069 return (DDI_FAILURE); 1070 } 1071 1072 if (rp->regspec_bustype != 0) { 1073 /* 1074 * I/O space - needs a handle. 1075 */ 1076 if (hp == NULL) { 1077 return (DDI_FAILURE); 1078 } 1079 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1080 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1081 impl_acc_hdl_init(hp); 1082 1083 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1084 #ifdef DDI_MAP_DEBUG 1085 ddi_map_debug("rootnex_map_regspec: mmap() " 1086 "to I/O space is not supported.\n"); 1087 #endif /* DDI_MAP_DEBUG */ 1088 return (DDI_ME_INVAL); 1089 } else { 1090 /* 1091 * 1275-compliant vs. compatibility i/o mapping 1092 */ 1093 *vaddrp = 1094 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1095 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1096 ((caddr_t)(uintptr_t)rp->regspec_addr); 1097 #ifdef __xpv 1098 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1099 hp->ah_pfn = xen_assign_pfn( 1100 mmu_btop((ulong_t)rp->regspec_addr & 1101 MMU_PAGEMASK)); 1102 } else { 1103 hp->ah_pfn = mmu_btop( 1104 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1105 } 1106 #else 1107 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1108 MMU_PAGEMASK); 1109 #endif 1110 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1111 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1112 } 1113 1114 #ifdef DDI_MAP_DEBUG 1115 ddi_map_debug( 1116 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1117 rp->regspec_size, *vaddrp); 1118 #endif /* DDI_MAP_DEBUG */ 1119 return (DDI_SUCCESS); 1120 } 1121 1122 /* 1123 * Memory space 1124 */ 1125 1126 if (hp != NULL) { 1127 /* 1128 * hat layer ignores 1129 * hp->ah_acc.devacc_attr_endian_flags. 1130 */ 1131 switch (hp->ah_acc.devacc_attr_dataorder) { 1132 case DDI_STRICTORDER_ACC: 1133 hat_acc_flags = HAT_STRICTORDER; 1134 break; 1135 case DDI_UNORDERED_OK_ACC: 1136 hat_acc_flags = HAT_UNORDERED_OK; 1137 break; 1138 case DDI_MERGING_OK_ACC: 1139 hat_acc_flags = HAT_MERGING_OK; 1140 break; 1141 case DDI_LOADCACHING_OK_ACC: 1142 hat_acc_flags = HAT_LOADCACHING_OK; 1143 break; 1144 case DDI_STORECACHING_OK_ACC: 1145 hat_acc_flags = HAT_STORECACHING_OK; 1146 break; 1147 } 1148 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1149 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1150 impl_acc_hdl_init(hp); 1151 hp->ah_hat_flags = hat_acc_flags; 1152 } else { 1153 hat_acc_flags = HAT_STRICTORDER; 1154 } 1155 1156 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1157 #ifdef __xpv 1158 /* 1159 * If we're dom0, we're using a real device so we need to translate 1160 * the MA to a PA. 1161 */ 1162 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1163 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1164 } else { 1165 pbase = rbase; 1166 } 1167 #else 1168 pbase = rbase; 1169 #endif 1170 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1171 1172 if (rp->regspec_size == 0) { 1173 #ifdef DDI_MAP_DEBUG 1174 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1175 #endif /* DDI_MAP_DEBUG */ 1176 return (DDI_ME_INVAL); 1177 } 1178 1179 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1180 /* extra cast to make gcc happy */ 1181 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1182 } else { 1183 npages = mmu_btopr(rp->regspec_size + pgoffset); 1184 1185 #ifdef DDI_MAP_DEBUG 1186 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1187 "physical %llx", npages, pbase); 1188 #endif /* DDI_MAP_DEBUG */ 1189 1190 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1191 if (cvaddr == NULL) 1192 return (DDI_ME_NORESOURCES); 1193 1194 /* 1195 * Now map in the pages we've allocated... 1196 */ 1197 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1198 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1199 HAT_LOAD_LOCK); 1200 *vaddrp = (caddr_t)cvaddr + pgoffset; 1201 1202 /* save away pfn and npages for FMA */ 1203 hp = mp->map_handlep; 1204 if (hp) { 1205 hp->ah_pfn = mmu_btop(pbase); 1206 hp->ah_pnum = npages; 1207 } 1208 } 1209 1210 #ifdef DDI_MAP_DEBUG 1211 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1212 #endif /* DDI_MAP_DEBUG */ 1213 return (DDI_SUCCESS); 1214 } 1215 1216 1217 /* 1218 * rootnex_unmap_regspec() 1219 * 1220 */ 1221 static int 1222 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1223 { 1224 caddr_t addr = (caddr_t)*vaddrp; 1225 uint_t npages, pgoffset; 1226 struct regspec *rp; 1227 1228 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1229 return (0); 1230 1231 rp = mp->map_obj.rp; 1232 1233 if (rp->regspec_size == 0) { 1234 #ifdef DDI_MAP_DEBUG 1235 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1236 #endif /* DDI_MAP_DEBUG */ 1237 return (DDI_ME_INVAL); 1238 } 1239 1240 /* 1241 * I/O or memory mapping: 1242 * 1243 * <bustype=0, addr=x, len=x>: memory 1244 * <bustype=1, addr=x, len=x>: i/o 1245 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1246 */ 1247 if (rp->regspec_bustype != 0) { 1248 /* 1249 * This is I/O space, which requires no particular 1250 * processing on unmap since it isn't mapped in the 1251 * first place. 1252 */ 1253 return (DDI_SUCCESS); 1254 } 1255 1256 /* 1257 * Memory space 1258 */ 1259 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1260 npages = mmu_btopr(rp->regspec_size + pgoffset); 1261 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1262 device_arena_free(addr - pgoffset, ptob(npages)); 1263 1264 /* 1265 * Destroy the pointer - the mapping has logically gone 1266 */ 1267 *vaddrp = NULL; 1268 1269 return (DDI_SUCCESS); 1270 } 1271 1272 1273 /* 1274 * rootnex_map_handle() 1275 * 1276 */ 1277 static int 1278 rootnex_map_handle(ddi_map_req_t *mp) 1279 { 1280 rootnex_addr_t rbase; 1281 ddi_acc_hdl_t *hp; 1282 uint_t pgoffset; 1283 struct regspec *rp; 1284 paddr_t pbase; 1285 1286 rp = mp->map_obj.rp; 1287 1288 #ifdef DDI_MAP_DEBUG 1289 ddi_map_debug( 1290 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1291 rp->regspec_bustype, rp->regspec_addr, 1292 rp->regspec_size, mp->map_handlep); 1293 #endif /* DDI_MAP_DEBUG */ 1294 1295 /* 1296 * I/O or memory mapping: 1297 * 1298 * <bustype=0, addr=x, len=x>: memory 1299 * <bustype=1, addr=x, len=x>: i/o 1300 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1301 */ 1302 if (rp->regspec_bustype != 0) { 1303 /* 1304 * This refers to I/O space, and we don't support "mapping" 1305 * I/O space to a user. 1306 */ 1307 return (DDI_FAILURE); 1308 } 1309 1310 /* 1311 * Set up the hat_flags for the mapping. 1312 */ 1313 hp = mp->map_handlep; 1314 1315 switch (hp->ah_acc.devacc_attr_endian_flags) { 1316 case DDI_NEVERSWAP_ACC: 1317 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1318 break; 1319 case DDI_STRUCTURE_LE_ACC: 1320 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1321 break; 1322 case DDI_STRUCTURE_BE_ACC: 1323 return (DDI_FAILURE); 1324 default: 1325 return (DDI_REGS_ACC_CONFLICT); 1326 } 1327 1328 switch (hp->ah_acc.devacc_attr_dataorder) { 1329 case DDI_STRICTORDER_ACC: 1330 break; 1331 case DDI_UNORDERED_OK_ACC: 1332 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1333 break; 1334 case DDI_MERGING_OK_ACC: 1335 hp->ah_hat_flags |= HAT_MERGING_OK; 1336 break; 1337 case DDI_LOADCACHING_OK_ACC: 1338 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1339 break; 1340 case DDI_STORECACHING_OK_ACC: 1341 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1342 break; 1343 default: 1344 return (DDI_FAILURE); 1345 } 1346 1347 rbase = (rootnex_addr_t)rp->regspec_addr & 1348 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1349 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1350 1351 if (rp->regspec_size == 0) 1352 return (DDI_ME_INVAL); 1353 1354 #ifdef __xpv 1355 /* 1356 * If we're dom0, we're using a real device so we need to translate 1357 * the MA to a PA. 1358 */ 1359 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1360 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1361 (rbase & MMU_PAGEOFFSET); 1362 } else { 1363 pbase = rbase; 1364 } 1365 #else 1366 pbase = rbase; 1367 #endif 1368 1369 hp->ah_pfn = mmu_btop(pbase); 1370 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1371 1372 return (DDI_SUCCESS); 1373 } 1374 1375 1376 1377 /* 1378 * ************************ 1379 * interrupt related code 1380 * ************************ 1381 */ 1382 1383 /* 1384 * rootnex_intr_ops() 1385 * bus_intr_op() function for interrupt support 1386 */ 1387 /* ARGSUSED */ 1388 static int 1389 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1390 ddi_intr_handle_impl_t *hdlp, void *result) 1391 { 1392 struct intrspec *ispec; 1393 1394 DDI_INTR_NEXDBG((CE_CONT, 1395 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1396 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1397 1398 /* Process the interrupt operation */ 1399 switch (intr_op) { 1400 case DDI_INTROP_GETCAP: 1401 /* First check with pcplusmp */ 1402 if (psm_intr_ops == NULL) 1403 return (DDI_FAILURE); 1404 1405 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1406 *(int *)result = 0; 1407 return (DDI_FAILURE); 1408 } 1409 break; 1410 case DDI_INTROP_SETCAP: 1411 if (psm_intr_ops == NULL) 1412 return (DDI_FAILURE); 1413 1414 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1415 return (DDI_FAILURE); 1416 break; 1417 case DDI_INTROP_ALLOC: 1418 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1419 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1420 case DDI_INTROP_FREE: 1421 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1422 return (rootnex_free_intr_fixed(rdip, hdlp)); 1423 case DDI_INTROP_GETPRI: 1424 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1425 return (DDI_FAILURE); 1426 *(int *)result = ispec->intrspec_pri; 1427 break; 1428 case DDI_INTROP_SETPRI: 1429 /* Validate the interrupt priority passed to us */ 1430 if (*(int *)result > LOCK_LEVEL) 1431 return (DDI_FAILURE); 1432 1433 /* Ensure that PSM is all initialized and ispec is ok */ 1434 if ((psm_intr_ops == NULL) || 1435 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1436 return (DDI_FAILURE); 1437 1438 /* Change the priority */ 1439 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1440 PSM_FAILURE) 1441 return (DDI_FAILURE); 1442 1443 /* update the ispec with the new priority */ 1444 ispec->intrspec_pri = *(int *)result; 1445 break; 1446 case DDI_INTROP_ADDISR: 1447 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1448 return (DDI_FAILURE); 1449 ispec->intrspec_func = hdlp->ih_cb_func; 1450 break; 1451 case DDI_INTROP_REMISR: 1452 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1453 return (DDI_FAILURE); 1454 ispec->intrspec_func = (uint_t (*)()) 0; 1455 break; 1456 case DDI_INTROP_ENABLE: 1457 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1458 return (DDI_FAILURE); 1459 1460 /* Call psmi to translate irq with the dip */ 1461 if (psm_intr_ops == NULL) 1462 return (DDI_FAILURE); 1463 1464 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1465 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1466 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1467 return (DDI_FAILURE); 1468 1469 /* Add the interrupt handler */ 1470 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1471 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1472 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1473 return (DDI_FAILURE); 1474 break; 1475 case DDI_INTROP_DISABLE: 1476 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1477 return (DDI_FAILURE); 1478 1479 /* Call psm_ops() to translate irq with the dip */ 1480 if (psm_intr_ops == NULL) 1481 return (DDI_FAILURE); 1482 1483 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1484 (void) (*psm_intr_ops)(rdip, hdlp, 1485 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1486 1487 /* Remove the interrupt handler */ 1488 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1489 hdlp->ih_cb_func, hdlp->ih_vector); 1490 break; 1491 case DDI_INTROP_SETMASK: 1492 if (psm_intr_ops == NULL) 1493 return (DDI_FAILURE); 1494 1495 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1496 return (DDI_FAILURE); 1497 break; 1498 case DDI_INTROP_CLRMASK: 1499 if (psm_intr_ops == NULL) 1500 return (DDI_FAILURE); 1501 1502 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1503 return (DDI_FAILURE); 1504 break; 1505 case DDI_INTROP_GETPENDING: 1506 if (psm_intr_ops == NULL) 1507 return (DDI_FAILURE); 1508 1509 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1510 result)) { 1511 *(int *)result = 0; 1512 return (DDI_FAILURE); 1513 } 1514 break; 1515 case DDI_INTROP_NAVAIL: 1516 case DDI_INTROP_NINTRS: 1517 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1518 if (*(int *)result == 0) { 1519 /* 1520 * Special case for 'pcic' driver' only. This driver 1521 * driver is a child of 'isa' and 'rootnex' drivers. 1522 * 1523 * See detailed comments on this in the function 1524 * rootnex_get_ispec(). 1525 * 1526 * Children of 'pcic' send 'NINITR' request all the 1527 * way to rootnex driver. But, the 'pdp->par_nintr' 1528 * field may not initialized. So, we fake it here 1529 * to return 1 (a la what PCMCIA nexus does). 1530 */ 1531 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1532 *(int *)result = 1; 1533 else 1534 return (DDI_FAILURE); 1535 } 1536 break; 1537 case DDI_INTROP_SUPPORTED_TYPES: 1538 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1539 break; 1540 default: 1541 return (DDI_FAILURE); 1542 } 1543 1544 return (DDI_SUCCESS); 1545 } 1546 1547 1548 /* 1549 * rootnex_get_ispec() 1550 * convert an interrupt number to an interrupt specification. 1551 * The interrupt number determines which interrupt spec will be 1552 * returned if more than one exists. 1553 * 1554 * Look into the parent private data area of the 'rdip' to find out 1555 * the interrupt specification. First check to make sure there is 1556 * one that matchs "inumber" and then return a pointer to it. 1557 * 1558 * Return NULL if one could not be found. 1559 * 1560 * NOTE: This is needed for rootnex_intr_ops() 1561 */ 1562 static struct intrspec * 1563 rootnex_get_ispec(dev_info_t *rdip, int inum) 1564 { 1565 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1566 1567 /* 1568 * Special case handling for drivers that provide their own 1569 * intrspec structures instead of relying on the DDI framework. 1570 * 1571 * A broken hardware driver in ON could potentially provide its 1572 * own intrspec structure, instead of relying on the hardware. 1573 * If these drivers are children of 'rootnex' then we need to 1574 * continue to provide backward compatibility to them here. 1575 * 1576 * Following check is a special case for 'pcic' driver which 1577 * was found to have broken hardwre andby provides its own intrspec. 1578 * 1579 * Verbatim comments from this driver are shown here: 1580 * "Don't use the ddi_add_intr since we don't have a 1581 * default intrspec in all cases." 1582 * 1583 * Since an 'ispec' may not be always created for it, 1584 * check for that and create one if so. 1585 * 1586 * NOTE: Currently 'pcic' is the only driver found to do this. 1587 */ 1588 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1589 pdp->par_nintr = 1; 1590 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1591 pdp->par_nintr, KM_SLEEP); 1592 } 1593 1594 /* Validate the interrupt number */ 1595 if (inum >= pdp->par_nintr) 1596 return (NULL); 1597 1598 /* Get the interrupt structure pointer and return that */ 1599 return ((struct intrspec *)&pdp->par_intr[inum]); 1600 } 1601 1602 /* 1603 * Allocate interrupt vector for FIXED (legacy) type. 1604 */ 1605 static int 1606 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1607 void *result) 1608 { 1609 struct intrspec *ispec; 1610 ddi_intr_handle_impl_t info_hdl; 1611 int ret; 1612 int free_phdl = 0; 1613 apic_get_type_t type_info; 1614 1615 if (psm_intr_ops == NULL) 1616 return (DDI_FAILURE); 1617 1618 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1619 return (DDI_FAILURE); 1620 1621 /* 1622 * If the PSM module is "APIX" then pass the request for it 1623 * to allocate the vector now. 1624 */ 1625 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1626 info_hdl.ih_private = &type_info; 1627 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1628 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1629 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1630 free_phdl = 1; 1631 i_ddi_alloc_intr_phdl(hdlp); 1632 } 1633 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1634 ret = (*psm_intr_ops)(rdip, hdlp, 1635 PSM_INTR_OP_ALLOC_VECTORS, result); 1636 if (free_phdl) { /* free up the phdl structure */ 1637 free_phdl = 0; 1638 i_ddi_free_intr_phdl(hdlp); 1639 hdlp->ih_private = NULL; 1640 } 1641 } else { 1642 /* 1643 * No APIX module; fall back to the old scheme where the 1644 * interrupt vector is allocated during ddi_enable_intr() call. 1645 */ 1646 hdlp->ih_pri = ispec->intrspec_pri; 1647 *(int *)result = hdlp->ih_scratch1; 1648 ret = DDI_SUCCESS; 1649 } 1650 1651 return (ret); 1652 } 1653 1654 /* 1655 * Free up interrupt vector for FIXED (legacy) type. 1656 */ 1657 static int 1658 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1659 { 1660 struct intrspec *ispec; 1661 struct ddi_parent_private_data *pdp; 1662 ddi_intr_handle_impl_t info_hdl; 1663 int ret; 1664 apic_get_type_t type_info; 1665 1666 if (psm_intr_ops == NULL) 1667 return (DDI_FAILURE); 1668 1669 /* 1670 * If the PSM module is "APIX" then pass the request for it 1671 * to free up the vector now. 1672 */ 1673 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1674 info_hdl.ih_private = &type_info; 1675 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1676 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1677 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1678 return (DDI_FAILURE); 1679 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1680 ret = (*psm_intr_ops)(rdip, hdlp, 1681 PSM_INTR_OP_FREE_VECTORS, NULL); 1682 } else { 1683 /* 1684 * No APIX module; fall back to the old scheme where 1685 * the interrupt vector was already freed during 1686 * ddi_disable_intr() call. 1687 */ 1688 ret = DDI_SUCCESS; 1689 } 1690 1691 pdp = ddi_get_parent_data(rdip); 1692 1693 /* 1694 * Special case for 'pcic' driver' only. 1695 * If an intrspec was created for it, clean it up here 1696 * See detailed comments on this in the function 1697 * rootnex_get_ispec(). 1698 */ 1699 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1700 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1701 pdp->par_nintr); 1702 /* 1703 * Set it to zero; so that 1704 * DDI framework doesn't free it again 1705 */ 1706 pdp->par_intr = NULL; 1707 pdp->par_nintr = 0; 1708 } 1709 1710 return (ret); 1711 } 1712 1713 1714 /* 1715 * ****************** 1716 * dma related code 1717 * ****************** 1718 */ 1719 1720 /*ARGSUSED*/ 1721 static int 1722 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1723 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1724 ddi_dma_handle_t *handlep) 1725 { 1726 uint64_t maxsegmentsize_ll; 1727 uint_t maxsegmentsize; 1728 ddi_dma_impl_t *hp; 1729 rootnex_dma_t *dma; 1730 uint64_t count_max; 1731 uint64_t seg; 1732 int kmflag; 1733 int e; 1734 1735 1736 /* convert our sleep flags */ 1737 if (waitfp == DDI_DMA_SLEEP) { 1738 kmflag = KM_SLEEP; 1739 } else { 1740 kmflag = KM_NOSLEEP; 1741 } 1742 1743 /* 1744 * We try to do only one memory allocation here. We'll do a little 1745 * pointer manipulation later. If the bind ends up taking more than 1746 * our prealloc's space, we'll have to allocate more memory in the 1747 * bind operation. Not great, but much better than before and the 1748 * best we can do with the current bind interfaces. 1749 */ 1750 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1751 if (hp == NULL) 1752 return (DDI_DMA_NORESOURCES); 1753 1754 /* Do our pointer manipulation now, align the structures */ 1755 hp->dmai_private = (void *)(((uintptr_t)hp + 1756 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1757 dma = (rootnex_dma_t *)hp->dmai_private; 1758 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1759 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1760 1761 /* setup the handle */ 1762 rootnex_clean_dmahdl(hp); 1763 hp->dmai_error.err_fep = NULL; 1764 hp->dmai_error.err_cf = NULL; 1765 dma->dp_dip = rdip; 1766 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1767 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1768 1769 /* 1770 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1771 * is being used. Set the upper limit to the seg value. 1772 * There will be enough DVMA space to always get addresses 1773 * that will match the constraints. 1774 */ 1775 if (IOMMU_USED(rdip) && 1776 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1777 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1778 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1779 } else 1780 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1781 1782 hp->dmai_minxfer = attr->dma_attr_minxfer; 1783 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1784 hp->dmai_rdip = rdip; 1785 hp->dmai_attr = *attr; 1786 1787 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1788 dma->dp_sglinfo.si_cancross = B_FALSE; 1789 else 1790 dma->dp_sglinfo.si_cancross = B_TRUE; 1791 1792 /* we don't need to worry about the SPL since we do a tryenter */ 1793 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1794 1795 /* 1796 * Figure out our maximum segment size. If the segment size is greater 1797 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1798 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1799 * dma_attr_count_max are size-1 type values. 1800 * 1801 * Maximum segment size is the largest physically contiguous chunk of 1802 * memory that we can return from a bind (i.e. the maximum size of a 1803 * single cookie). 1804 */ 1805 1806 /* handle the rollover cases */ 1807 seg = attr->dma_attr_seg + 1; 1808 if (seg < attr->dma_attr_seg) { 1809 seg = attr->dma_attr_seg; 1810 } 1811 count_max = attr->dma_attr_count_max + 1; 1812 if (count_max < attr->dma_attr_count_max) { 1813 count_max = attr->dma_attr_count_max; 1814 } 1815 1816 /* 1817 * granularity may or may not be a power of two. If it isn't, we can't 1818 * use a simple mask. 1819 */ 1820 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1821 dma->dp_granularity_power_2 = B_FALSE; 1822 } else { 1823 dma->dp_granularity_power_2 = B_TRUE; 1824 } 1825 1826 /* 1827 * maxxfer should be a whole multiple of granularity. If we're going to 1828 * break up a window because we're greater than maxxfer, we might as 1829 * well make sure it's maxxfer is a whole multiple so we don't have to 1830 * worry about triming the window later on for this case. 1831 */ 1832 if (attr->dma_attr_granular > 1) { 1833 if (dma->dp_granularity_power_2) { 1834 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1835 (attr->dma_attr_maxxfer & 1836 (attr->dma_attr_granular - 1)); 1837 } else { 1838 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1839 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1840 } 1841 } else { 1842 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1843 } 1844 1845 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1846 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1847 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1848 maxsegmentsize = 0xFFFFFFFF; 1849 } else { 1850 maxsegmentsize = maxsegmentsize_ll; 1851 } 1852 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1853 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1854 1855 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1856 if (rootnex_alloc_check_parms) { 1857 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1858 if (e != DDI_SUCCESS) { 1859 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1860 (void) rootnex_dma_freehdl(dip, rdip, 1861 (ddi_dma_handle_t)hp); 1862 return (e); 1863 } 1864 } 1865 1866 *handlep = (ddi_dma_handle_t)hp; 1867 1868 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1869 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1870 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1871 1872 return (DDI_SUCCESS); 1873 } 1874 1875 1876 /* 1877 * rootnex_dma_allochdl() 1878 * called from ddi_dma_alloc_handle(). 1879 */ 1880 static int 1881 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1882 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1883 { 1884 int retval = DDI_SUCCESS; 1885 #if defined(__amd64) && !defined(__xpv) 1886 1887 if (IOMMU_UNITIALIZED(rdip)) { 1888 retval = iommulib_nex_open(dip, rdip); 1889 1890 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1891 return (retval); 1892 } 1893 1894 if (IOMMU_UNUSED(rdip)) { 1895 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1896 handlep); 1897 } else { 1898 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1899 waitfp, arg, handlep); 1900 } 1901 #else 1902 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1903 handlep); 1904 #endif 1905 switch (retval) { 1906 case DDI_DMA_NORESOURCES: 1907 if (waitfp != DDI_DMA_DONTWAIT) { 1908 ddi_set_callback(waitfp, arg, 1909 &rootnex_state->r_dvma_call_list_id); 1910 } 1911 break; 1912 case DDI_SUCCESS: 1913 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1914 break; 1915 default: 1916 break; 1917 } 1918 return (retval); 1919 } 1920 1921 /*ARGSUSED*/ 1922 static int 1923 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1924 ddi_dma_handle_t handle) 1925 { 1926 ddi_dma_impl_t *hp; 1927 rootnex_dma_t *dma; 1928 1929 1930 hp = (ddi_dma_impl_t *)handle; 1931 dma = (rootnex_dma_t *)hp->dmai_private; 1932 1933 /* unbind should have been called first */ 1934 ASSERT(!dma->dp_inuse); 1935 1936 mutex_destroy(&dma->dp_mutex); 1937 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1938 1939 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1940 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1941 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1942 1943 return (DDI_SUCCESS); 1944 } 1945 1946 /* 1947 * rootnex_dma_freehdl() 1948 * called from ddi_dma_free_handle(). 1949 */ 1950 static int 1951 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1952 { 1953 int ret; 1954 1955 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1956 #if defined(__amd64) && !defined(__xpv) 1957 if (IOMMU_USED(rdip)) 1958 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1959 else 1960 #endif 1961 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1962 1963 if (rootnex_state->r_dvma_call_list_id) 1964 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1965 1966 return (ret); 1967 } 1968 1969 /*ARGSUSED*/ 1970 static int 1971 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1972 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1973 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1974 { 1975 rootnex_sglinfo_t *sinfo; 1976 ddi_dma_obj_t *dmao; 1977 #if defined(__amd64) && !defined(__xpv) 1978 struct dvmaseg *dvs; 1979 ddi_dma_cookie_t *cookie; 1980 #endif 1981 ddi_dma_attr_t *attr; 1982 ddi_dma_impl_t *hp; 1983 rootnex_dma_t *dma; 1984 int kmflag; 1985 int e; 1986 uint_t ncookies; 1987 1988 hp = (ddi_dma_impl_t *)handle; 1989 dma = (rootnex_dma_t *)hp->dmai_private; 1990 dmao = &dma->dp_dma; 1991 sinfo = &dma->dp_sglinfo; 1992 attr = &hp->dmai_attr; 1993 1994 /* convert the sleep flags */ 1995 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1996 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1997 } else { 1998 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1999 } 2000 2001 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 2002 2003 /* 2004 * This is useful for debugging a driver. Not as useful in a production 2005 * system. The only time this will fail is if you have a driver bug. 2006 */ 2007 if (rootnex_bind_check_inuse) { 2008 /* 2009 * No one else should ever have this lock unless someone else 2010 * is trying to use this handle. So contention on the lock 2011 * is the same as inuse being set. 2012 */ 2013 e = mutex_tryenter(&dma->dp_mutex); 2014 if (e == 0) { 2015 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2016 return (DDI_DMA_INUSE); 2017 } 2018 if (dma->dp_inuse) { 2019 mutex_exit(&dma->dp_mutex); 2020 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2021 return (DDI_DMA_INUSE); 2022 } 2023 dma->dp_inuse = B_TRUE; 2024 mutex_exit(&dma->dp_mutex); 2025 } 2026 2027 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 2028 if (rootnex_bind_check_parms) { 2029 e = rootnex_valid_bind_parms(dmareq, attr); 2030 if (e != DDI_SUCCESS) { 2031 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2032 rootnex_clean_dmahdl(hp); 2033 return (e); 2034 } 2035 } 2036 2037 /* save away the original bind info */ 2038 dma->dp_dma = dmareq->dmar_object; 2039 2040 #if defined(__amd64) && !defined(__xpv) 2041 if (IOMMU_USED(rdip)) { 2042 dmao = &dma->dp_dvma; 2043 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2044 switch (e) { 2045 case DDI_SUCCESS: 2046 if (sinfo->si_cancross || 2047 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2048 dmao->dmao_size > sinfo->si_max_cookie_size) { 2049 dma->dp_dvma_used = B_TRUE; 2050 break; 2051 } 2052 sinfo->si_sgl_size = 1; 2053 hp->dmai_rflags |= DMP_NOSYNC; 2054 2055 dma->dp_dvma_used = B_TRUE; 2056 dma->dp_need_to_free_cookie = B_FALSE; 2057 2058 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2059 cookie = hp->dmai_cookie = dma->dp_cookies = 2060 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2061 cookie->dmac_laddress = dvs->dvs_start + 2062 dmao->dmao_obj.dvma_obj.dv_off; 2063 cookie->dmac_size = dvs->dvs_len; 2064 cookie->dmac_type = 0; 2065 2066 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2067 rdip); 2068 goto fast; 2069 case DDI_ENOTSUP: 2070 break; 2071 default: 2072 rootnex_clean_dmahdl(hp); 2073 return (e); 2074 } 2075 } 2076 #endif 2077 2078 /* 2079 * Figure out a rough estimate of what maximum number of pages 2080 * this buffer could use (a high estimate of course). 2081 */ 2082 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2083 2084 if (dma->dp_dvma_used) { 2085 /* 2086 * The number of physical pages is the worst case. 2087 * 2088 * For DVMA, the worst case is the length divided 2089 * by the maximum cookie length, plus 1. Add to that 2090 * the number of segment boundaries potentially crossed, and 2091 * the additional number of DVMA segments that was returned. 2092 * 2093 * In the normal case, for modern devices, si_cancross will 2094 * be false, and dv_nseg will be 1, and the fast path will 2095 * have been taken above. 2096 */ 2097 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2098 + 1; 2099 if (sinfo->si_cancross) 2100 ncookies += 2101 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2102 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2103 2104 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2105 } 2106 2107 /* 2108 * We'll use the pre-allocated cookies for any bind that will *always* 2109 * fit (more important to be consistent, we don't want to create 2110 * additional degenerate cases). 2111 */ 2112 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2113 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2114 dma->dp_need_to_free_cookie = B_FALSE; 2115 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2116 uint_t, sinfo->si_max_pages); 2117 2118 /* 2119 * For anything larger than that, we'll go ahead and allocate the 2120 * maximum number of pages we expect to see. Hopefuly, we won't be 2121 * seeing this path in the fast path for high performance devices very 2122 * frequently. 2123 * 2124 * a ddi bind interface that allowed the driver to provide storage to 2125 * the bind interface would speed this case up. 2126 */ 2127 } else { 2128 /* 2129 * Save away how much memory we allocated. If we're doing a 2130 * nosleep, the alloc could fail... 2131 */ 2132 dma->dp_cookie_size = sinfo->si_max_pages * 2133 sizeof (ddi_dma_cookie_t); 2134 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2135 if (dma->dp_cookies == NULL) { 2136 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2137 rootnex_clean_dmahdl(hp); 2138 return (DDI_DMA_NORESOURCES); 2139 } 2140 dma->dp_need_to_free_cookie = B_TRUE; 2141 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2142 uint_t, sinfo->si_max_pages); 2143 } 2144 hp->dmai_cookie = dma->dp_cookies; 2145 2146 /* 2147 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2148 * looking at the constraints in the dma structure. It will then put 2149 * some additional state about the sgl in the dma struct (i.e. is 2150 * the sgl clean, or do we need to do some munging; how many pages 2151 * need to be copied, etc.) 2152 */ 2153 if (dma->dp_dvma_used) 2154 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2155 else 2156 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2157 2158 out: 2159 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2160 /* if we don't need a copy buffer, we don't need to sync */ 2161 if (sinfo->si_copybuf_req == 0) { 2162 hp->dmai_rflags |= DMP_NOSYNC; 2163 } 2164 2165 /* 2166 * if we don't need the copybuf and we don't need to do a partial, we 2167 * hit the fast path. All the high performance devices should be trying 2168 * to hit this path. To hit this path, a device should be able to reach 2169 * all of memory, shouldn't try to bind more than it can transfer, and 2170 * the buffer shouldn't require more cookies than the driver/device can 2171 * handle [sgllen]). 2172 * 2173 * Note that negative values of dma_attr_sgllen are supposed 2174 * to mean unlimited, but we just cast them to mean a 2175 * "ridiculous large limit". This saves some extra checks on 2176 * hot paths. 2177 */ 2178 if ((sinfo->si_copybuf_req == 0) && 2179 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2180 (dmao->dmao_size < dma->dp_maxxfer)) { 2181 fast: 2182 /* 2183 * If the driver supports FMA, insert the handle in the FMA DMA 2184 * handle cache. 2185 */ 2186 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2187 hp->dmai_error.err_cf = rootnex_dma_check; 2188 2189 /* 2190 * copy out the first cookie and ccountp, set the cookie 2191 * pointer to the second cookie. The first cookie is passed 2192 * back on the stack. Additional cookies are accessed via 2193 * ddi_dma_nextcookie() 2194 */ 2195 *cookiep = dma->dp_cookies[0]; 2196 *ccountp = sinfo->si_sgl_size; 2197 hp->dmai_cookie++; 2198 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2199 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2200 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2201 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2202 uint_t, dmao->dmao_size, uint_t, *ccountp); 2203 2204 2205 return (DDI_DMA_MAPPED); 2206 } 2207 2208 /* 2209 * go to the slow path, we may need to alloc more memory, create 2210 * multiple windows, and munge up a sgl to make the device happy. 2211 */ 2212 2213 /* 2214 * With the IOMMU mapobject method used, we should never hit 2215 * the slow path. If we do, something is seriously wrong. 2216 * Clean up and return an error. 2217 */ 2218 2219 #if defined(__amd64) && !defined(__xpv) 2220 2221 if (dma->dp_dvma_used) { 2222 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2223 &dma->dp_dvma); 2224 e = DDI_DMA_NOMAPPING; 2225 } else { 2226 #endif 2227 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2228 kmflag); 2229 #if defined(__amd64) && !defined(__xpv) 2230 } 2231 #endif 2232 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2233 if (dma->dp_need_to_free_cookie) { 2234 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2235 } 2236 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2237 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2238 return (e); 2239 } 2240 2241 /* 2242 * If the driver supports FMA, insert the handle in the FMA DMA handle 2243 * cache. 2244 */ 2245 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2246 hp->dmai_error.err_cf = rootnex_dma_check; 2247 2248 /* if the first window uses the copy buffer, sync it for the device */ 2249 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2250 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2251 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2252 DDI_DMA_SYNC_FORDEV); 2253 } 2254 2255 /* 2256 * copy out the first cookie and ccountp, set the cookie pointer to the 2257 * second cookie. Make sure the partial flag is set/cleared correctly. 2258 * If we have a partial map (i.e. multiple windows), the number of 2259 * cookies we return is the number of cookies in the first window. 2260 */ 2261 if (e == DDI_DMA_MAPPED) { 2262 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2263 *ccountp = sinfo->si_sgl_size; 2264 hp->dmai_nwin = 1; 2265 } else { 2266 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2267 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2268 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2269 } 2270 *cookiep = dma->dp_cookies[0]; 2271 hp->dmai_cookie++; 2272 2273 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2274 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2275 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2276 dmao->dmao_size, uint_t, *ccountp); 2277 return (e); 2278 } 2279 2280 /* 2281 * rootnex_dma_bindhdl() 2282 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2283 */ 2284 static int 2285 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2286 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2287 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2288 { 2289 int ret; 2290 #if defined(__amd64) && !defined(__xpv) 2291 if (IOMMU_USED(rdip)) 2292 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2293 cookiep, ccountp); 2294 else 2295 #endif 2296 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2297 cookiep, ccountp); 2298 2299 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2300 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2301 &rootnex_state->r_dvma_call_list_id); 2302 } 2303 2304 return (ret); 2305 } 2306 2307 2308 2309 /*ARGSUSED*/ 2310 static int 2311 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2312 ddi_dma_handle_t handle) 2313 { 2314 ddi_dma_impl_t *hp; 2315 rootnex_dma_t *dma; 2316 int e; 2317 2318 hp = (ddi_dma_impl_t *)handle; 2319 dma = (rootnex_dma_t *)hp->dmai_private; 2320 2321 /* make sure the buffer wasn't free'd before calling unbind */ 2322 if (rootnex_unbind_verify_buffer) { 2323 e = rootnex_verify_buffer(dma); 2324 if (e != DDI_SUCCESS) { 2325 ASSERT(0); 2326 return (DDI_FAILURE); 2327 } 2328 } 2329 2330 /* sync the current window before unbinding the buffer */ 2331 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2332 (hp->dmai_rflags & DDI_DMA_READ)) { 2333 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2334 DDI_DMA_SYNC_FORCPU); 2335 } 2336 2337 /* 2338 * cleanup and copy buffer or window state. if we didn't use the copy 2339 * buffer or windows, there won't be much to do :-) 2340 */ 2341 rootnex_teardown_copybuf(dma); 2342 rootnex_teardown_windows(dma); 2343 2344 #if defined(__amd64) && !defined(__xpv) 2345 if (IOMMU_USED(rdip)) 2346 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2347 &dma->dp_dvma); 2348 #endif 2349 2350 /* 2351 * If we had to allocate space to for the worse case sgl (it didn't 2352 * fit into our pre-allocate buffer), free that up now 2353 */ 2354 if (dma->dp_need_to_free_cookie) { 2355 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2356 } 2357 2358 /* 2359 * clean up the handle so it's ready for the next bind (i.e. if the 2360 * handle is reused). 2361 */ 2362 rootnex_clean_dmahdl(hp); 2363 hp->dmai_error.err_cf = NULL; 2364 2365 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2366 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2367 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2368 2369 return (DDI_SUCCESS); 2370 } 2371 2372 /* 2373 * rootnex_dma_unbindhdl() 2374 * called from ddi_dma_unbind_handle() 2375 */ 2376 /*ARGSUSED*/ 2377 static int 2378 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2379 ddi_dma_handle_t handle) 2380 { 2381 int ret; 2382 2383 #if defined(__amd64) && !defined(__xpv) 2384 if (IOMMU_USED(rdip)) 2385 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2386 else 2387 #endif 2388 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2389 2390 if (rootnex_state->r_dvma_call_list_id) 2391 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2392 2393 return (ret); 2394 } 2395 2396 #if defined(__amd64) && !defined(__xpv) 2397 2398 static int 2399 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2400 { 2401 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2402 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2403 2404 if (dma->dp_sleep_flags != KM_SLEEP && 2405 dma->dp_sleep_flags != KM_NOSLEEP) 2406 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2407 return (dma->dp_sleep_flags); 2408 } 2409 /*ARGSUSED*/ 2410 static void 2411 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2412 { 2413 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2414 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2415 rootnex_window_t *window; 2416 2417 if (dma->dp_window) { 2418 window = &dma->dp_window[dma->dp_current_win]; 2419 hp->dmai_cookie = window->wd_first_cookie; 2420 } else { 2421 hp->dmai_cookie = dma->dp_cookies; 2422 } 2423 hp->dmai_cookie++; 2424 } 2425 2426 /*ARGSUSED*/ 2427 static int 2428 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2429 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2430 { 2431 int i; 2432 int km_flags; 2433 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2434 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2435 rootnex_window_t *window; 2436 ddi_dma_cookie_t *cp; 2437 ddi_dma_cookie_t *cookie; 2438 2439 ASSERT(*cookiepp == NULL); 2440 ASSERT(*ccountp == 0); 2441 2442 if (dma->dp_window) { 2443 window = &dma->dp_window[dma->dp_current_win]; 2444 cp = window->wd_first_cookie; 2445 *ccountp = window->wd_cookie_cnt; 2446 } else { 2447 cp = dma->dp_cookies; 2448 *ccountp = dma->dp_sglinfo.si_sgl_size; 2449 } 2450 2451 km_flags = rootnex_coredma_get_sleep_flags(handle); 2452 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2453 if (cookie == NULL) { 2454 return (DDI_DMA_NORESOURCES); 2455 } 2456 2457 for (i = 0; i < *ccountp; i++) { 2458 cookie[i].dmac_notused = cp[i].dmac_notused; 2459 cookie[i].dmac_type = cp[i].dmac_type; 2460 cookie[i].dmac_address = cp[i].dmac_address; 2461 cookie[i].dmac_size = cp[i].dmac_size; 2462 } 2463 2464 *cookiepp = cookie; 2465 2466 return (DDI_SUCCESS); 2467 } 2468 2469 /*ARGSUSED*/ 2470 static int 2471 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2472 ddi_dma_cookie_t *cookiep, uint_t ccount) 2473 { 2474 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2475 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2476 rootnex_window_t *window; 2477 ddi_dma_cookie_t *cur_cookiep; 2478 2479 ASSERT(cookiep); 2480 ASSERT(ccount != 0); 2481 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2482 2483 if (dma->dp_window) { 2484 window = &dma->dp_window[dma->dp_current_win]; 2485 dma->dp_saved_cookies = window->wd_first_cookie; 2486 window->wd_first_cookie = cookiep; 2487 ASSERT(ccount == window->wd_cookie_cnt); 2488 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2489 + window->wd_first_cookie; 2490 } else { 2491 dma->dp_saved_cookies = dma->dp_cookies; 2492 dma->dp_cookies = cookiep; 2493 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2494 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2495 + dma->dp_cookies; 2496 } 2497 2498 dma->dp_need_to_switch_cookies = B_TRUE; 2499 hp->dmai_cookie = cur_cookiep; 2500 2501 return (DDI_SUCCESS); 2502 } 2503 2504 /*ARGSUSED*/ 2505 static int 2506 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2507 { 2508 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2509 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2510 rootnex_window_t *window; 2511 ddi_dma_cookie_t *cur_cookiep; 2512 ddi_dma_cookie_t *cookie_array; 2513 uint_t ccount; 2514 2515 /* check if cookies have not been switched */ 2516 if (dma->dp_need_to_switch_cookies == B_FALSE) 2517 return (DDI_SUCCESS); 2518 2519 ASSERT(dma->dp_saved_cookies); 2520 2521 if (dma->dp_window) { 2522 window = &dma->dp_window[dma->dp_current_win]; 2523 cookie_array = window->wd_first_cookie; 2524 window->wd_first_cookie = dma->dp_saved_cookies; 2525 dma->dp_saved_cookies = NULL; 2526 ccount = window->wd_cookie_cnt; 2527 cur_cookiep = (hp->dmai_cookie - cookie_array) 2528 + window->wd_first_cookie; 2529 } else { 2530 cookie_array = dma->dp_cookies; 2531 dma->dp_cookies = dma->dp_saved_cookies; 2532 dma->dp_saved_cookies = NULL; 2533 ccount = dma->dp_sglinfo.si_sgl_size; 2534 cur_cookiep = (hp->dmai_cookie - cookie_array) 2535 + dma->dp_cookies; 2536 } 2537 2538 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2539 2540 hp->dmai_cookie = cur_cookiep; 2541 2542 dma->dp_need_to_switch_cookies = B_FALSE; 2543 2544 return (DDI_SUCCESS); 2545 } 2546 2547 #endif 2548 2549 static struct as * 2550 rootnex_get_as(ddi_dma_obj_t *dmao) 2551 { 2552 struct as *asp; 2553 2554 switch (dmao->dmao_type) { 2555 case DMA_OTYP_VADDR: 2556 case DMA_OTYP_BUFVADDR: 2557 asp = dmao->dmao_obj.virt_obj.v_as; 2558 if (asp == NULL) 2559 asp = &kas; 2560 break; 2561 default: 2562 asp = NULL; 2563 break; 2564 } 2565 return (asp); 2566 } 2567 2568 /* 2569 * rootnex_verify_buffer() 2570 * verify buffer wasn't free'd 2571 */ 2572 static int 2573 rootnex_verify_buffer(rootnex_dma_t *dma) 2574 { 2575 page_t **pplist; 2576 caddr_t vaddr; 2577 uint_t pcnt; 2578 uint_t poff; 2579 page_t *pp; 2580 char b; 2581 int i; 2582 2583 /* Figure out how many pages this buffer occupies */ 2584 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2585 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2586 } else { 2587 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2588 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2589 } 2590 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2591 2592 switch (dma->dp_dma.dmao_type) { 2593 case DMA_OTYP_PAGES: 2594 /* 2595 * for a linked list of pp's walk through them to make sure 2596 * they're locked and not free. 2597 */ 2598 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2599 for (i = 0; i < pcnt; i++) { 2600 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2601 return (DDI_FAILURE); 2602 } 2603 pp = pp->p_next; 2604 } 2605 break; 2606 2607 case DMA_OTYP_VADDR: 2608 case DMA_OTYP_BUFVADDR: 2609 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2610 /* 2611 * for an array of pp's walk through them to make sure they're 2612 * not free. It's possible that they may not be locked. 2613 */ 2614 if (pplist) { 2615 for (i = 0; i < pcnt; i++) { 2616 if (PP_ISFREE(pplist[i])) { 2617 return (DDI_FAILURE); 2618 } 2619 } 2620 2621 /* For a virtual address, try to peek at each page */ 2622 } else { 2623 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2624 for (i = 0; i < pcnt; i++) { 2625 if (ddi_peek8(NULL, vaddr, &b) == 2626 DDI_FAILURE) 2627 return (DDI_FAILURE); 2628 vaddr += MMU_PAGESIZE; 2629 } 2630 } 2631 } 2632 break; 2633 2634 default: 2635 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2636 break; 2637 } 2638 2639 return (DDI_SUCCESS); 2640 } 2641 2642 2643 /* 2644 * rootnex_clean_dmahdl() 2645 * Clean the dma handle. This should be called on a handle alloc and an 2646 * unbind handle. Set the handle state to the default settings. 2647 */ 2648 static void 2649 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2650 { 2651 rootnex_dma_t *dma; 2652 2653 2654 dma = (rootnex_dma_t *)hp->dmai_private; 2655 2656 hp->dmai_nwin = 0; 2657 dma->dp_current_cookie = 0; 2658 dma->dp_copybuf_size = 0; 2659 dma->dp_window = NULL; 2660 dma->dp_cbaddr = NULL; 2661 dma->dp_inuse = B_FALSE; 2662 dma->dp_dvma_used = B_FALSE; 2663 dma->dp_need_to_free_cookie = B_FALSE; 2664 dma->dp_need_to_switch_cookies = B_FALSE; 2665 dma->dp_saved_cookies = NULL; 2666 dma->dp_sleep_flags = KM_PANIC; 2667 dma->dp_need_to_free_window = B_FALSE; 2668 dma->dp_partial_required = B_FALSE; 2669 dma->dp_trim_required = B_FALSE; 2670 dma->dp_sglinfo.si_copybuf_req = 0; 2671 #if !defined(__amd64) 2672 dma->dp_cb_remaping = B_FALSE; 2673 dma->dp_kva = NULL; 2674 #endif 2675 2676 /* FMA related initialization */ 2677 hp->dmai_fault = 0; 2678 hp->dmai_fault_check = NULL; 2679 hp->dmai_fault_notify = NULL; 2680 hp->dmai_error.err_ena = 0; 2681 hp->dmai_error.err_status = DDI_FM_OK; 2682 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2683 hp->dmai_error.err_ontrap = NULL; 2684 } 2685 2686 2687 /* 2688 * rootnex_valid_alloc_parms() 2689 * Called in ddi_dma_alloc_handle path to validate its parameters. 2690 */ 2691 static int 2692 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2693 { 2694 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2695 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2696 (attr->dma_attr_granular > MMU_PAGESIZE) || 2697 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2698 return (DDI_DMA_BADATTR); 2699 } 2700 2701 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2702 return (DDI_DMA_BADATTR); 2703 } 2704 2705 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2706 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2707 attr->dma_attr_sgllen == 0) { 2708 return (DDI_DMA_BADATTR); 2709 } 2710 2711 /* We should be able to DMA into every byte offset in a page */ 2712 if (maxsegmentsize < MMU_PAGESIZE) { 2713 return (DDI_DMA_BADATTR); 2714 } 2715 2716 /* if we're bouncing on seg, seg must be <= addr_hi */ 2717 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2718 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2719 return (DDI_DMA_BADATTR); 2720 } 2721 return (DDI_SUCCESS); 2722 } 2723 2724 /* 2725 * rootnex_valid_bind_parms() 2726 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2727 */ 2728 /* ARGSUSED */ 2729 static int 2730 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2731 { 2732 #if !defined(__amd64) 2733 /* 2734 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2735 * we can track the offset for the obsoleted interfaces. 2736 */ 2737 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2738 return (DDI_DMA_TOOBIG); 2739 } 2740 #endif 2741 2742 return (DDI_SUCCESS); 2743 } 2744 2745 2746 /* 2747 * rootnex_need_bounce_seg() 2748 * check to see if the buffer lives on both side of the seg. 2749 */ 2750 static boolean_t 2751 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2752 { 2753 ddi_dma_atyp_t buftype; 2754 rootnex_addr_t raddr; 2755 boolean_t lower_addr; 2756 boolean_t upper_addr; 2757 uint64_t offset; 2758 page_t **pplist; 2759 uint64_t paddr; 2760 uint32_t psize; 2761 uint32_t size; 2762 caddr_t vaddr; 2763 uint_t pcnt; 2764 page_t *pp; 2765 2766 2767 /* shortcuts */ 2768 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2769 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2770 buftype = dmar_object->dmao_type; 2771 size = dmar_object->dmao_size; 2772 2773 lower_addr = B_FALSE; 2774 upper_addr = B_FALSE; 2775 pcnt = 0; 2776 2777 /* 2778 * Process the first page to handle the initial offset of the buffer. 2779 * We'll use the base address we get later when we loop through all 2780 * the pages. 2781 */ 2782 if (buftype == DMA_OTYP_PAGES) { 2783 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2784 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2785 MMU_PAGEOFFSET; 2786 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2787 psize = MIN(size, (MMU_PAGESIZE - offset)); 2788 pp = pp->p_next; 2789 sglinfo->si_asp = NULL; 2790 } else if (pplist != NULL) { 2791 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2792 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2793 if (sglinfo->si_asp == NULL) { 2794 sglinfo->si_asp = &kas; 2795 } 2796 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2797 paddr += offset; 2798 psize = MIN(size, (MMU_PAGESIZE - offset)); 2799 pcnt++; 2800 } else { 2801 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2802 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2803 if (sglinfo->si_asp == NULL) { 2804 sglinfo->si_asp = &kas; 2805 } 2806 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2807 paddr += offset; 2808 psize = MIN(size, (MMU_PAGESIZE - offset)); 2809 vaddr += psize; 2810 } 2811 2812 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2813 2814 if ((raddr + psize) > sglinfo->si_segmask) { 2815 upper_addr = B_TRUE; 2816 } else { 2817 lower_addr = B_TRUE; 2818 } 2819 size -= psize; 2820 2821 /* 2822 * Walk through the rest of the pages in the buffer. Track to see 2823 * if we have pages on both sides of the segment boundary. 2824 */ 2825 while (size > 0) { 2826 /* partial or full page */ 2827 psize = MIN(size, MMU_PAGESIZE); 2828 2829 if (buftype == DMA_OTYP_PAGES) { 2830 /* get the paddr from the page_t */ 2831 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2832 paddr = pfn_to_pa(pp->p_pagenum); 2833 pp = pp->p_next; 2834 } else if (pplist != NULL) { 2835 /* index into the array of page_t's to get the paddr */ 2836 ASSERT(!PP_ISFREE(pplist[pcnt])); 2837 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2838 pcnt++; 2839 } else { 2840 /* call into the VM to get the paddr */ 2841 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2842 vaddr)); 2843 vaddr += psize; 2844 } 2845 2846 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2847 2848 if ((raddr + psize) > sglinfo->si_segmask) { 2849 upper_addr = B_TRUE; 2850 } else { 2851 lower_addr = B_TRUE; 2852 } 2853 /* 2854 * if the buffer lives both above and below the segment 2855 * boundary, or the current page is the page immediately 2856 * after the segment, we will use a copy/bounce buffer for 2857 * all pages > seg. 2858 */ 2859 if ((lower_addr && upper_addr) || 2860 (raddr == (sglinfo->si_segmask + 1))) { 2861 return (B_TRUE); 2862 } 2863 2864 size -= psize; 2865 } 2866 2867 return (B_FALSE); 2868 } 2869 2870 /* 2871 * rootnex_get_sgl() 2872 * Called in bind fastpath to get the sgl. Most of this will be replaced 2873 * with a call to the vm layer when vm2.0 comes around... 2874 */ 2875 static void 2876 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2877 rootnex_sglinfo_t *sglinfo) 2878 { 2879 ddi_dma_atyp_t buftype; 2880 rootnex_addr_t raddr; 2881 uint64_t last_page; 2882 uint64_t offset; 2883 uint64_t addrhi; 2884 uint64_t addrlo; 2885 uint64_t maxseg; 2886 page_t **pplist; 2887 uint64_t paddr; 2888 uint32_t psize; 2889 uint32_t size; 2890 caddr_t vaddr; 2891 uint_t pcnt; 2892 page_t *pp; 2893 uint_t cnt; 2894 2895 2896 /* shortcuts */ 2897 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2898 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2899 maxseg = sglinfo->si_max_cookie_size; 2900 buftype = dmar_object->dmao_type; 2901 addrhi = sglinfo->si_max_addr; 2902 addrlo = sglinfo->si_min_addr; 2903 size = dmar_object->dmao_size; 2904 2905 pcnt = 0; 2906 cnt = 0; 2907 2908 2909 /* 2910 * check to see if we need to use the copy buffer for pages over 2911 * the segment attr. 2912 */ 2913 sglinfo->si_bounce_on_seg = B_FALSE; 2914 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2915 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2916 dmar_object, sglinfo); 2917 } 2918 2919 /* 2920 * if we were passed down a linked list of pages, i.e. pointer to 2921 * page_t, use this to get our physical address and buf offset. 2922 */ 2923 if (buftype == DMA_OTYP_PAGES) { 2924 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2925 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2926 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2927 MMU_PAGEOFFSET; 2928 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2929 psize = MIN(size, (MMU_PAGESIZE - offset)); 2930 pp = pp->p_next; 2931 sglinfo->si_asp = NULL; 2932 2933 /* 2934 * We weren't passed down a linked list of pages, but if we were passed 2935 * down an array of pages, use this to get our physical address and buf 2936 * offset. 2937 */ 2938 } else if (pplist != NULL) { 2939 ASSERT((buftype == DMA_OTYP_VADDR) || 2940 (buftype == DMA_OTYP_BUFVADDR)); 2941 2942 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2943 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2944 if (sglinfo->si_asp == NULL) { 2945 sglinfo->si_asp = &kas; 2946 } 2947 2948 ASSERT(!PP_ISFREE(pplist[pcnt])); 2949 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2950 paddr += offset; 2951 psize = MIN(size, (MMU_PAGESIZE - offset)); 2952 pcnt++; 2953 2954 /* 2955 * All we have is a virtual address, we'll need to call into the VM 2956 * to get the physical address. 2957 */ 2958 } else { 2959 ASSERT((buftype == DMA_OTYP_VADDR) || 2960 (buftype == DMA_OTYP_BUFVADDR)); 2961 2962 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2963 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2964 if (sglinfo->si_asp == NULL) { 2965 sglinfo->si_asp = &kas; 2966 } 2967 2968 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2969 paddr += offset; 2970 psize = MIN(size, (MMU_PAGESIZE - offset)); 2971 vaddr += psize; 2972 } 2973 2974 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2975 2976 /* 2977 * Setup the first cookie with the physical address of the page and the 2978 * size of the page (which takes into account the initial offset into 2979 * the page. 2980 */ 2981 sgl[cnt].dmac_laddress = raddr; 2982 sgl[cnt].dmac_size = psize; 2983 sgl[cnt].dmac_type = 0; 2984 2985 /* 2986 * Save away the buffer offset into the page. We'll need this later in 2987 * the copy buffer code to help figure out the page index within the 2988 * buffer and the offset into the current page. 2989 */ 2990 sglinfo->si_buf_offset = offset; 2991 2992 /* 2993 * If we are using the copy buffer for anything over the segment 2994 * boundary, and this page is over the segment boundary. 2995 * OR 2996 * if the DMA engine can't reach the physical address. 2997 */ 2998 if (((sglinfo->si_bounce_on_seg) && 2999 ((raddr + psize) > sglinfo->si_segmask)) || 3000 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3001 /* 3002 * Increase how much copy buffer we use. We always increase by 3003 * pagesize so we don't have to worry about converting offsets. 3004 * Set a flag in the cookies dmac_type to indicate that it uses 3005 * the copy buffer. If this isn't the last cookie, go to the 3006 * next cookie (since we separate each page which uses the copy 3007 * buffer in case the copy buffer is not physically contiguous. 3008 */ 3009 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3010 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3011 if ((cnt + 1) < sglinfo->si_max_pages) { 3012 cnt++; 3013 sgl[cnt].dmac_laddress = 0; 3014 sgl[cnt].dmac_size = 0; 3015 sgl[cnt].dmac_type = 0; 3016 } 3017 } 3018 3019 /* 3020 * save this page's physical address so we can figure out if the next 3021 * page is physically contiguous. Keep decrementing size until we are 3022 * done with the buffer. 3023 */ 3024 last_page = raddr & MMU_PAGEMASK; 3025 size -= psize; 3026 3027 while (size > 0) { 3028 /* Get the size for this page (i.e. partial or full page) */ 3029 psize = MIN(size, MMU_PAGESIZE); 3030 3031 if (buftype == DMA_OTYP_PAGES) { 3032 /* get the paddr from the page_t */ 3033 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 3034 paddr = pfn_to_pa(pp->p_pagenum); 3035 pp = pp->p_next; 3036 } else if (pplist != NULL) { 3037 /* index into the array of page_t's to get the paddr */ 3038 ASSERT(!PP_ISFREE(pplist[pcnt])); 3039 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 3040 pcnt++; 3041 } else { 3042 /* call into the VM to get the paddr */ 3043 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 3044 vaddr)); 3045 vaddr += psize; 3046 } 3047 3048 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3049 3050 /* 3051 * If we are using the copy buffer for anything over the 3052 * segment boundary, and this page is over the segment 3053 * boundary. 3054 * OR 3055 * if the DMA engine can't reach the physical address. 3056 */ 3057 if (((sglinfo->si_bounce_on_seg) && 3058 ((raddr + psize) > sglinfo->si_segmask)) || 3059 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3060 3061 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3062 3063 /* 3064 * if there is something in the current cookie, go to 3065 * the next one. We only want one page in a cookie which 3066 * uses the copybuf since the copybuf doesn't have to 3067 * be physically contiguous. 3068 */ 3069 if (sgl[cnt].dmac_size != 0) { 3070 cnt++; 3071 } 3072 sgl[cnt].dmac_laddress = raddr; 3073 sgl[cnt].dmac_size = psize; 3074 #if defined(__amd64) 3075 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3076 #else 3077 /* 3078 * save the buf offset for 32-bit kernel. used in the 3079 * obsoleted interfaces. 3080 */ 3081 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 3082 (dmar_object->dmao_size - size); 3083 #endif 3084 /* if this isn't the last cookie, go to the next one */ 3085 if ((cnt + 1) < sglinfo->si_max_pages) { 3086 cnt++; 3087 sgl[cnt].dmac_laddress = 0; 3088 sgl[cnt].dmac_size = 0; 3089 sgl[cnt].dmac_type = 0; 3090 } 3091 3092 /* 3093 * this page didn't need the copy buffer, if it's not physically 3094 * contiguous, or it would put us over a segment boundary, or it 3095 * puts us over the max cookie size, or the current sgl doesn't 3096 * have anything in it. 3097 */ 3098 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3099 !(raddr & sglinfo->si_segmask) || 3100 ((sgl[cnt].dmac_size + psize) > maxseg) || 3101 (sgl[cnt].dmac_size == 0)) { 3102 /* 3103 * if we're not already in a new cookie, go to the next 3104 * cookie. 3105 */ 3106 if (sgl[cnt].dmac_size != 0) { 3107 cnt++; 3108 } 3109 3110 /* save the cookie information */ 3111 sgl[cnt].dmac_laddress = raddr; 3112 sgl[cnt].dmac_size = psize; 3113 #if defined(__amd64) 3114 sgl[cnt].dmac_type = 0; 3115 #else 3116 /* 3117 * save the buf offset for 32-bit kernel. used in the 3118 * obsoleted interfaces. 3119 */ 3120 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 3121 #endif 3122 3123 /* 3124 * this page didn't need the copy buffer, it is physically 3125 * contiguous with the last page, and it's <= the max cookie 3126 * size. 3127 */ 3128 } else { 3129 sgl[cnt].dmac_size += psize; 3130 3131 /* 3132 * if this exactly == the maximum cookie size, and 3133 * it isn't the last cookie, go to the next cookie. 3134 */ 3135 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3136 ((cnt + 1) < sglinfo->si_max_pages)) { 3137 cnt++; 3138 sgl[cnt].dmac_laddress = 0; 3139 sgl[cnt].dmac_size = 0; 3140 sgl[cnt].dmac_type = 0; 3141 } 3142 } 3143 3144 /* 3145 * save this page's physical address so we can figure out if the 3146 * next page is physically contiguous. Keep decrementing size 3147 * until we are done with the buffer. 3148 */ 3149 last_page = raddr; 3150 size -= psize; 3151 } 3152 3153 /* we're done, save away how many cookies the sgl has */ 3154 if (sgl[cnt].dmac_size == 0) { 3155 ASSERT(cnt < sglinfo->si_max_pages); 3156 sglinfo->si_sgl_size = cnt; 3157 } else { 3158 sglinfo->si_sgl_size = cnt + 1; 3159 } 3160 } 3161 3162 static void 3163 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3164 rootnex_sglinfo_t *sglinfo) 3165 { 3166 uint64_t offset; 3167 uint64_t maxseg; 3168 uint64_t dvaddr; 3169 struct dvmaseg *dvs; 3170 uint64_t paddr; 3171 uint32_t psize, ssize; 3172 uint32_t size; 3173 uint_t cnt; 3174 int physcontig; 3175 3176 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3177 3178 /* shortcuts */ 3179 maxseg = sglinfo->si_max_cookie_size; 3180 size = dmar_object->dmao_size; 3181 3182 cnt = 0; 3183 sglinfo->si_bounce_on_seg = B_FALSE; 3184 3185 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3186 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3187 ssize = dvs->dvs_len; 3188 paddr = dvs->dvs_start; 3189 paddr += offset; 3190 psize = MIN(ssize, (maxseg - offset)); 3191 dvaddr = paddr + psize; 3192 ssize -= psize; 3193 3194 sgl[cnt].dmac_laddress = paddr; 3195 sgl[cnt].dmac_size = psize; 3196 sgl[cnt].dmac_type = 0; 3197 3198 size -= psize; 3199 while (size > 0) { 3200 if (ssize == 0) { 3201 dvs++; 3202 ssize = dvs->dvs_len; 3203 dvaddr = dvs->dvs_start; 3204 physcontig = 0; 3205 } else 3206 physcontig = 1; 3207 3208 paddr = dvaddr; 3209 psize = MIN(ssize, maxseg); 3210 dvaddr += psize; 3211 ssize -= psize; 3212 3213 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3214 ((sgl[cnt].dmac_size + psize) > maxseg) || 3215 (sgl[cnt].dmac_size == 0)) { 3216 /* 3217 * if we're not already in a new cookie, go to the next 3218 * cookie. 3219 */ 3220 if (sgl[cnt].dmac_size != 0) { 3221 cnt++; 3222 } 3223 3224 /* save the cookie information */ 3225 sgl[cnt].dmac_laddress = paddr; 3226 sgl[cnt].dmac_size = psize; 3227 sgl[cnt].dmac_type = 0; 3228 } else { 3229 sgl[cnt].dmac_size += psize; 3230 3231 /* 3232 * if this exactly == the maximum cookie size, and 3233 * it isn't the last cookie, go to the next cookie. 3234 */ 3235 if (((sgl[cnt].dmac_size + psize) == maxseg) && 3236 ((cnt + 1) < sglinfo->si_max_pages)) { 3237 cnt++; 3238 sgl[cnt].dmac_laddress = 0; 3239 sgl[cnt].dmac_size = 0; 3240 sgl[cnt].dmac_type = 0; 3241 } 3242 } 3243 size -= psize; 3244 } 3245 3246 /* we're done, save away how many cookies the sgl has */ 3247 if (sgl[cnt].dmac_size == 0) { 3248 sglinfo->si_sgl_size = cnt; 3249 } else { 3250 sglinfo->si_sgl_size = cnt + 1; 3251 } 3252 } 3253 3254 /* 3255 * rootnex_bind_slowpath() 3256 * Call in the bind path if the calling driver can't use the sgl without 3257 * modifying it. We either need to use the copy buffer and/or we will end up 3258 * with a partial bind. 3259 */ 3260 static int 3261 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3262 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3263 { 3264 rootnex_sglinfo_t *sinfo; 3265 rootnex_window_t *window; 3266 ddi_dma_cookie_t *cookie; 3267 size_t copybuf_used; 3268 size_t dmac_size; 3269 boolean_t partial; 3270 off_t cur_offset; 3271 page_t *cur_pp; 3272 major_t mnum; 3273 int e; 3274 int i; 3275 3276 3277 sinfo = &dma->dp_sglinfo; 3278 copybuf_used = 0; 3279 partial = B_FALSE; 3280 3281 /* 3282 * If we're using the copybuf, set the copybuf state in dma struct. 3283 * Needs to be first since it sets the copy buffer size. 3284 */ 3285 if (sinfo->si_copybuf_req != 0) { 3286 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3287 if (e != DDI_SUCCESS) { 3288 return (e); 3289 } 3290 } else { 3291 dma->dp_copybuf_size = 0; 3292 } 3293 3294 /* 3295 * Figure out if we need to do a partial mapping. If so, figure out 3296 * if we need to trim the buffers when we munge the sgl. 3297 */ 3298 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3299 (dmao->dmao_size > dma->dp_maxxfer) || 3300 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3301 dma->dp_partial_required = B_TRUE; 3302 if (attr->dma_attr_granular != 1) { 3303 dma->dp_trim_required = B_TRUE; 3304 } 3305 } else { 3306 dma->dp_partial_required = B_FALSE; 3307 dma->dp_trim_required = B_FALSE; 3308 } 3309 3310 /* If we need to do a partial bind, make sure the driver supports it */ 3311 if (dma->dp_partial_required && 3312 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3313 3314 mnum = ddi_driver_major(dma->dp_dip); 3315 /* 3316 * patchable which allows us to print one warning per major 3317 * number. 3318 */ 3319 if ((rootnex_bind_warn) && 3320 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3321 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3322 cmn_err(CE_WARN, "!%s: coding error detected, the " 3323 "driver is using ddi_dma_attr(9S) incorrectly. " 3324 "There is a small risk of data corruption in " 3325 "particular with large I/Os. The driver should be " 3326 "replaced with a corrected version for proper " 3327 "system operation. To disable this warning, add " 3328 "'set rootnex:rootnex_bind_warn=0' to " 3329 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3330 } 3331 return (DDI_DMA_TOOBIG); 3332 } 3333 3334 /* 3335 * we might need multiple windows, setup state to handle them. In this 3336 * code path, we will have at least one window. 3337 */ 3338 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3339 if (e != DDI_SUCCESS) { 3340 rootnex_teardown_copybuf(dma); 3341 return (e); 3342 } 3343 3344 window = &dma->dp_window[0]; 3345 cookie = &dma->dp_cookies[0]; 3346 cur_offset = 0; 3347 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3348 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3349 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3350 } 3351 3352 /* loop though all the cookies we got back from get_sgl() */ 3353 for (i = 0; i < sinfo->si_sgl_size; i++) { 3354 /* 3355 * If we're using the copy buffer, check this cookie and setup 3356 * its associated copy buffer state. If this cookie uses the 3357 * copy buffer, make sure we sync this window during dma_sync. 3358 */ 3359 if (dma->dp_copybuf_size > 0) { 3360 rootnex_setup_cookie(dmao, dma, cookie, 3361 cur_offset, ©buf_used, &cur_pp); 3362 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3363 window->wd_dosync = B_TRUE; 3364 } 3365 } 3366 3367 /* 3368 * save away the cookie size, since it could be modified in 3369 * the windowing code. 3370 */ 3371 dmac_size = cookie->dmac_size; 3372 3373 /* if we went over max copybuf size */ 3374 if (dma->dp_copybuf_size && 3375 (copybuf_used > dma->dp_copybuf_size)) { 3376 partial = B_TRUE; 3377 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3378 cookie, cur_offset, ©buf_used); 3379 if (e != DDI_SUCCESS) { 3380 rootnex_teardown_copybuf(dma); 3381 rootnex_teardown_windows(dma); 3382 return (e); 3383 } 3384 3385 /* 3386 * if the coookie uses the copy buffer, make sure the 3387 * new window we just moved to is set to sync. 3388 */ 3389 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3390 window->wd_dosync = B_TRUE; 3391 } 3392 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3393 dma->dp_dip); 3394 3395 /* if the cookie cnt == max sgllen, move to the next window */ 3396 } else if (window->wd_cookie_cnt >= 3397 (unsigned)attr->dma_attr_sgllen) { 3398 partial = B_TRUE; 3399 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3400 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3401 cookie, attr, cur_offset); 3402 if (e != DDI_SUCCESS) { 3403 rootnex_teardown_copybuf(dma); 3404 rootnex_teardown_windows(dma); 3405 return (e); 3406 } 3407 3408 /* 3409 * if the coookie uses the copy buffer, make sure the 3410 * new window we just moved to is set to sync. 3411 */ 3412 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3413 window->wd_dosync = B_TRUE; 3414 } 3415 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3416 dma->dp_dip); 3417 3418 /* else if we will be over maxxfer */ 3419 } else if ((window->wd_size + dmac_size) > 3420 dma->dp_maxxfer) { 3421 partial = B_TRUE; 3422 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3423 cookie); 3424 if (e != DDI_SUCCESS) { 3425 rootnex_teardown_copybuf(dma); 3426 rootnex_teardown_windows(dma); 3427 return (e); 3428 } 3429 3430 /* 3431 * if the coookie uses the copy buffer, make sure the 3432 * new window we just moved to is set to sync. 3433 */ 3434 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3435 window->wd_dosync = B_TRUE; 3436 } 3437 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3438 dma->dp_dip); 3439 3440 /* else this cookie fits in the current window */ 3441 } else { 3442 window->wd_cookie_cnt++; 3443 window->wd_size += dmac_size; 3444 } 3445 3446 /* track our offset into the buffer, go to the next cookie */ 3447 ASSERT(dmac_size <= dmao->dmao_size); 3448 ASSERT(cookie->dmac_size <= dmac_size); 3449 cur_offset += dmac_size; 3450 cookie++; 3451 } 3452 3453 /* if we ended up with a zero sized window in the end, clean it up */ 3454 if (window->wd_size == 0) { 3455 hp->dmai_nwin--; 3456 window--; 3457 } 3458 3459 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3460 3461 if (!partial) { 3462 return (DDI_DMA_MAPPED); 3463 } 3464 3465 ASSERT(dma->dp_partial_required); 3466 return (DDI_DMA_PARTIAL_MAP); 3467 } 3468 3469 /* 3470 * rootnex_setup_copybuf() 3471 * Called in bind slowpath. Figures out if we're going to use the copy 3472 * buffer, and if we do, sets up the basic state to handle it. 3473 */ 3474 static int 3475 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3476 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3477 { 3478 rootnex_sglinfo_t *sinfo; 3479 ddi_dma_attr_t lattr; 3480 size_t max_copybuf; 3481 int cansleep; 3482 int e; 3483 #if !defined(__amd64) 3484 int vmflag; 3485 #endif 3486 3487 ASSERT(!dma->dp_dvma_used); 3488 3489 sinfo = &dma->dp_sglinfo; 3490 3491 /* read this first so it's consistent through the routine */ 3492 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3493 3494 /* We need to call into the rootnex on ddi_dma_sync() */ 3495 hp->dmai_rflags &= ~DMP_NOSYNC; 3496 3497 /* make sure the copybuf size <= the max size */ 3498 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3499 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3500 3501 #if !defined(__amd64) 3502 /* 3503 * if we don't have kva space to copy to/from, allocate the KVA space 3504 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3505 * the 64-bit kernel. 3506 */ 3507 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3508 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3509 3510 /* convert the sleep flags */ 3511 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3512 vmflag = VM_SLEEP; 3513 } else { 3514 vmflag = VM_NOSLEEP; 3515 } 3516 3517 /* allocate Kernel VA space that we can bcopy to/from */ 3518 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3519 vmflag); 3520 if (dma->dp_kva == NULL) { 3521 return (DDI_DMA_NORESOURCES); 3522 } 3523 } 3524 #endif 3525 3526 /* convert the sleep flags */ 3527 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3528 cansleep = 1; 3529 } else { 3530 cansleep = 0; 3531 } 3532 3533 /* 3534 * Allocate the actual copy buffer. This needs to fit within the DMA 3535 * engine limits, so we can't use kmem_alloc... We don't need 3536 * contiguous memory (sgllen) since we will be forcing windows on 3537 * sgllen anyway. 3538 */ 3539 lattr = *attr; 3540 lattr.dma_attr_align = MMU_PAGESIZE; 3541 lattr.dma_attr_sgllen = -1; /* no limit */ 3542 /* 3543 * if we're using the copy buffer because of seg, use that for our 3544 * upper address limit. 3545 */ 3546 if (sinfo->si_bounce_on_seg) { 3547 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3548 } 3549 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3550 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3551 if (e != DDI_SUCCESS) { 3552 #if !defined(__amd64) 3553 if (dma->dp_kva != NULL) { 3554 vmem_free(heap_arena, dma->dp_kva, 3555 dma->dp_copybuf_size); 3556 } 3557 #endif 3558 return (DDI_DMA_NORESOURCES); 3559 } 3560 3561 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3562 size_t, dma->dp_copybuf_size); 3563 3564 return (DDI_SUCCESS); 3565 } 3566 3567 3568 /* 3569 * rootnex_setup_windows() 3570 * Called in bind slowpath to setup the window state. We always have windows 3571 * in the slowpath. Even if the window count = 1. 3572 */ 3573 static int 3574 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3575 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3576 { 3577 rootnex_window_t *windowp; 3578 rootnex_sglinfo_t *sinfo; 3579 size_t copy_state_size; 3580 size_t win_state_size; 3581 size_t state_available; 3582 size_t space_needed; 3583 uint_t copybuf_win; 3584 uint_t maxxfer_win; 3585 size_t space_used; 3586 uint_t sglwin; 3587 3588 3589 sinfo = &dma->dp_sglinfo; 3590 3591 dma->dp_current_win = 0; 3592 hp->dmai_nwin = 0; 3593 3594 /* If we don't need to do a partial, we only have one window */ 3595 if (!dma->dp_partial_required) { 3596 dma->dp_max_win = 1; 3597 3598 /* 3599 * we need multiple windows, need to figure out the worse case number 3600 * of windows. 3601 */ 3602 } else { 3603 /* 3604 * if we need windows because we need more copy buffer that 3605 * we allow, the worse case number of windows we could need 3606 * here would be (copybuf space required / copybuf space that 3607 * we have) plus one for remainder, and plus 2 to handle the 3608 * extra pages on the trim for the first and last pages of the 3609 * buffer (a page is the minimum window size so under the right 3610 * attr settings, you could have a window for each page). 3611 * The last page will only be hit here if the size is not a 3612 * multiple of the granularity (which theoretically shouldn't 3613 * be the case but never has been enforced, so we could have 3614 * broken things without it). 3615 */ 3616 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3617 ASSERT(dma->dp_copybuf_size > 0); 3618 copybuf_win = (sinfo->si_copybuf_req / 3619 dma->dp_copybuf_size) + 1 + 2; 3620 } else { 3621 copybuf_win = 0; 3622 } 3623 3624 /* 3625 * if we need windows because we have more cookies than the H/W 3626 * can handle, the number of windows we would need here would 3627 * be (cookie count / cookies count H/W supports minus 1[for 3628 * trim]) plus one for remainder. 3629 */ 3630 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3631 sglwin = (sinfo->si_sgl_size / 3632 (attr->dma_attr_sgllen - 1)) + 1; 3633 } else { 3634 sglwin = 0; 3635 } 3636 3637 /* 3638 * if we need windows because we're binding more memory than the 3639 * H/W can transfer at once, the number of windows we would need 3640 * here would be (xfer count / max xfer H/W supports) plus one 3641 * for remainder, and plus 2 to handle the extra pages on the 3642 * trim (see above comment about trim) 3643 */ 3644 if (dmao->dmao_size > dma->dp_maxxfer) { 3645 maxxfer_win = (dmao->dmao_size / 3646 dma->dp_maxxfer) + 1 + 2; 3647 } else { 3648 maxxfer_win = 0; 3649 } 3650 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3651 ASSERT(dma->dp_max_win > 0); 3652 } 3653 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3654 3655 /* 3656 * Get space for window and potential copy buffer state. Before we 3657 * go and allocate memory, see if we can get away with using what's 3658 * left in the pre-allocted state or the dynamically allocated sgl. 3659 */ 3660 space_used = (uintptr_t)(sinfo->si_sgl_size * 3661 sizeof (ddi_dma_cookie_t)); 3662 3663 /* if we dynamically allocated space for the cookies */ 3664 if (dma->dp_need_to_free_cookie) { 3665 /* if we have more space in the pre-allocted buffer, use it */ 3666 ASSERT(space_used <= dma->dp_cookie_size); 3667 if ((dma->dp_cookie_size - space_used) <= 3668 rootnex_state->r_prealloc_size) { 3669 state_available = rootnex_state->r_prealloc_size; 3670 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3671 3672 /* 3673 * else, we have more free space in the dynamically allocated 3674 * buffer, i.e. the buffer wasn't worse case fragmented so we 3675 * didn't need a lot of cookies. 3676 */ 3677 } else { 3678 state_available = dma->dp_cookie_size - space_used; 3679 windowp = (rootnex_window_t *) 3680 &dma->dp_cookies[sinfo->si_sgl_size]; 3681 } 3682 3683 /* we used the pre-alloced buffer */ 3684 } else { 3685 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3686 state_available = rootnex_state->r_prealloc_size - space_used; 3687 windowp = (rootnex_window_t *) 3688 &dma->dp_cookies[sinfo->si_sgl_size]; 3689 } 3690 3691 /* 3692 * figure out how much state we need to track the copy buffer. Add an 3693 * addition 8 bytes for pointer alignemnt later. 3694 */ 3695 if (dma->dp_copybuf_size > 0) { 3696 copy_state_size = sinfo->si_max_pages * 3697 sizeof (rootnex_pgmap_t); 3698 } else { 3699 copy_state_size = 0; 3700 } 3701 /* add an additional 8 bytes for pointer alignment */ 3702 space_needed = win_state_size + copy_state_size + 0x8; 3703 3704 /* if we have enough space already, use it */ 3705 if (state_available >= space_needed) { 3706 dma->dp_window = windowp; 3707 dma->dp_need_to_free_window = B_FALSE; 3708 3709 /* not enough space, need to allocate more. */ 3710 } else { 3711 dma->dp_window = kmem_alloc(space_needed, kmflag); 3712 if (dma->dp_window == NULL) { 3713 return (DDI_DMA_NORESOURCES); 3714 } 3715 dma->dp_need_to_free_window = B_TRUE; 3716 dma->dp_window_size = space_needed; 3717 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3718 dma->dp_dip, size_t, space_needed); 3719 } 3720 3721 /* 3722 * we allocate copy buffer state and window state at the same time. 3723 * setup our copy buffer state pointers. Make sure it's aligned. 3724 */ 3725 if (dma->dp_copybuf_size > 0) { 3726 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3727 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3728 3729 #if !defined(__amd64) 3730 /* 3731 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3732 * false/NULL. Should be quicker to bzero vs loop and set. 3733 */ 3734 bzero(dma->dp_pgmap, copy_state_size); 3735 #endif 3736 } else { 3737 dma->dp_pgmap = NULL; 3738 } 3739 3740 return (DDI_SUCCESS); 3741 } 3742 3743 3744 /* 3745 * rootnex_teardown_copybuf() 3746 * cleans up after rootnex_setup_copybuf() 3747 */ 3748 static void 3749 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3750 { 3751 #if !defined(__amd64) 3752 int i; 3753 3754 /* 3755 * if we allocated kernel heap VMEM space, go through all the pages and 3756 * map out any of the ones that we're mapped into the kernel heap VMEM 3757 * arena. Then free the VMEM space. 3758 */ 3759 if (dma->dp_kva != NULL) { 3760 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3761 if (dma->dp_pgmap[i].pm_mapped) { 3762 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3763 MMU_PAGESIZE, HAT_UNLOAD); 3764 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3765 } 3766 } 3767 3768 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3769 } 3770 3771 #endif 3772 3773 /* if we allocated a copy buffer, free it */ 3774 if (dma->dp_cbaddr != NULL) { 3775 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3776 } 3777 } 3778 3779 3780 /* 3781 * rootnex_teardown_windows() 3782 * cleans up after rootnex_setup_windows() 3783 */ 3784 static void 3785 rootnex_teardown_windows(rootnex_dma_t *dma) 3786 { 3787 /* 3788 * if we had to allocate window state on the last bind (because we 3789 * didn't have enough pre-allocated space in the handle), free it. 3790 */ 3791 if (dma->dp_need_to_free_window) { 3792 kmem_free(dma->dp_window, dma->dp_window_size); 3793 } 3794 } 3795 3796 3797 /* 3798 * rootnex_init_win() 3799 * Called in bind slow path during creation of a new window. Initializes 3800 * window state to default values. 3801 */ 3802 /*ARGSUSED*/ 3803 static void 3804 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3805 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3806 { 3807 hp->dmai_nwin++; 3808 window->wd_dosync = B_FALSE; 3809 window->wd_offset = cur_offset; 3810 window->wd_size = 0; 3811 window->wd_first_cookie = cookie; 3812 window->wd_cookie_cnt = 0; 3813 window->wd_trim.tr_trim_first = B_FALSE; 3814 window->wd_trim.tr_trim_last = B_FALSE; 3815 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3816 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3817 #if !defined(__amd64) 3818 window->wd_remap_copybuf = dma->dp_cb_remaping; 3819 #endif 3820 } 3821 3822 3823 /* 3824 * rootnex_setup_cookie() 3825 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3826 * the sgl uses the copy buffer, we need to go through each cookie, figure 3827 * out if it uses the copy buffer, and if it does, save away everything we'll 3828 * need during sync. 3829 */ 3830 static void 3831 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3832 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3833 page_t **cur_pp) 3834 { 3835 boolean_t copybuf_sz_power_2; 3836 rootnex_sglinfo_t *sinfo; 3837 paddr_t paddr; 3838 uint_t pidx; 3839 uint_t pcnt; 3840 off_t poff; 3841 #if defined(__amd64) 3842 pfn_t pfn; 3843 #else 3844 page_t **pplist; 3845 #endif 3846 3847 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3848 3849 sinfo = &dma->dp_sglinfo; 3850 3851 /* 3852 * Calculate the page index relative to the start of the buffer. The 3853 * index to the current page for our buffer is the offset into the 3854 * first page of the buffer plus our current offset into the buffer 3855 * itself, shifted of course... 3856 */ 3857 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3858 ASSERT(pidx < sinfo->si_max_pages); 3859 3860 /* if this cookie uses the copy buffer */ 3861 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3862 /* 3863 * NOTE: we know that since this cookie uses the copy buffer, it 3864 * is <= MMU_PAGESIZE. 3865 */ 3866 3867 /* 3868 * get the offset into the page. For the 64-bit kernel, get the 3869 * pfn which we'll use with seg kpm. 3870 */ 3871 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3872 #if defined(__amd64) 3873 /* mfn_to_pfn() is a NOP on i86pc */ 3874 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3875 #endif /* __amd64 */ 3876 3877 /* figure out if the copybuf size is a power of 2 */ 3878 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3879 copybuf_sz_power_2 = B_FALSE; 3880 } else { 3881 copybuf_sz_power_2 = B_TRUE; 3882 } 3883 3884 /* This page uses the copy buffer */ 3885 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3886 3887 /* 3888 * save the copy buffer KVA that we'll use with this page. 3889 * if we still fit within the copybuf, it's a simple add. 3890 * otherwise, we need to wrap over using & or % accordingly. 3891 */ 3892 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3893 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3894 *copybuf_used; 3895 } else { 3896 if (copybuf_sz_power_2) { 3897 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3898 (uintptr_t)dma->dp_cbaddr + 3899 (*copybuf_used & 3900 (dma->dp_copybuf_size - 1))); 3901 } else { 3902 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3903 (uintptr_t)dma->dp_cbaddr + 3904 (*copybuf_used % dma->dp_copybuf_size)); 3905 } 3906 } 3907 3908 /* 3909 * over write the cookie physical address with the address of 3910 * the physical address of the copy buffer page that we will 3911 * use. 3912 */ 3913 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3914 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3915 3916 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3917 3918 /* if we have a kernel VA, it's easy, just save that address */ 3919 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3920 (sinfo->si_asp == &kas)) { 3921 /* 3922 * save away the page aligned virtual address of the 3923 * driver buffer. Offsets are handled in the sync code. 3924 */ 3925 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3926 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3927 & MMU_PAGEMASK); 3928 #if !defined(__amd64) 3929 /* 3930 * we didn't need to, and will never need to map this 3931 * page. 3932 */ 3933 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3934 #endif 3935 3936 /* we don't have a kernel VA. We need one for the bcopy. */ 3937 } else { 3938 #if defined(__amd64) 3939 /* 3940 * for the 64-bit kernel, it's easy. We use seg kpm to 3941 * get a Kernel VA for the corresponding pfn. 3942 */ 3943 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3944 #else 3945 /* 3946 * for the 32-bit kernel, this is a pain. First we'll 3947 * save away the page_t or user VA for this page. This 3948 * is needed in rootnex_dma_win() when we switch to a 3949 * new window which requires us to re-map the copy 3950 * buffer. 3951 */ 3952 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3953 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3954 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3955 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3956 } else if (pplist != NULL) { 3957 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3958 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3959 } else { 3960 dma->dp_pgmap[pidx].pm_pp = NULL; 3961 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3962 (((uintptr_t) 3963 dmar_object->dmao_obj.virt_obj.v_addr + 3964 cur_offset) & MMU_PAGEMASK); 3965 } 3966 3967 /* 3968 * save away the page aligned virtual address which was 3969 * allocated from the kernel heap arena (taking into 3970 * account if we need more copy buffer than we alloced 3971 * and use multiple windows to handle this, i.e. &,%). 3972 * NOTE: there isn't and physical memory backing up this 3973 * virtual address space currently. 3974 */ 3975 if ((*copybuf_used + MMU_PAGESIZE) <= 3976 dma->dp_copybuf_size) { 3977 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3978 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3979 MMU_PAGEMASK); 3980 } else { 3981 if (copybuf_sz_power_2) { 3982 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3983 (((uintptr_t)dma->dp_kva + 3984 (*copybuf_used & 3985 (dma->dp_copybuf_size - 1))) & 3986 MMU_PAGEMASK); 3987 } else { 3988 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3989 (((uintptr_t)dma->dp_kva + 3990 (*copybuf_used % 3991 dma->dp_copybuf_size)) & 3992 MMU_PAGEMASK); 3993 } 3994 } 3995 3996 /* 3997 * if we haven't used up the available copy buffer yet, 3998 * map the kva to the physical page. 3999 */ 4000 if (!dma->dp_cb_remaping && ((*copybuf_used + 4001 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 4002 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 4003 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 4004 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 4005 dma->dp_pgmap[pidx].pm_kaddr); 4006 } else { 4007 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 4008 sinfo->si_asp, 4009 dma->dp_pgmap[pidx].pm_kaddr); 4010 } 4011 4012 /* 4013 * we've used up the available copy buffer, this page 4014 * will have to be mapped during rootnex_dma_win() when 4015 * we switch to a new window which requires a re-map 4016 * the copy buffer. (32-bit kernel only) 4017 */ 4018 } else { 4019 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4020 } 4021 #endif 4022 /* go to the next page_t */ 4023 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4024 *cur_pp = (*cur_pp)->p_next; 4025 } 4026 } 4027 4028 /* add to the copy buffer count */ 4029 *copybuf_used += MMU_PAGESIZE; 4030 4031 /* 4032 * This cookie doesn't use the copy buffer. Walk through the pages this 4033 * cookie occupies to reflect this. 4034 */ 4035 } else { 4036 /* 4037 * figure out how many pages the cookie occupies. We need to 4038 * use the original page offset of the buffer and the cookies 4039 * offset in the buffer to do this. 4040 */ 4041 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 4042 pcnt = mmu_btopr(cookie->dmac_size + poff); 4043 4044 while (pcnt > 0) { 4045 #if !defined(__amd64) 4046 /* 4047 * the 32-bit kernel doesn't have seg kpm, so we need 4048 * to map in the driver buffer (if it didn't come down 4049 * with a kernel VA) on the fly. Since this page doesn't 4050 * use the copy buffer, it's not, or will it ever, have 4051 * to be mapped in. 4052 */ 4053 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4054 #endif 4055 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 4056 4057 /* 4058 * we need to update pidx and cur_pp or we'll loose 4059 * track of where we are. 4060 */ 4061 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 4062 *cur_pp = (*cur_pp)->p_next; 4063 } 4064 pidx++; 4065 pcnt--; 4066 } 4067 } 4068 } 4069 4070 4071 /* 4072 * rootnex_sgllen_window_boundary() 4073 * Called in the bind slow path when the next cookie causes us to exceed (in 4074 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 4075 * length supported by the DMA H/W. 4076 */ 4077 static int 4078 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4079 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 4080 off_t cur_offset) 4081 { 4082 off_t new_offset; 4083 size_t trim_sz; 4084 off_t coffset; 4085 4086 4087 /* 4088 * if we know we'll never have to trim, it's pretty easy. Just move to 4089 * the next window and init it. We're done. 4090 */ 4091 if (!dma->dp_trim_required) { 4092 (*windowp)++; 4093 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4094 (*windowp)->wd_cookie_cnt++; 4095 (*windowp)->wd_size = cookie->dmac_size; 4096 return (DDI_SUCCESS); 4097 } 4098 4099 /* figure out how much we need to trim from the window */ 4100 ASSERT(attr->dma_attr_granular != 0); 4101 if (dma->dp_granularity_power_2) { 4102 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 4103 } else { 4104 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 4105 } 4106 4107 /* The window's a whole multiple of granularity. We're done */ 4108 if (trim_sz == 0) { 4109 (*windowp)++; 4110 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4111 (*windowp)->wd_cookie_cnt++; 4112 (*windowp)->wd_size = cookie->dmac_size; 4113 return (DDI_SUCCESS); 4114 } 4115 4116 /* 4117 * The window's not a whole multiple of granularity, since we know this 4118 * is due to the sgllen, we need to go back to the last cookie and trim 4119 * that one, add the left over part of the old cookie into the new 4120 * window, and then add in the new cookie into the new window. 4121 */ 4122 4123 /* 4124 * make sure the driver isn't making us do something bad... Trimming and 4125 * sgllen == 1 don't go together. 4126 */ 4127 if (attr->dma_attr_sgllen == 1) { 4128 return (DDI_DMA_NOMAPPING); 4129 } 4130 4131 /* 4132 * first, setup the current window to account for the trim. Need to go 4133 * back to the last cookie for this. 4134 */ 4135 cookie--; 4136 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4137 (*windowp)->wd_trim.tr_last_cookie = cookie; 4138 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4139 ASSERT(cookie->dmac_size > trim_sz); 4140 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4141 (*windowp)->wd_size -= trim_sz; 4142 4143 /* save the buffer offsets for the next window */ 4144 coffset = cookie->dmac_size - trim_sz; 4145 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4146 4147 /* 4148 * set this now in case this is the first window. all other cases are 4149 * set in dma_win() 4150 */ 4151 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4152 4153 /* 4154 * initialize the next window using what's left over in the previous 4155 * cookie. 4156 */ 4157 (*windowp)++; 4158 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4159 (*windowp)->wd_cookie_cnt++; 4160 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4161 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4162 (*windowp)->wd_trim.tr_first_size = trim_sz; 4163 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4164 (*windowp)->wd_dosync = B_TRUE; 4165 } 4166 4167 /* 4168 * now go back to the current cookie and add it to the new window. set 4169 * the new window size to the what was left over from the previous 4170 * cookie and what's in the current cookie. 4171 */ 4172 cookie++; 4173 (*windowp)->wd_cookie_cnt++; 4174 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4175 4176 /* 4177 * trim plus the next cookie could put us over maxxfer (a cookie can be 4178 * a max size of maxxfer). Handle that case. 4179 */ 4180 if ((*windowp)->wd_size > dma->dp_maxxfer) { 4181 /* 4182 * maxxfer is already a whole multiple of granularity, and this 4183 * trim will be <= the previous trim (since a cookie can't be 4184 * larger than maxxfer). Make things simple here. 4185 */ 4186 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 4187 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4188 (*windowp)->wd_trim.tr_last_cookie = cookie; 4189 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4190 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4191 (*windowp)->wd_size -= trim_sz; 4192 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 4193 4194 /* save the buffer offsets for the next window */ 4195 coffset = cookie->dmac_size - trim_sz; 4196 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4197 4198 /* setup the next window */ 4199 (*windowp)++; 4200 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4201 (*windowp)->wd_cookie_cnt++; 4202 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4203 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4204 coffset; 4205 (*windowp)->wd_trim.tr_first_size = trim_sz; 4206 } 4207 4208 return (DDI_SUCCESS); 4209 } 4210 4211 4212 /* 4213 * rootnex_copybuf_window_boundary() 4214 * Called in bind slowpath when we get to a window boundary because we used 4215 * up all the copy buffer that we have. 4216 */ 4217 static int 4218 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4219 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 4220 size_t *copybuf_used) 4221 { 4222 rootnex_sglinfo_t *sinfo; 4223 off_t new_offset; 4224 size_t trim_sz; 4225 paddr_t paddr; 4226 off_t coffset; 4227 uint_t pidx; 4228 off_t poff; 4229 4230 4231 sinfo = &dma->dp_sglinfo; 4232 4233 /* 4234 * the copy buffer should be a whole multiple of page size. We know that 4235 * this cookie is <= MMU_PAGESIZE. 4236 */ 4237 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4238 4239 /* 4240 * from now on, all new windows in this bind need to be re-mapped during 4241 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 4242 * space... 4243 */ 4244 #if !defined(__amd64) 4245 dma->dp_cb_remaping = B_TRUE; 4246 #endif 4247 4248 /* reset copybuf used */ 4249 *copybuf_used = 0; 4250 4251 /* 4252 * if we don't have to trim (since granularity is set to 1), go to the 4253 * next window and add the current cookie to it. We know the current 4254 * cookie uses the copy buffer since we're in this code path. 4255 */ 4256 if (!dma->dp_trim_required) { 4257 (*windowp)++; 4258 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4259 4260 /* Add this cookie to the new window */ 4261 (*windowp)->wd_cookie_cnt++; 4262 (*windowp)->wd_size += cookie->dmac_size; 4263 *copybuf_used += MMU_PAGESIZE; 4264 return (DDI_SUCCESS); 4265 } 4266 4267 /* 4268 * *** may need to trim, figure it out. 4269 */ 4270 4271 /* figure out how much we need to trim from the window */ 4272 if (dma->dp_granularity_power_2) { 4273 trim_sz = (*windowp)->wd_size & 4274 (hp->dmai_attr.dma_attr_granular - 1); 4275 } else { 4276 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4277 } 4278 4279 /* 4280 * if the window's a whole multiple of granularity, go to the next 4281 * window, init it, then add in the current cookie. We know the current 4282 * cookie uses the copy buffer since we're in this code path. 4283 */ 4284 if (trim_sz == 0) { 4285 (*windowp)++; 4286 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4287 4288 /* Add this cookie to the new window */ 4289 (*windowp)->wd_cookie_cnt++; 4290 (*windowp)->wd_size += cookie->dmac_size; 4291 *copybuf_used += MMU_PAGESIZE; 4292 return (DDI_SUCCESS); 4293 } 4294 4295 /* 4296 * *** We figured it out, we definitly need to trim 4297 */ 4298 4299 /* 4300 * make sure the driver isn't making us do something bad... 4301 * Trimming and sgllen == 1 don't go together. 4302 */ 4303 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4304 return (DDI_DMA_NOMAPPING); 4305 } 4306 4307 /* 4308 * first, setup the current window to account for the trim. Need to go 4309 * back to the last cookie for this. Some of the last cookie will be in 4310 * the current window, and some of the last cookie will be in the new 4311 * window. All of the current cookie will be in the new window. 4312 */ 4313 cookie--; 4314 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4315 (*windowp)->wd_trim.tr_last_cookie = cookie; 4316 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4317 ASSERT(cookie->dmac_size > trim_sz); 4318 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4319 (*windowp)->wd_size -= trim_sz; 4320 4321 /* 4322 * we're trimming the last cookie (not the current cookie). So that 4323 * last cookie may have or may not have been using the copy buffer ( 4324 * we know the cookie passed in uses the copy buffer since we're in 4325 * this code path). 4326 * 4327 * If the last cookie doesn't use the copy buffer, nothing special to 4328 * do. However, if it does uses the copy buffer, it will be both the 4329 * last page in the current window and the first page in the next 4330 * window. Since we are reusing the copy buffer (and KVA space on the 4331 * 32-bit kernel), this page will use the end of the copy buffer in the 4332 * current window, and the start of the copy buffer in the next window. 4333 * Track that info... The cookie physical address was already set to 4334 * the copy buffer physical address in setup_cookie.. 4335 */ 4336 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4337 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4338 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4339 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4340 (*windowp)->wd_trim.tr_last_pidx = pidx; 4341 (*windowp)->wd_trim.tr_last_cbaddr = 4342 dma->dp_pgmap[pidx].pm_cbaddr; 4343 #if !defined(__amd64) 4344 (*windowp)->wd_trim.tr_last_kaddr = 4345 dma->dp_pgmap[pidx].pm_kaddr; 4346 #endif 4347 } 4348 4349 /* save the buffer offsets for the next window */ 4350 coffset = cookie->dmac_size - trim_sz; 4351 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4352 4353 /* 4354 * set this now in case this is the first window. all other cases are 4355 * set in dma_win() 4356 */ 4357 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4358 4359 /* 4360 * initialize the next window using what's left over in the previous 4361 * cookie. 4362 */ 4363 (*windowp)++; 4364 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4365 (*windowp)->wd_cookie_cnt++; 4366 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4367 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4368 (*windowp)->wd_trim.tr_first_size = trim_sz; 4369 4370 /* 4371 * again, we're tracking if the last cookie uses the copy buffer. 4372 * read the comment above for more info on why we need to track 4373 * additional state. 4374 * 4375 * For the first cookie in the new window, we need reset the physical 4376 * address to DMA into to the start of the copy buffer plus any 4377 * initial page offset which may be present. 4378 */ 4379 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4380 (*windowp)->wd_dosync = B_TRUE; 4381 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4382 (*windowp)->wd_trim.tr_first_pidx = pidx; 4383 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4384 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4385 4386 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4387 poff; 4388 (*windowp)->wd_trim.tr_first_paddr = 4389 ROOTNEX_PADDR_TO_RBASE(paddr); 4390 4391 #if !defined(__amd64) 4392 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4393 #endif 4394 /* account for the cookie copybuf usage in the new window */ 4395 *copybuf_used += MMU_PAGESIZE; 4396 4397 /* 4398 * every piece of code has to have a hack, and here is this 4399 * ones :-) 4400 * 4401 * There is a complex interaction between setup_cookie and the 4402 * copybuf window boundary. The complexity had to be in either 4403 * the maxxfer window, or the copybuf window, and I chose the 4404 * copybuf code. 4405 * 4406 * So in this code path, we have taken the last cookie, 4407 * virtually broken it in half due to the trim, and it happens 4408 * to use the copybuf which further complicates life. At the 4409 * same time, we have already setup the current cookie, which 4410 * is now wrong. More background info: the current cookie uses 4411 * the copybuf, so it is only a page long max. So we need to 4412 * fix the current cookies copy buffer address, physical 4413 * address, and kva for the 32-bit kernel. We due this by 4414 * bumping them by page size (of course, we can't due this on 4415 * the physical address since the copy buffer may not be 4416 * physically contiguous). 4417 */ 4418 cookie++; 4419 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4420 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4421 4422 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4423 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4424 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4425 4426 #if !defined(__amd64) 4427 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4428 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4429 #endif 4430 } else { 4431 /* go back to the current cookie */ 4432 cookie++; 4433 } 4434 4435 /* 4436 * add the current cookie to the new window. set the new window size to 4437 * the what was left over from the previous cookie and what's in the 4438 * current cookie. 4439 */ 4440 (*windowp)->wd_cookie_cnt++; 4441 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4442 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4443 4444 /* 4445 * we know that the cookie passed in always uses the copy buffer. We 4446 * wouldn't be here if it didn't. 4447 */ 4448 *copybuf_used += MMU_PAGESIZE; 4449 4450 return (DDI_SUCCESS); 4451 } 4452 4453 4454 /* 4455 * rootnex_maxxfer_window_boundary() 4456 * Called in bind slowpath when we get to a window boundary because we will 4457 * go over maxxfer. 4458 */ 4459 static int 4460 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4461 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4462 { 4463 size_t dmac_size; 4464 off_t new_offset; 4465 size_t trim_sz; 4466 off_t coffset; 4467 4468 4469 /* 4470 * calculate how much we have to trim off of the current cookie to equal 4471 * maxxfer. We don't have to account for granularity here since our 4472 * maxxfer already takes that into account. 4473 */ 4474 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4475 ASSERT(trim_sz <= cookie->dmac_size); 4476 ASSERT(trim_sz <= dma->dp_maxxfer); 4477 4478 /* save cookie size since we need it later and we might change it */ 4479 dmac_size = cookie->dmac_size; 4480 4481 /* 4482 * if we're not trimming the entire cookie, setup the current window to 4483 * account for the trim. 4484 */ 4485 if (trim_sz < cookie->dmac_size) { 4486 (*windowp)->wd_cookie_cnt++; 4487 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4488 (*windowp)->wd_trim.tr_last_cookie = cookie; 4489 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4490 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4491 (*windowp)->wd_size = dma->dp_maxxfer; 4492 4493 /* 4494 * set the adjusted cookie size now in case this is the first 4495 * window. All other windows are taken care of in get win 4496 */ 4497 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4498 } 4499 4500 /* 4501 * coffset is the current offset within the cookie, new_offset is the 4502 * current offset with the entire buffer. 4503 */ 4504 coffset = dmac_size - trim_sz; 4505 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4506 4507 /* initialize the next window */ 4508 (*windowp)++; 4509 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4510 (*windowp)->wd_cookie_cnt++; 4511 (*windowp)->wd_size = trim_sz; 4512 if (trim_sz < dmac_size) { 4513 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4514 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4515 coffset; 4516 (*windowp)->wd_trim.tr_first_size = trim_sz; 4517 } 4518 4519 return (DDI_SUCCESS); 4520 } 4521 4522 4523 /*ARGSUSED*/ 4524 static int 4525 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4526 off_t off, size_t len, uint_t cache_flags) 4527 { 4528 rootnex_sglinfo_t *sinfo; 4529 rootnex_pgmap_t *cbpage; 4530 rootnex_window_t *win; 4531 ddi_dma_impl_t *hp; 4532 rootnex_dma_t *dma; 4533 caddr_t fromaddr; 4534 caddr_t toaddr; 4535 uint_t psize; 4536 off_t offset; 4537 uint_t pidx; 4538 size_t size; 4539 off_t poff; 4540 int e; 4541 4542 4543 hp = (ddi_dma_impl_t *)handle; 4544 dma = (rootnex_dma_t *)hp->dmai_private; 4545 sinfo = &dma->dp_sglinfo; 4546 4547 /* 4548 * if we don't have any windows, we don't need to sync. A copybuf 4549 * will cause us to have at least one window. 4550 */ 4551 if (dma->dp_window == NULL) { 4552 return (DDI_SUCCESS); 4553 } 4554 4555 /* This window may not need to be sync'd */ 4556 win = &dma->dp_window[dma->dp_current_win]; 4557 if (!win->wd_dosync) { 4558 return (DDI_SUCCESS); 4559 } 4560 4561 /* handle off and len special cases */ 4562 if ((off == 0) || (rootnex_sync_ignore_params)) { 4563 offset = win->wd_offset; 4564 } else { 4565 offset = off; 4566 } 4567 if ((len == 0) || (rootnex_sync_ignore_params)) { 4568 size = win->wd_size; 4569 } else { 4570 size = len; 4571 } 4572 4573 /* check the sync args to make sure they make a little sense */ 4574 if (rootnex_sync_check_parms) { 4575 e = rootnex_valid_sync_parms(hp, win, offset, size, 4576 cache_flags); 4577 if (e != DDI_SUCCESS) { 4578 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4579 return (DDI_FAILURE); 4580 } 4581 } 4582 4583 /* 4584 * special case the first page to handle the offset into the page. The 4585 * offset to the current page for our buffer is the offset into the 4586 * first page of the buffer plus our current offset into the buffer 4587 * itself, masked of course. 4588 */ 4589 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4590 psize = MIN((MMU_PAGESIZE - poff), size); 4591 4592 /* go through all the pages that we want to sync */ 4593 while (size > 0) { 4594 /* 4595 * Calculate the page index relative to the start of the buffer. 4596 * The index to the current page for our buffer is the offset 4597 * into the first page of the buffer plus our current offset 4598 * into the buffer itself, shifted of course... 4599 */ 4600 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4601 ASSERT(pidx < sinfo->si_max_pages); 4602 4603 /* 4604 * if this page uses the copy buffer, we need to sync it, 4605 * otherwise, go on to the next page. 4606 */ 4607 cbpage = &dma->dp_pgmap[pidx]; 4608 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4609 (cbpage->pm_uses_copybuf == B_FALSE)); 4610 if (cbpage->pm_uses_copybuf) { 4611 /* cbaddr and kaddr should be page aligned */ 4612 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4613 MMU_PAGEOFFSET) == 0); 4614 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4615 MMU_PAGEOFFSET) == 0); 4616 4617 /* 4618 * if we're copying for the device, we are going to 4619 * copy from the drivers buffer and to the rootnex 4620 * allocated copy buffer. 4621 */ 4622 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4623 fromaddr = cbpage->pm_kaddr + poff; 4624 toaddr = cbpage->pm_cbaddr + poff; 4625 ROOTNEX_DPROBE2(rootnex__sync__dev, 4626 dev_info_t *, dma->dp_dip, size_t, psize); 4627 4628 /* 4629 * if we're copying for the cpu/kernel, we are going to 4630 * copy from the rootnex allocated copy buffer to the 4631 * drivers buffer. 4632 */ 4633 } else { 4634 fromaddr = cbpage->pm_cbaddr + poff; 4635 toaddr = cbpage->pm_kaddr + poff; 4636 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4637 dev_info_t *, dma->dp_dip, size_t, psize); 4638 } 4639 4640 bcopy(fromaddr, toaddr, psize); 4641 } 4642 4643 /* 4644 * decrement size until we're done, update our offset into the 4645 * buffer, and get the next page size. 4646 */ 4647 size -= psize; 4648 offset += psize; 4649 psize = MIN(MMU_PAGESIZE, size); 4650 4651 /* page offset is zero for the rest of this loop */ 4652 poff = 0; 4653 } 4654 4655 return (DDI_SUCCESS); 4656 } 4657 4658 /* 4659 * rootnex_dma_sync() 4660 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4661 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4662 * is set, ddi_dma_sync() returns immediately passing back success. 4663 */ 4664 /*ARGSUSED*/ 4665 static int 4666 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4667 off_t off, size_t len, uint_t cache_flags) 4668 { 4669 #if defined(__amd64) && !defined(__xpv) 4670 if (IOMMU_USED(rdip)) { 4671 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4672 cache_flags)); 4673 } 4674 #endif 4675 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4676 cache_flags)); 4677 } 4678 4679 /* 4680 * rootnex_valid_sync_parms() 4681 * checks the parameters passed to sync to verify they are correct. 4682 */ 4683 static int 4684 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4685 off_t offset, size_t size, uint_t cache_flags) 4686 { 4687 off_t woffset; 4688 4689 4690 /* 4691 * the first part of the test to make sure the offset passed in is 4692 * within the window. 4693 */ 4694 if (offset < win->wd_offset) { 4695 return (DDI_FAILURE); 4696 } 4697 4698 /* 4699 * second and last part of the test to make sure the offset and length 4700 * passed in is within the window. 4701 */ 4702 woffset = offset - win->wd_offset; 4703 if ((woffset + size) > win->wd_size) { 4704 return (DDI_FAILURE); 4705 } 4706 4707 /* 4708 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4709 * be set too. 4710 */ 4711 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4712 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4713 return (DDI_SUCCESS); 4714 } 4715 4716 /* 4717 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4718 * should be set. Also DDI_DMA_READ should be set in the flags. 4719 */ 4720 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4721 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4722 (hp->dmai_rflags & DDI_DMA_READ)) { 4723 return (DDI_SUCCESS); 4724 } 4725 4726 return (DDI_FAILURE); 4727 } 4728 4729 4730 /*ARGSUSED*/ 4731 static int 4732 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4733 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4734 uint_t *ccountp) 4735 { 4736 rootnex_window_t *window; 4737 rootnex_trim_t *trim; 4738 ddi_dma_impl_t *hp; 4739 rootnex_dma_t *dma; 4740 ddi_dma_obj_t *dmao; 4741 #if !defined(__amd64) 4742 rootnex_sglinfo_t *sinfo; 4743 rootnex_pgmap_t *pmap; 4744 uint_t pidx; 4745 uint_t pcnt; 4746 off_t poff; 4747 int i; 4748 #endif 4749 4750 4751 hp = (ddi_dma_impl_t *)handle; 4752 dma = (rootnex_dma_t *)hp->dmai_private; 4753 #if !defined(__amd64) 4754 sinfo = &dma->dp_sglinfo; 4755 #endif 4756 4757 /* If we try and get a window which doesn't exist, return failure */ 4758 if (win >= hp->dmai_nwin) { 4759 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4760 return (DDI_FAILURE); 4761 } 4762 4763 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4764 4765 /* 4766 * if we don't have any windows, and they're asking for the first 4767 * window, setup the cookie pointer to the first cookie in the bind. 4768 * setup our return values, then increment the cookie since we return 4769 * the first cookie on the stack. 4770 */ 4771 if (dma->dp_window == NULL) { 4772 if (win != 0) { 4773 ROOTNEX_DPROF_INC( 4774 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4775 return (DDI_FAILURE); 4776 } 4777 hp->dmai_cookie = dma->dp_cookies; 4778 *offp = 0; 4779 *lenp = dmao->dmao_size; 4780 *ccountp = dma->dp_sglinfo.si_sgl_size; 4781 *cookiep = hp->dmai_cookie[0]; 4782 hp->dmai_cookie++; 4783 return (DDI_SUCCESS); 4784 } 4785 4786 /* sync the old window before moving on to the new one */ 4787 window = &dma->dp_window[dma->dp_current_win]; 4788 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4789 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4790 DDI_DMA_SYNC_FORCPU); 4791 } 4792 4793 #if !defined(__amd64) 4794 /* 4795 * before we move to the next window, if we need to re-map, unmap all 4796 * the pages in this window. 4797 */ 4798 if (dma->dp_cb_remaping) { 4799 /* 4800 * If we switch to this window again, we'll need to map in 4801 * on the fly next time. 4802 */ 4803 window->wd_remap_copybuf = B_TRUE; 4804 4805 /* 4806 * calculate the page index into the buffer where this window 4807 * starts, and the number of pages this window takes up. 4808 */ 4809 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4810 MMU_PAGESHIFT; 4811 poff = (sinfo->si_buf_offset + window->wd_offset) & 4812 MMU_PAGEOFFSET; 4813 pcnt = mmu_btopr(window->wd_size + poff); 4814 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4815 4816 /* unmap pages which are currently mapped in this window */ 4817 for (i = 0; i < pcnt; i++) { 4818 if (dma->dp_pgmap[pidx].pm_mapped) { 4819 hat_unload(kas.a_hat, 4820 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4821 HAT_UNLOAD); 4822 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4823 } 4824 pidx++; 4825 } 4826 } 4827 #endif 4828 4829 /* 4830 * Move to the new window. 4831 * NOTE: current_win must be set for sync to work right 4832 */ 4833 dma->dp_current_win = win; 4834 window = &dma->dp_window[win]; 4835 4836 /* if needed, adjust the first and/or last cookies for trim */ 4837 trim = &window->wd_trim; 4838 if (trim->tr_trim_first) { 4839 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4840 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4841 #if !defined(__amd64) 4842 window->wd_first_cookie->dmac_type = 4843 (window->wd_first_cookie->dmac_type & 4844 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4845 #endif 4846 if (trim->tr_first_copybuf_win) { 4847 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4848 trim->tr_first_cbaddr; 4849 #if !defined(__amd64) 4850 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4851 trim->tr_first_kaddr; 4852 #endif 4853 } 4854 } 4855 if (trim->tr_trim_last) { 4856 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4857 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4858 if (trim->tr_last_copybuf_win) { 4859 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4860 trim->tr_last_cbaddr; 4861 #if !defined(__amd64) 4862 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4863 trim->tr_last_kaddr; 4864 #endif 4865 } 4866 } 4867 4868 /* 4869 * setup the cookie pointer to the first cookie in the window. setup 4870 * our return values, then increment the cookie since we return the 4871 * first cookie on the stack. 4872 */ 4873 hp->dmai_cookie = window->wd_first_cookie; 4874 *offp = window->wd_offset; 4875 *lenp = window->wd_size; 4876 *ccountp = window->wd_cookie_cnt; 4877 *cookiep = hp->dmai_cookie[0]; 4878 hp->dmai_cookie++; 4879 4880 #if !defined(__amd64) 4881 /* re-map copybuf if required for this window */ 4882 if (dma->dp_cb_remaping) { 4883 /* 4884 * calculate the page index into the buffer where this 4885 * window starts. 4886 */ 4887 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4888 MMU_PAGESHIFT; 4889 ASSERT(pidx < sinfo->si_max_pages); 4890 4891 /* 4892 * the first page can get unmapped if it's shared with the 4893 * previous window. Even if the rest of this window is already 4894 * mapped in, we need to still check this one. 4895 */ 4896 pmap = &dma->dp_pgmap[pidx]; 4897 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4898 if (pmap->pm_pp != NULL) { 4899 pmap->pm_mapped = B_TRUE; 4900 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4901 } else if (pmap->pm_vaddr != NULL) { 4902 pmap->pm_mapped = B_TRUE; 4903 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4904 pmap->pm_kaddr); 4905 } 4906 } 4907 pidx++; 4908 4909 /* map in the rest of the pages if required */ 4910 if (window->wd_remap_copybuf) { 4911 window->wd_remap_copybuf = B_FALSE; 4912 4913 /* figure out many pages this window takes up */ 4914 poff = (sinfo->si_buf_offset + window->wd_offset) & 4915 MMU_PAGEOFFSET; 4916 pcnt = mmu_btopr(window->wd_size + poff); 4917 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4918 4919 /* map pages which require it */ 4920 for (i = 1; i < pcnt; i++) { 4921 pmap = &dma->dp_pgmap[pidx]; 4922 if (pmap->pm_uses_copybuf) { 4923 ASSERT(pmap->pm_mapped == B_FALSE); 4924 if (pmap->pm_pp != NULL) { 4925 pmap->pm_mapped = B_TRUE; 4926 i86_pp_map(pmap->pm_pp, 4927 pmap->pm_kaddr); 4928 } else if (pmap->pm_vaddr != NULL) { 4929 pmap->pm_mapped = B_TRUE; 4930 i86_va_map(pmap->pm_vaddr, 4931 sinfo->si_asp, 4932 pmap->pm_kaddr); 4933 } 4934 } 4935 pidx++; 4936 } 4937 } 4938 } 4939 #endif 4940 4941 /* if the new window uses the copy buffer, sync it for the device */ 4942 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4943 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4944 DDI_DMA_SYNC_FORDEV); 4945 } 4946 4947 return (DDI_SUCCESS); 4948 } 4949 4950 /* 4951 * rootnex_dma_win() 4952 * called from ddi_dma_getwin() 4953 */ 4954 /*ARGSUSED*/ 4955 static int 4956 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4957 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4958 uint_t *ccountp) 4959 { 4960 #if defined(__amd64) && !defined(__xpv) 4961 if (IOMMU_USED(rdip)) { 4962 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4963 cookiep, ccountp)); 4964 } 4965 #endif 4966 4967 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4968 cookiep, ccountp)); 4969 } 4970 4971 #if defined(__amd64) && !defined(__xpv) 4972 /*ARGSUSED*/ 4973 static int 4974 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4975 ddi_dma_handle_t handle, void *v) 4976 { 4977 ddi_dma_impl_t *hp; 4978 rootnex_dma_t *dma; 4979 4980 hp = (ddi_dma_impl_t *)handle; 4981 dma = (rootnex_dma_t *)hp->dmai_private; 4982 dma->dp_iommu_private = v; 4983 4984 return (DDI_SUCCESS); 4985 } 4986 4987 /*ARGSUSED*/ 4988 static void * 4989 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4990 ddi_dma_handle_t handle) 4991 { 4992 ddi_dma_impl_t *hp; 4993 rootnex_dma_t *dma; 4994 4995 hp = (ddi_dma_impl_t *)handle; 4996 dma = (rootnex_dma_t *)hp->dmai_private; 4997 4998 return (dma->dp_iommu_private); 4999 } 5000 #endif 5001 5002 /* 5003 * ************************ 5004 * obsoleted dma routines 5005 * ************************ 5006 */ 5007 5008 /* 5009 * rootnex_dma_map() 5010 * called from ddi_dma_setup() 5011 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 5012 */ 5013 /* ARGSUSED */ 5014 static int 5015 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 5016 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 5017 { 5018 #if defined(__amd64) 5019 /* 5020 * this interface is not supported in 64-bit x86 kernel. See comment in 5021 * rootnex_dma_mctl() 5022 */ 5023 return (DDI_DMA_NORESOURCES); 5024 5025 #else /* 32-bit x86 kernel */ 5026 ddi_dma_handle_t *lhandlep; 5027 ddi_dma_handle_t lhandle; 5028 ddi_dma_cookie_t cookie; 5029 ddi_dma_attr_t dma_attr; 5030 ddi_dma_lim_t *dma_lim; 5031 uint_t ccnt; 5032 int e; 5033 5034 5035 /* 5036 * if the driver is just testing to see if it's possible to do the bind, 5037 * we'll use local state. Otherwise, use the handle pointer passed in. 5038 */ 5039 if (handlep == NULL) { 5040 lhandlep = &lhandle; 5041 } else { 5042 lhandlep = handlep; 5043 } 5044 5045 /* convert the limit structure to a dma_attr one */ 5046 dma_lim = dmareq->dmar_limits; 5047 dma_attr.dma_attr_version = DMA_ATTR_V0; 5048 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 5049 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 5050 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 5051 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 5052 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 5053 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 5054 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 5055 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 5056 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 5057 dma_attr.dma_attr_align = MMU_PAGESIZE; 5058 dma_attr.dma_attr_flags = 0; 5059 5060 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 5061 dmareq->dmar_arg, lhandlep); 5062 if (e != DDI_SUCCESS) { 5063 return (e); 5064 } 5065 5066 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 5067 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 5068 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5069 return (e); 5070 } 5071 5072 /* 5073 * if the driver is just testing to see if it's possible to do the bind, 5074 * free up the local state and return the result. 5075 */ 5076 if (handlep == NULL) { 5077 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 5078 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 5079 if (e == DDI_DMA_MAPPED) { 5080 return (DDI_DMA_MAPOK); 5081 } else { 5082 return (DDI_DMA_NOMAPPING); 5083 } 5084 } 5085 5086 return (e); 5087 #endif /* defined(__amd64) */ 5088 } 5089 5090 /* 5091 * rootnex_dma_mctl() 5092 * 5093 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 5094 */ 5095 /* ARGSUSED */ 5096 static int 5097 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 5098 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 5099 uint_t cache_flags) 5100 { 5101 #if defined(__amd64) 5102 /* 5103 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 5104 * common implementation in genunix, so they no longer have x86 5105 * specific functionality which called into dma_ctl. 5106 * 5107 * The rest of the obsoleted interfaces were never supported in the 5108 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 5109 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 5110 * implementation issues. 5111 * 5112 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 5113 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 5114 * reflect that now too... 5115 * 5116 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 5117 * not going to put this functionality into the 64-bit x86 kernel now. 5118 * It wasn't ported to the 64-bit kernel for s10, no reason to change 5119 * that in a future release. 5120 */ 5121 return (DDI_FAILURE); 5122 5123 #else /* 32-bit x86 kernel */ 5124 ddi_dma_cookie_t lcookie; 5125 ddi_dma_cookie_t *cookie; 5126 rootnex_window_t *window; 5127 ddi_dma_impl_t *hp; 5128 rootnex_dma_t *dma; 5129 uint_t nwin; 5130 uint_t ccnt; 5131 size_t len; 5132 off_t off; 5133 int e; 5134 5135 5136 /* 5137 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 5138 * hacky since were optimizing for the current interfaces and so we can 5139 * cleanup the mess in genunix. Hopefully we will remove the this 5140 * obsoleted routines someday soon. 5141 */ 5142 5143 switch (request) { 5144 5145 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 5146 hp = (ddi_dma_impl_t *)handle; 5147 cookie = (ddi_dma_cookie_t *)objpp; 5148 5149 /* 5150 * convert segment to cookie. We don't distinguish between the 5151 * two :-) 5152 */ 5153 *cookie = *hp->dmai_cookie; 5154 *lenp = cookie->dmac_size; 5155 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 5156 return (DDI_SUCCESS); 5157 5158 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 5159 hp = (ddi_dma_impl_t *)handle; 5160 dma = (rootnex_dma_t *)hp->dmai_private; 5161 5162 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 5163 return (DDI_DMA_STALE); 5164 } 5165 5166 /* handle the case where we don't have any windows */ 5167 if (dma->dp_window == NULL) { 5168 /* 5169 * if seg == NULL, and we don't have any windows, 5170 * return the first cookie in the sgl. 5171 */ 5172 if (*lenp == NULL) { 5173 dma->dp_current_cookie = 0; 5174 hp->dmai_cookie = dma->dp_cookies; 5175 *objpp = (caddr_t)handle; 5176 return (DDI_SUCCESS); 5177 5178 /* if we have more cookies, go to the next cookie */ 5179 } else { 5180 if ((dma->dp_current_cookie + 1) >= 5181 dma->dp_sglinfo.si_sgl_size) { 5182 return (DDI_DMA_DONE); 5183 } 5184 dma->dp_current_cookie++; 5185 hp->dmai_cookie++; 5186 return (DDI_SUCCESS); 5187 } 5188 } 5189 5190 /* We have one or more windows */ 5191 window = &dma->dp_window[dma->dp_current_win]; 5192 5193 /* 5194 * if seg == NULL, return the first cookie in the current 5195 * window 5196 */ 5197 if (*lenp == NULL) { 5198 dma->dp_current_cookie = 0; 5199 hp->dmai_cookie = window->wd_first_cookie; 5200 5201 /* 5202 * go to the next cookie in the window then see if we done with 5203 * this window. 5204 */ 5205 } else { 5206 if ((dma->dp_current_cookie + 1) >= 5207 window->wd_cookie_cnt) { 5208 return (DDI_DMA_DONE); 5209 } 5210 dma->dp_current_cookie++; 5211 hp->dmai_cookie++; 5212 } 5213 *objpp = (caddr_t)handle; 5214 return (DDI_SUCCESS); 5215 5216 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 5217 hp = (ddi_dma_impl_t *)handle; 5218 dma = (rootnex_dma_t *)hp->dmai_private; 5219 5220 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 5221 return (DDI_DMA_STALE); 5222 } 5223 5224 /* if win == NULL, return the first window in the bind */ 5225 if (*offp == NULL) { 5226 nwin = 0; 5227 5228 /* 5229 * else, go to the next window then see if we're done with all 5230 * the windows. 5231 */ 5232 } else { 5233 nwin = dma->dp_current_win + 1; 5234 if (nwin >= hp->dmai_nwin) { 5235 return (DDI_DMA_DONE); 5236 } 5237 } 5238 5239 /* switch to the next window */ 5240 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 5241 &lcookie, &ccnt); 5242 ASSERT(e == DDI_SUCCESS); 5243 if (e != DDI_SUCCESS) { 5244 return (DDI_DMA_STALE); 5245 } 5246 5247 /* reset the cookie back to the first cookie in the window */ 5248 if (dma->dp_window != NULL) { 5249 window = &dma->dp_window[dma->dp_current_win]; 5250 hp->dmai_cookie = window->wd_first_cookie; 5251 } else { 5252 hp->dmai_cookie = dma->dp_cookies; 5253 } 5254 5255 *objpp = (caddr_t)handle; 5256 return (DDI_SUCCESS); 5257 5258 case DDI_DMA_FREE: /* ddi_dma_free() */ 5259 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 5260 (void) rootnex_dma_freehdl(dip, rdip, handle); 5261 if (rootnex_state->r_dvma_call_list_id) { 5262 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 5263 } 5264 return (DDI_SUCCESS); 5265 5266 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 5267 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 5268 /* should never get here, handled in genunix */ 5269 ASSERT(0); 5270 return (DDI_FAILURE); 5271 5272 case DDI_DMA_KVADDR: 5273 case DDI_DMA_GETERR: 5274 case DDI_DMA_COFF: 5275 return (DDI_FAILURE); 5276 } 5277 5278 return (DDI_FAILURE); 5279 #endif /* defined(__amd64) */ 5280 } 5281 5282 /* 5283 * ********* 5284 * FMA Code 5285 * ********* 5286 */ 5287 5288 /* 5289 * rootnex_fm_init() 5290 * FMA init busop 5291 */ 5292 /* ARGSUSED */ 5293 static int 5294 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 5295 ddi_iblock_cookie_t *ibc) 5296 { 5297 *ibc = rootnex_state->r_err_ibc; 5298 5299 return (ddi_system_fmcap); 5300 } 5301 5302 /* 5303 * rootnex_dma_check() 5304 * Function called after a dma fault occurred to find out whether the 5305 * fault address is associated with a driver that is able to handle faults 5306 * and recover from faults. 5307 */ 5308 /* ARGSUSED */ 5309 static int 5310 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 5311 const void *not_used) 5312 { 5313 rootnex_window_t *window; 5314 uint64_t start_addr; 5315 uint64_t fault_addr; 5316 ddi_dma_impl_t *hp; 5317 rootnex_dma_t *dma; 5318 uint64_t end_addr; 5319 size_t csize; 5320 int i; 5321 int j; 5322 5323 5324 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5325 hp = (ddi_dma_impl_t *)handle; 5326 ASSERT(hp); 5327 5328 dma = (rootnex_dma_t *)hp->dmai_private; 5329 5330 /* Get the address that we need to search for */ 5331 fault_addr = *(uint64_t *)addr; 5332 5333 /* 5334 * if we don't have any windows, we can just walk through all the 5335 * cookies. 5336 */ 5337 if (dma->dp_window == NULL) { 5338 /* for each cookie */ 5339 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5340 /* 5341 * if the faulted address is within the physical address 5342 * range of the cookie, return DDI_FM_NONFATAL. 5343 */ 5344 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5345 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5346 dma->dp_cookies[i].dmac_size))) { 5347 return (DDI_FM_NONFATAL); 5348 } 5349 } 5350 5351 /* fault_addr not within this DMA handle */ 5352 return (DDI_FM_UNKNOWN); 5353 } 5354 5355 /* we have mutiple windows, walk through each window */ 5356 for (i = 0; i < hp->dmai_nwin; i++) { 5357 window = &dma->dp_window[i]; 5358 5359 /* Go through all the cookies in the window */ 5360 for (j = 0; j < window->wd_cookie_cnt; j++) { 5361 5362 start_addr = window->wd_first_cookie[j].dmac_laddress; 5363 csize = window->wd_first_cookie[j].dmac_size; 5364 5365 /* 5366 * if we are trimming the first cookie in the window, 5367 * and this is the first cookie, adjust the start 5368 * address and size of the cookie to account for the 5369 * trim. 5370 */ 5371 if (window->wd_trim.tr_trim_first && (j == 0)) { 5372 start_addr = window->wd_trim.tr_first_paddr; 5373 csize = window->wd_trim.tr_first_size; 5374 } 5375 5376 /* 5377 * if we are trimming the last cookie in the window, 5378 * and this is the last cookie, adjust the start 5379 * address and size of the cookie to account for the 5380 * trim. 5381 */ 5382 if (window->wd_trim.tr_trim_last && 5383 (j == (window->wd_cookie_cnt - 1))) { 5384 start_addr = window->wd_trim.tr_last_paddr; 5385 csize = window->wd_trim.tr_last_size; 5386 } 5387 5388 end_addr = start_addr + csize; 5389 5390 /* 5391 * if the faulted address is within the physical 5392 * address of the cookie, return DDI_FM_NONFATAL. 5393 */ 5394 if ((fault_addr >= start_addr) && 5395 (fault_addr <= end_addr)) { 5396 return (DDI_FM_NONFATAL); 5397 } 5398 } 5399 } 5400 5401 /* fault_addr not within this DMA handle */ 5402 return (DDI_FM_UNKNOWN); 5403 } 5404 5405 /*ARGSUSED*/ 5406 static int 5407 rootnex_quiesce(dev_info_t *dip) 5408 { 5409 #if defined(__amd64) && !defined(__xpv) 5410 return (immu_quiesce()); 5411 #else 5412 return (DDI_SUCCESS); 5413 #endif 5414 } 5415 5416 #if defined(__xpv) 5417 void 5418 immu_init(void) 5419 { 5420 ; 5421 } 5422 5423 void 5424 immu_startup(void) 5425 { 5426 ; 5427 } 5428 /*ARGSUSED*/ 5429 void 5430 immu_physmem_update(uint64_t addr, uint64_t size) 5431 { 5432 ; 5433 } 5434 #endif 5435