1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 * Copyright 2017 Joyent, Inc. 29 * Copyright 2020 Ryan Zezeski 30 */ 31 32 /* 33 * x86 root nexus driver 34 */ 35 36 #include <sys/sysmacros.h> 37 #include <sys/conf.h> 38 #include <sys/autoconf.h> 39 #include <sys/sysmacros.h> 40 #include <sys/debug.h> 41 #include <sys/psw.h> 42 #include <sys/ddidmareq.h> 43 #include <sys/promif.h> 44 #include <sys/devops.h> 45 #include <sys/kmem.h> 46 #include <sys/cmn_err.h> 47 #include <vm/seg.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_dev.h> 50 #include <sys/vmem.h> 51 #include <sys/mman.h> 52 #include <vm/hat.h> 53 #include <vm/as.h> 54 #include <vm/page.h> 55 #include <sys/avintr.h> 56 #include <sys/errno.h> 57 #include <sys/modctl.h> 58 #include <sys/ddi_impldefs.h> 59 #include <sys/sunddi.h> 60 #include <sys/sunndi.h> 61 #include <sys/mach_intr.h> 62 #include <sys/psm.h> 63 #include <sys/ontrap.h> 64 #include <sys/atomic.h> 65 #include <sys/sdt.h> 66 #include <sys/rootnex.h> 67 #include <vm/hat_i86.h> 68 #include <sys/ddifm.h> 69 #include <sys/ddi_isa.h> 70 #include <sys/apic.h> 71 72 #ifdef __xpv 73 #include <sys/bootinfo.h> 74 #include <sys/hypervisor.h> 75 #include <sys/bootconf.h> 76 #include <vm/kboot_mmu.h> 77 #endif 78 79 #if !defined(__xpv) 80 #include <sys/immu.h> 81 #endif 82 83 84 /* 85 * enable/disable extra checking of function parameters. Useful for debugging 86 * drivers. 87 */ 88 #ifdef DEBUG 89 int rootnex_alloc_check_parms = 1; 90 int rootnex_bind_check_inuse = 1; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 1; 93 #else 94 int rootnex_alloc_check_parms = 0; 95 int rootnex_bind_check_inuse = 0; 96 int rootnex_unbind_verify_buffer = 0; 97 int rootnex_sync_check_parms = 0; 98 #endif 99 100 boolean_t rootnex_dmar_not_setup; 101 102 /* Master Abort and Target Abort panic flag */ 103 int rootnex_fm_ma_ta_panic_flag = 0; 104 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 106 int rootnex_bind_fail = 1; 107 int rootnex_bind_warn = 1; 108 uint8_t *rootnex_warn_list; 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 110 #define ROOTNEX_BIND_WARNING (0x1 << 0) 111 112 /* 113 * revert back to old broken behavior of always sync'ing entire copy buffer. 114 * This is useful if be have a buggy driver which doesn't correctly pass in 115 * the offset and size into ddi_dma_sync(). 116 */ 117 int rootnex_sync_ignore_params = 0; 118 119 /* 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 121 * page for alignment. Allocate enough windows to handle a 256K buffer w/ at 122 * least 65 sgllen DMA engine, and enough copybuf buffer state pages to handle 123 * 2 pages (< 8K). We will still need to allocate the copy buffer during bind 124 * though (if we need one). These can only be modified in /etc/system before 125 * rootnex attach. 126 */ 127 int rootnex_prealloc_cookies = 65; 128 int rootnex_prealloc_windows = 4; 129 int rootnex_prealloc_copybuf = 2; 130 131 /* driver global state */ 132 static rootnex_state_t *rootnex_state; 133 134 #ifdef DEBUG 135 /* shortcut to rootnex counters */ 136 static uint64_t *rootnex_cnt; 137 #endif 138 139 /* 140 * XXX - does x86 even need these or are they left over from the SPARC days? 141 */ 142 /* statically defined integer/boolean properties for the root node */ 143 static rootnex_intprop_t rootnex_intprp[] = { 144 { "PAGESIZE", PAGESIZE }, 145 { "MMU_PAGESIZE", MMU_PAGESIZE }, 146 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 147 { DDI_RELATIVE_ADDRESSING, 1 }, 148 }; 149 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 150 151 /* 152 * If we're dom0, we're using a real device so we need to load 153 * the cookies with MFNs instead of PFNs. 154 */ 155 #ifdef __xpv 156 typedef maddr_t rootnex_addr_t; 157 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 158 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 159 #else 160 typedef paddr_t rootnex_addr_t; 161 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 162 #endif 163 164 static struct cb_ops rootnex_cb_ops = { 165 nodev, /* open */ 166 nodev, /* close */ 167 nodev, /* strategy */ 168 nodev, /* print */ 169 nodev, /* dump */ 170 nodev, /* read */ 171 nodev, /* write */ 172 nodev, /* ioctl */ 173 nodev, /* devmap */ 174 nodev, /* mmap */ 175 nodev, /* segmap */ 176 nochpoll, /* chpoll */ 177 ddi_prop_op, /* cb_prop_op */ 178 NULL, /* struct streamtab */ 179 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 180 CB_REV, /* Rev */ 181 nodev, /* cb_aread */ 182 nodev /* cb_awrite */ 183 }; 184 185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 186 off_t offset, off_t len, caddr_t *vaddrp); 187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 188 struct hat *hat, struct seg *seg, caddr_t addr, 189 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 190 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 191 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 192 ddi_dma_handle_t *handlep); 193 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 194 ddi_dma_handle_t handle); 195 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 196 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 197 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 198 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_handle_t handle); 200 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 202 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 204 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 205 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 206 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 207 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 208 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 209 ddi_ctl_enum_t ctlop, void *arg, void *result); 210 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 211 ddi_iblock_cookie_t *ibc); 212 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 213 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 214 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 215 void *); 216 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 217 218 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 219 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 220 ddi_dma_handle_t *handlep); 221 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle); 223 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 224 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 225 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 226 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 227 ddi_dma_handle_t handle); 228 #if !defined(__xpv) 229 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 230 ddi_dma_handle_t handle); 231 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 232 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 233 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 234 ddi_dma_cookie_t *cookiep, uint_t ccount); 235 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 236 ddi_dma_handle_t handle); 237 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 238 #endif 239 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 240 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 241 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 242 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 243 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 244 245 #if !defined(__xpv) 246 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 247 ddi_dma_handle_t handle, void *v); 248 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 249 ddi_dma_handle_t handle); 250 #endif 251 252 253 static struct bus_ops rootnex_bus_ops = { 254 BUSO_REV, 255 rootnex_map, 256 NULL, 257 NULL, 258 NULL, 259 rootnex_map_fault, 260 0, 261 rootnex_dma_allochdl, 262 rootnex_dma_freehdl, 263 rootnex_dma_bindhdl, 264 rootnex_dma_unbindhdl, 265 rootnex_dma_sync, 266 rootnex_dma_win, 267 rootnex_dma_mctl, 268 rootnex_ctlops, 269 ddi_bus_prop_op, 270 i_ddi_rootnex_get_eventcookie, 271 i_ddi_rootnex_add_eventcall, 272 i_ddi_rootnex_remove_eventcall, 273 i_ddi_rootnex_post_event, 274 0, /* bus_intr_ctl */ 275 0, /* bus_config */ 276 0, /* bus_unconfig */ 277 rootnex_fm_init, /* bus_fm_init */ 278 NULL, /* bus_fm_fini */ 279 NULL, /* bus_fm_access_enter */ 280 NULL, /* bus_fm_access_exit */ 281 NULL, /* bus_powr */ 282 rootnex_intr_ops /* bus_intr_op */ 283 }; 284 285 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 286 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 287 static int rootnex_quiesce(dev_info_t *dip); 288 289 static struct dev_ops rootnex_ops = { 290 DEVO_REV, 291 0, 292 ddi_no_info, 293 nulldev, 294 nulldev, 295 rootnex_attach, 296 rootnex_detach, 297 nulldev, 298 &rootnex_cb_ops, 299 &rootnex_bus_ops, 300 NULL, 301 rootnex_quiesce, /* quiesce */ 302 }; 303 304 static struct modldrv rootnex_modldrv = { 305 &mod_driverops, 306 "i86pc root nexus", 307 &rootnex_ops 308 }; 309 310 static struct modlinkage rootnex_modlinkage = { 311 MODREV_1, 312 (void *)&rootnex_modldrv, 313 NULL 314 }; 315 316 #if !defined(__xpv) 317 static iommulib_nexops_t iommulib_nexops = { 318 IOMMU_NEXOPS_VERSION, 319 "Rootnex IOMMU ops Vers 1.1", 320 NULL, 321 rootnex_coredma_allochdl, 322 rootnex_coredma_freehdl, 323 rootnex_coredma_bindhdl, 324 rootnex_coredma_unbindhdl, 325 rootnex_coredma_reset_cookies, 326 rootnex_coredma_get_cookies, 327 rootnex_coredma_set_cookies, 328 rootnex_coredma_clear_cookies, 329 rootnex_coredma_get_sleep_flags, 330 rootnex_coredma_sync, 331 rootnex_coredma_win, 332 rootnex_coredma_hdl_setprivate, 333 rootnex_coredma_hdl_getprivate 334 }; 335 #endif 336 337 /* 338 * extern hacks 339 */ 340 extern struct seg_ops segdev_ops; 341 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 342 #ifdef DDI_MAP_DEBUG 343 extern int ddi_map_debug_flag; 344 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 345 #endif 346 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 347 psm_intr_op_t, int *); 348 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 349 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 350 351 /* 352 * Use device arena to use for device control register mappings. 353 * Various kernel memory walkers (debugger, dtrace) need to know 354 * to avoid this address range to prevent undesired device activity. 355 */ 356 extern void *device_arena_alloc(size_t size, int vm_flag); 357 extern void device_arena_free(void * vaddr, size_t size); 358 359 360 /* 361 * Internal functions 362 */ 363 static int rootnex_dma_init(); 364 static void rootnex_add_props(dev_info_t *); 365 static int rootnex_ctl_reportdev(dev_info_t *dip); 366 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 367 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 368 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 369 static int rootnex_map_handle(ddi_map_req_t *mp); 370 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 371 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 372 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 373 rootnex_sglinfo_t *sglinfo); 374 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 375 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 376 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 377 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 378 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 379 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 380 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 381 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 382 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 383 static void rootnex_teardown_windows(rootnex_dma_t *dma); 384 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 385 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 386 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 387 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 388 size_t *copybuf_used, page_t **cur_pp); 389 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 390 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 391 ddi_dma_attr_t *attr, off_t cur_offset); 392 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 393 rootnex_dma_t *dma, rootnex_window_t **windowp, 394 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 395 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 396 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 397 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 398 off_t offset, size_t size, uint_t cache_flags); 399 static int rootnex_verify_buffer(rootnex_dma_t *dma); 400 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 401 const void *comp_addr, const void *not_used); 402 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 403 rootnex_sglinfo_t *sglinfo); 404 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 405 406 /* 407 * _init() 408 * 409 */ 410 int 411 _init(void) 412 { 413 414 rootnex_state = NULL; 415 return (mod_install(&rootnex_modlinkage)); 416 } 417 418 419 /* 420 * _info() 421 * 422 */ 423 int 424 _info(struct modinfo *modinfop) 425 { 426 return (mod_info(&rootnex_modlinkage, modinfop)); 427 } 428 429 430 /* 431 * _fini() 432 * 433 */ 434 int 435 _fini(void) 436 { 437 return (EBUSY); 438 } 439 440 441 /* 442 * rootnex_attach() 443 * 444 */ 445 static int 446 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 447 { 448 int fmcap; 449 int e; 450 451 switch (cmd) { 452 case DDI_ATTACH: 453 break; 454 case DDI_RESUME: 455 #if !defined(__xpv) 456 return (immu_unquiesce()); 457 #else 458 return (DDI_SUCCESS); 459 #endif 460 default: 461 return (DDI_FAILURE); 462 } 463 464 /* 465 * We should only have one instance of rootnex. Save it away since we 466 * don't have an easy way to get it back later. 467 */ 468 ASSERT(rootnex_state == NULL); 469 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 470 471 rootnex_state->r_dip = dip; 472 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 473 rootnex_state->r_reserved_msg_printed = B_FALSE; 474 #ifdef DEBUG 475 rootnex_cnt = &rootnex_state->r_counters[0]; 476 #endif 477 478 /* 479 * Set minimum fm capability level for i86pc platforms and then 480 * initialize error handling. Since we're the rootnex, we don't 481 * care what's returned in the fmcap field. 482 */ 483 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 484 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 485 fmcap = ddi_system_fmcap; 486 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 487 488 /* initialize DMA related state */ 489 e = rootnex_dma_init(); 490 if (e != DDI_SUCCESS) { 491 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 492 return (DDI_FAILURE); 493 } 494 495 /* Add static root node properties */ 496 rootnex_add_props(dip); 497 498 /* since we can't call ddi_report_dev() */ 499 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 500 501 /* Initialize rootnex event handle */ 502 i_ddi_rootnex_init_events(dip); 503 504 #if !defined(__xpv) 505 e = iommulib_nexus_register(dip, &iommulib_nexops, 506 &rootnex_state->r_iommulib_handle); 507 508 ASSERT(e == DDI_SUCCESS); 509 #endif 510 511 return (DDI_SUCCESS); 512 } 513 514 515 /* 516 * rootnex_detach() 517 * 518 */ 519 /*ARGSUSED*/ 520 static int 521 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 522 { 523 switch (cmd) { 524 case DDI_SUSPEND: 525 #if !defined(__xpv) 526 return (immu_quiesce()); 527 #else 528 return (DDI_SUCCESS); 529 #endif 530 default: 531 return (DDI_FAILURE); 532 } 533 /*NOTREACHED*/ 534 535 } 536 537 538 /* 539 * rootnex_dma_init() 540 * 541 */ 542 /*ARGSUSED*/ 543 static int 544 rootnex_dma_init() 545 { 546 size_t bufsize; 547 548 549 /* 550 * size of our cookie/window/copybuf state needed in dma bind that we 551 * pre-alloc in dma_alloc_handle 552 */ 553 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 554 rootnex_state->r_prealloc_size = 555 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 556 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 557 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 558 559 /* 560 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 561 * allocate 16 extra bytes for struct pointer alignment 562 * (p->dmai_private & dma->dp_prealloc_buffer) 563 */ 564 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 565 rootnex_state->r_prealloc_size + 0x10; 566 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 567 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 568 if (rootnex_state->r_dmahdl_cache == NULL) { 569 return (DDI_FAILURE); 570 } 571 572 /* 573 * allocate array to track which major numbers we have printed warnings 574 * for. 575 */ 576 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 577 KM_SLEEP); 578 579 return (DDI_SUCCESS); 580 } 581 582 583 /* 584 * rootnex_add_props() 585 * 586 */ 587 static void 588 rootnex_add_props(dev_info_t *dip) 589 { 590 rootnex_intprop_t *rpp; 591 int i; 592 593 /* Add static integer/boolean properties to the root node */ 594 rpp = rootnex_intprp; 595 for (i = 0; i < NROOT_INTPROPS; i++) { 596 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 597 rpp[i].prop_name, rpp[i].prop_value); 598 } 599 } 600 601 602 603 /* 604 * ************************* 605 * ctlops related routines 606 * ************************* 607 */ 608 609 /* 610 * rootnex_ctlops() 611 * 612 */ 613 /*ARGSUSED*/ 614 static int 615 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 616 void *arg, void *result) 617 { 618 int n, *ptr; 619 struct ddi_parent_private_data *pdp; 620 621 switch (ctlop) { 622 case DDI_CTLOPS_DMAPMAPC: 623 /* 624 * Return 'partial' to indicate that dma mapping 625 * has to be done in the main MMU. 626 */ 627 return (DDI_DMA_PARTIAL); 628 629 case DDI_CTLOPS_BTOP: 630 /* 631 * Convert byte count input to physical page units. 632 * (byte counts that are not a page-size multiple 633 * are rounded down) 634 */ 635 *(ulong_t *)result = btop(*(ulong_t *)arg); 636 return (DDI_SUCCESS); 637 638 case DDI_CTLOPS_PTOB: 639 /* 640 * Convert size in physical pages to bytes 641 */ 642 *(ulong_t *)result = ptob(*(ulong_t *)arg); 643 return (DDI_SUCCESS); 644 645 case DDI_CTLOPS_BTOPR: 646 /* 647 * Convert byte count input to physical page units 648 * (byte counts that are not a page-size multiple 649 * are rounded up) 650 */ 651 *(ulong_t *)result = btopr(*(ulong_t *)arg); 652 return (DDI_SUCCESS); 653 654 case DDI_CTLOPS_INITCHILD: 655 return (impl_ddi_sunbus_initchild(arg)); 656 657 case DDI_CTLOPS_UNINITCHILD: 658 impl_ddi_sunbus_removechild(arg); 659 return (DDI_SUCCESS); 660 661 case DDI_CTLOPS_REPORTDEV: 662 return (rootnex_ctl_reportdev(rdip)); 663 664 case DDI_CTLOPS_IOMIN: 665 /* 666 * Nothing to do here but reflect back.. 667 */ 668 return (DDI_SUCCESS); 669 670 case DDI_CTLOPS_REGSIZE: 671 case DDI_CTLOPS_NREGS: 672 break; 673 674 case DDI_CTLOPS_SIDDEV: 675 if (ndi_dev_is_prom_node(rdip)) 676 return (DDI_SUCCESS); 677 if (ndi_dev_is_persistent_node(rdip)) 678 return (DDI_SUCCESS); 679 return (DDI_FAILURE); 680 681 case DDI_CTLOPS_POWER: 682 return ((*pm_platform_power)((power_req_t *)arg)); 683 684 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 685 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 686 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 687 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 688 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 689 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 690 if (!rootnex_state->r_reserved_msg_printed) { 691 rootnex_state->r_reserved_msg_printed = B_TRUE; 692 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 693 "1 or more reserved/obsolete operations."); 694 } 695 return (DDI_FAILURE); 696 697 default: 698 return (DDI_FAILURE); 699 } 700 /* 701 * The rest are for "hardware" properties 702 */ 703 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 704 return (DDI_FAILURE); 705 706 if (ctlop == DDI_CTLOPS_NREGS) { 707 ptr = (int *)result; 708 *ptr = pdp->par_nreg; 709 } else { 710 off_t *size = (off_t *)result; 711 712 ptr = (int *)arg; 713 n = *ptr; 714 if (n >= pdp->par_nreg) { 715 return (DDI_FAILURE); 716 } 717 *size = (off_t)pdp->par_reg[n].regspec_size; 718 } 719 return (DDI_SUCCESS); 720 } 721 722 723 /* 724 * rootnex_ctl_reportdev() 725 * 726 */ 727 static int 728 rootnex_ctl_reportdev(dev_info_t *dev) 729 { 730 int i, n, len, f_len = 0; 731 char *buf; 732 733 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 734 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 735 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 736 len = strlen(buf); 737 738 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 739 740 struct regspec *rp = sparc_pd_getreg(dev, i); 741 742 if (i == 0) 743 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 744 ": "); 745 else 746 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 747 " and "); 748 len = strlen(buf); 749 750 switch (rp->regspec_bustype) { 751 752 case BTEISA: 753 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 754 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 755 break; 756 757 case BTISA: 758 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 759 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 760 break; 761 762 default: 763 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 764 "space %x offset %x", 765 rp->regspec_bustype, rp->regspec_addr); 766 break; 767 } 768 len = strlen(buf); 769 } 770 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 771 int pri; 772 773 if (i != 0) { 774 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 775 ","); 776 len = strlen(buf); 777 } 778 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 779 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 780 " sparc ipl %d", pri); 781 len = strlen(buf); 782 } 783 #ifdef DEBUG 784 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 785 cmn_err(CE_NOTE, "next message is truncated: " 786 "printed length 1024, real length %d", f_len); 787 } 788 #endif /* DEBUG */ 789 cmn_err(CE_CONT, "?%s\n", buf); 790 kmem_free(buf, REPORTDEV_BUFSIZE); 791 return (DDI_SUCCESS); 792 } 793 794 795 /* 796 * ****************** 797 * map related code 798 * ****************** 799 */ 800 801 /* 802 * rootnex_map() 803 * 804 */ 805 static int 806 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 807 off_t len, caddr_t *vaddrp) 808 { 809 struct regspec *orp = NULL; 810 struct regspec64 rp = { 0 }; 811 ddi_map_req_t mr = *mp; /* Get private copy of request */ 812 813 mp = &mr; 814 815 switch (mp->map_op) { 816 case DDI_MO_MAP_LOCKED: 817 case DDI_MO_UNMAP: 818 case DDI_MO_MAP_HANDLE: 819 break; 820 default: 821 #ifdef DDI_MAP_DEBUG 822 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 823 mp->map_op); 824 #endif /* DDI_MAP_DEBUG */ 825 return (DDI_ME_UNIMPLEMENTED); 826 } 827 828 if (mp->map_flags & DDI_MF_USER_MAPPING) { 829 #ifdef DDI_MAP_DEBUG 830 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 831 #endif /* DDI_MAP_DEBUG */ 832 return (DDI_ME_UNIMPLEMENTED); 833 } 834 835 /* 836 * First, we need to get the original regspec out before we convert it 837 * to the extended format. If we have a register number, then we need to 838 * convert that to a regspec. 839 */ 840 if (mp->map_type == DDI_MT_RNUMBER) { 841 842 int rnumber = mp->map_obj.rnumber; 843 #ifdef DDI_MAP_DEBUG 844 static char *out_of_range = 845 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 846 #endif /* DDI_MAP_DEBUG */ 847 848 orp = i_ddi_rnumber_to_regspec(rdip, rnumber); 849 if (orp == NULL) { 850 #ifdef DDI_MAP_DEBUG 851 cmn_err(CE_WARN, out_of_range, rnumber, 852 ddi_get_name(rdip)); 853 #endif /* DDI_MAP_DEBUG */ 854 return (DDI_ME_RNUMBER_RANGE); 855 } 856 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) { 857 orp = mp->map_obj.rp; 858 } 859 860 /* 861 * Ensure that we are always using a 64-bit extended regspec regardless 862 * of what was passed into us. If the child driver is using a 64-bit 863 * regspec, then we need to make sure that we copy this to the local 864 * regspec64, rp. 865 */ 866 if (orp != NULL) { 867 rp.regspec_bustype = orp->regspec_bustype; 868 rp.regspec_addr = orp->regspec_addr; 869 rp.regspec_size = orp->regspec_size; 870 } else { 871 struct regspec64 *rp64; 872 rp64 = (struct regspec64 *)mp->map_obj.rp; 873 rp = *rp64; 874 } 875 876 mp->map_type = DDI_MT_REGSPEC; 877 mp->map_flags |= DDI_MF_EXT_REGSPEC; 878 mp->map_obj.rp = (struct regspec *)&rp; 879 880 /* 881 * Adjust offset and length correspnding to called values... 882 * XXX: A non-zero length means override the one in the regspec 883 * XXX: (regardless of what's in the parent's range?) 884 */ 885 886 #ifdef DDI_MAP_DEBUG 887 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 888 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 889 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset, 890 len, mp->map_handlep); 891 #endif /* DDI_MAP_DEBUG */ 892 893 /* 894 * I/O or memory mapping: 895 * 896 * <bustype=0, addr=x, len=x>: memory 897 * <bustype=1, addr=x, len=x>: i/o 898 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 899 */ 900 901 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) { 902 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 903 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">", 904 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 905 rp.regspec_addr, rp.regspec_size); 906 return (DDI_ME_INVAL); 907 } 908 909 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) { 910 /* 911 * compatibility i/o mapping 912 */ 913 rp.regspec_bustype += offset; 914 } else { 915 /* 916 * Normal memory or i/o mapping 917 */ 918 rp.regspec_addr += offset; 919 } 920 921 if (len != 0) 922 rp.regspec_size = len; 923 924 #ifdef DDI_MAP_DEBUG 925 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64 926 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n", 927 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 928 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep); 929 #endif /* DDI_MAP_DEBUG */ 930 931 932 /* 933 * The x86 root nexus does not have any notion of valid ranges of 934 * addresses. Its children have valid ranges, but because there are none 935 * for the nexus, we don't need to call i_ddi_apply_range(). Verify 936 * that is the case. 937 */ 938 ASSERT0(sparc_pd_getnrng(dip)); 939 940 switch (mp->map_op) { 941 case DDI_MO_MAP_LOCKED: 942 943 /* 944 * Set up the locked down kernel mapping to the regspec... 945 */ 946 947 return (rootnex_map_regspec(mp, vaddrp)); 948 949 case DDI_MO_UNMAP: 950 951 /* 952 * Release mapping... 953 */ 954 955 return (rootnex_unmap_regspec(mp, vaddrp)); 956 957 case DDI_MO_MAP_HANDLE: 958 959 return (rootnex_map_handle(mp)); 960 961 default: 962 return (DDI_ME_UNIMPLEMENTED); 963 } 964 } 965 966 967 /* 968 * rootnex_map_fault() 969 * 970 * fault in mappings for requestors 971 */ 972 /*ARGSUSED*/ 973 static int 974 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 975 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 976 uint_t lock) 977 { 978 979 #ifdef DDI_MAP_DEBUG 980 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 981 ddi_map_debug(" Seg <%s>\n", 982 seg->s_ops == &segdev_ops ? "segdev" : 983 seg == &kvseg ? "segkmem" : "NONE!"); 984 #endif /* DDI_MAP_DEBUG */ 985 986 /* 987 * This is all terribly broken, but it is a start 988 * 989 * XXX Note that this test means that segdev_ops 990 * must be exported from seg_dev.c. 991 * XXX What about devices with their own segment drivers? 992 */ 993 if (seg->s_ops == &segdev_ops) { 994 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 995 996 if (hat == NULL) { 997 /* 998 * This is one plausible interpretation of 999 * a null hat i.e. use the first hat on the 1000 * address space hat list which by convention is 1001 * the hat of the system MMU. At alternative 1002 * would be to panic .. this might well be better .. 1003 */ 1004 ASSERT(AS_READ_HELD(seg->s_as)); 1005 hat = seg->s_as->a_hat; 1006 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1007 } 1008 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1009 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1010 } else if (seg == &kvseg && dp == NULL) { 1011 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1012 HAT_LOAD_LOCK); 1013 } else 1014 return (DDI_FAILURE); 1015 return (DDI_SUCCESS); 1016 } 1017 1018 1019 static int 1020 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1021 { 1022 rootnex_addr_t rbase; 1023 void *cvaddr; 1024 uint64_t npages, pgoffset; 1025 struct regspec64 *rp; 1026 ddi_acc_hdl_t *hp; 1027 ddi_acc_impl_t *ap; 1028 uint_t hat_acc_flags; 1029 paddr_t pbase; 1030 1031 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1032 rp = (struct regspec64 *)mp->map_obj.rp; 1033 hp = mp->map_handlep; 1034 1035 #ifdef DDI_MAP_DEBUG 1036 ddi_map_debug( 1037 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1038 rp->regspec_bustype, rp->regspec_addr, 1039 rp->regspec_size, mp->map_handlep); 1040 #endif /* DDI_MAP_DEBUG */ 1041 1042 /* 1043 * I/O or memory mapping 1044 * 1045 * <bustype=0, addr=x, len=x>: memory 1046 * <bustype=1, addr=x, len=x>: i/o 1047 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1048 */ 1049 1050 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1051 cmn_err(CE_WARN, "rootnex: invalid register spec" 1052 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">", 1053 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size); 1054 return (DDI_FAILURE); 1055 } 1056 1057 if (rp->regspec_bustype != 0) { 1058 /* 1059 * I/O space - needs a handle. 1060 */ 1061 if (hp == NULL) { 1062 return (DDI_FAILURE); 1063 } 1064 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1065 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1066 impl_acc_hdl_init(hp); 1067 1068 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1069 #ifdef DDI_MAP_DEBUG 1070 ddi_map_debug("rootnex_map_regspec: mmap() " 1071 "to I/O space is not supported.\n"); 1072 #endif /* DDI_MAP_DEBUG */ 1073 return (DDI_ME_INVAL); 1074 } else { 1075 /* 1076 * 1275-compliant vs. compatibility i/o mapping 1077 */ 1078 *vaddrp = 1079 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1080 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1081 ((caddr_t)(uintptr_t)rp->regspec_addr); 1082 #ifdef __xpv 1083 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1084 hp->ah_pfn = xen_assign_pfn( 1085 mmu_btop((ulong_t)rp->regspec_addr & 1086 MMU_PAGEMASK)); 1087 } else { 1088 hp->ah_pfn = mmu_btop( 1089 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1090 } 1091 #else 1092 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1093 MMU_PAGEMASK); 1094 #endif 1095 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1096 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1097 } 1098 1099 #ifdef DDI_MAP_DEBUG 1100 ddi_map_debug( 1101 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1102 rp->regspec_size, *vaddrp); 1103 #endif /* DDI_MAP_DEBUG */ 1104 return (DDI_SUCCESS); 1105 } 1106 1107 /* 1108 * Memory space 1109 */ 1110 1111 if (hp != NULL) { 1112 /* 1113 * hat layer ignores 1114 * hp->ah_acc.devacc_attr_endian_flags. 1115 */ 1116 switch (hp->ah_acc.devacc_attr_dataorder) { 1117 case DDI_STRICTORDER_ACC: 1118 hat_acc_flags = HAT_STRICTORDER; 1119 break; 1120 case DDI_UNORDERED_OK_ACC: 1121 hat_acc_flags = HAT_UNORDERED_OK; 1122 break; 1123 case DDI_MERGING_OK_ACC: 1124 hat_acc_flags = HAT_MERGING_OK; 1125 break; 1126 case DDI_LOADCACHING_OK_ACC: 1127 hat_acc_flags = HAT_LOADCACHING_OK; 1128 break; 1129 case DDI_STORECACHING_OK_ACC: 1130 hat_acc_flags = HAT_STORECACHING_OK; 1131 break; 1132 default: 1133 return (DDI_ME_INVAL); 1134 } 1135 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1136 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1137 impl_acc_hdl_init(hp); 1138 hp->ah_hat_flags = hat_acc_flags; 1139 } else { 1140 hat_acc_flags = HAT_STRICTORDER; 1141 } 1142 1143 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1144 #ifdef __xpv 1145 /* 1146 * If we're dom0, we're using a real device so we need to translate 1147 * the MA to a PA. 1148 */ 1149 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1150 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1151 } else { 1152 pbase = rbase; 1153 } 1154 #else 1155 pbase = rbase; 1156 #endif 1157 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1158 1159 if (rp->regspec_size == 0) { 1160 #ifdef DDI_MAP_DEBUG 1161 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1162 #endif /* DDI_MAP_DEBUG */ 1163 return (DDI_ME_INVAL); 1164 } 1165 1166 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1167 /* extra cast to make gcc happy */ 1168 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1169 } else { 1170 npages = mmu_btopr(rp->regspec_size + pgoffset); 1171 1172 #ifdef DDI_MAP_DEBUG 1173 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1174 "physical %llx", npages, pbase); 1175 #endif /* DDI_MAP_DEBUG */ 1176 1177 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1178 if (cvaddr == NULL) 1179 return (DDI_ME_NORESOURCES); 1180 1181 /* 1182 * Now map in the pages we've allocated... 1183 */ 1184 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1185 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1186 HAT_LOAD_LOCK); 1187 *vaddrp = (caddr_t)cvaddr + pgoffset; 1188 1189 /* save away pfn and npages for FMA */ 1190 hp = mp->map_handlep; 1191 if (hp) { 1192 hp->ah_pfn = mmu_btop(pbase); 1193 hp->ah_pnum = npages; 1194 } 1195 } 1196 1197 #ifdef DDI_MAP_DEBUG 1198 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1199 #endif /* DDI_MAP_DEBUG */ 1200 return (DDI_SUCCESS); 1201 } 1202 1203 1204 static int 1205 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1206 { 1207 caddr_t addr = (caddr_t)*vaddrp; 1208 uint64_t npages, pgoffset; 1209 struct regspec64 *rp; 1210 1211 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1212 return (0); 1213 1214 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1215 rp = (struct regspec64 *)mp->map_obj.rp; 1216 1217 if (rp->regspec_size == 0) { 1218 #ifdef DDI_MAP_DEBUG 1219 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1220 #endif /* DDI_MAP_DEBUG */ 1221 return (DDI_ME_INVAL); 1222 } 1223 1224 /* 1225 * I/O or memory mapping: 1226 * 1227 * <bustype=0, addr=x, len=x>: memory 1228 * <bustype=1, addr=x, len=x>: i/o 1229 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1230 */ 1231 if (rp->regspec_bustype != 0) { 1232 /* 1233 * This is I/O space, which requires no particular 1234 * processing on unmap since it isn't mapped in the 1235 * first place. 1236 */ 1237 return (DDI_SUCCESS); 1238 } 1239 1240 /* 1241 * Memory space 1242 */ 1243 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1244 npages = mmu_btopr(rp->regspec_size + pgoffset); 1245 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1246 device_arena_free(addr - pgoffset, ptob(npages)); 1247 1248 /* 1249 * Destroy the pointer - the mapping has logically gone 1250 */ 1251 *vaddrp = NULL; 1252 1253 return (DDI_SUCCESS); 1254 } 1255 1256 static int 1257 rootnex_map_handle(ddi_map_req_t *mp) 1258 { 1259 rootnex_addr_t rbase; 1260 ddi_acc_hdl_t *hp; 1261 uint64_t pgoffset; 1262 struct regspec64 *rp; 1263 paddr_t pbase; 1264 1265 rp = (struct regspec64 *)mp->map_obj.rp; 1266 1267 #ifdef DDI_MAP_DEBUG 1268 ddi_map_debug( 1269 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1270 rp->regspec_bustype, rp->regspec_addr, 1271 rp->regspec_size, mp->map_handlep); 1272 #endif /* DDI_MAP_DEBUG */ 1273 1274 /* 1275 * I/O or memory mapping: 1276 * 1277 * <bustype=0, addr=x, len=x>: memory 1278 * <bustype=1, addr=x, len=x>: i/o 1279 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1280 */ 1281 if (rp->regspec_bustype != 0) { 1282 /* 1283 * This refers to I/O space, and we don't support "mapping" 1284 * I/O space to a user. 1285 */ 1286 return (DDI_FAILURE); 1287 } 1288 1289 /* 1290 * Set up the hat_flags for the mapping. 1291 */ 1292 hp = mp->map_handlep; 1293 1294 switch (hp->ah_acc.devacc_attr_endian_flags) { 1295 case DDI_NEVERSWAP_ACC: 1296 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1297 break; 1298 case DDI_STRUCTURE_LE_ACC: 1299 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1300 break; 1301 case DDI_STRUCTURE_BE_ACC: 1302 return (DDI_FAILURE); 1303 default: 1304 return (DDI_REGS_ACC_CONFLICT); 1305 } 1306 1307 switch (hp->ah_acc.devacc_attr_dataorder) { 1308 case DDI_STRICTORDER_ACC: 1309 break; 1310 case DDI_UNORDERED_OK_ACC: 1311 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1312 break; 1313 case DDI_MERGING_OK_ACC: 1314 hp->ah_hat_flags |= HAT_MERGING_OK; 1315 break; 1316 case DDI_LOADCACHING_OK_ACC: 1317 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1318 break; 1319 case DDI_STORECACHING_OK_ACC: 1320 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1321 break; 1322 default: 1323 return (DDI_FAILURE); 1324 } 1325 1326 rbase = (rootnex_addr_t)rp->regspec_addr & 1327 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1328 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1329 1330 if (rp->regspec_size == 0) 1331 return (DDI_ME_INVAL); 1332 1333 #ifdef __xpv 1334 /* 1335 * If we're dom0, we're using a real device so we need to translate 1336 * the MA to a PA. 1337 */ 1338 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1339 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1340 (rbase & MMU_PAGEOFFSET); 1341 } else { 1342 pbase = rbase; 1343 } 1344 #else 1345 pbase = rbase; 1346 #endif 1347 1348 hp->ah_pfn = mmu_btop(pbase); 1349 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1350 1351 return (DDI_SUCCESS); 1352 } 1353 1354 1355 1356 /* 1357 * ************************ 1358 * interrupt related code 1359 * ************************ 1360 */ 1361 1362 /* 1363 * rootnex_intr_ops() 1364 * bus_intr_op() function for interrupt support 1365 */ 1366 /* ARGSUSED */ 1367 static int 1368 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1369 ddi_intr_handle_impl_t *hdlp, void *result) 1370 { 1371 struct intrspec *ispec; 1372 1373 DDI_INTR_NEXDBG((CE_CONT, 1374 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1375 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1376 1377 /* Process the interrupt operation */ 1378 switch (intr_op) { 1379 case DDI_INTROP_GETCAP: 1380 /* First check with pcplusmp */ 1381 if (psm_intr_ops == NULL) 1382 return (DDI_FAILURE); 1383 1384 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1385 *(int *)result = 0; 1386 return (DDI_FAILURE); 1387 } 1388 break; 1389 case DDI_INTROP_SETCAP: 1390 if (psm_intr_ops == NULL) 1391 return (DDI_FAILURE); 1392 1393 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1394 return (DDI_FAILURE); 1395 break; 1396 case DDI_INTROP_ALLOC: 1397 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1398 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1399 case DDI_INTROP_FREE: 1400 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1401 return (rootnex_free_intr_fixed(rdip, hdlp)); 1402 case DDI_INTROP_GETPRI: 1403 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1404 return (DDI_FAILURE); 1405 *(int *)result = ispec->intrspec_pri; 1406 break; 1407 case DDI_INTROP_SETPRI: 1408 /* Validate the interrupt priority passed to us */ 1409 if (*(int *)result > LOCK_LEVEL) 1410 return (DDI_FAILURE); 1411 1412 /* Ensure that PSM is all initialized and ispec is ok */ 1413 if ((psm_intr_ops == NULL) || 1414 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1415 return (DDI_FAILURE); 1416 1417 /* Change the priority */ 1418 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1419 PSM_FAILURE) 1420 return (DDI_FAILURE); 1421 1422 /* update the ispec with the new priority */ 1423 ispec->intrspec_pri = *(int *)result; 1424 break; 1425 case DDI_INTROP_ADDISR: 1426 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1427 return (DDI_FAILURE); 1428 ispec->intrspec_func = hdlp->ih_cb_func; 1429 break; 1430 case DDI_INTROP_REMISR: 1431 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1432 return (DDI_FAILURE); 1433 ispec->intrspec_func = (uint_t (*)()) 0; 1434 break; 1435 case DDI_INTROP_ENABLE: 1436 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1437 return (DDI_FAILURE); 1438 1439 /* Call psmi to translate irq with the dip */ 1440 if (psm_intr_ops == NULL) 1441 return (DDI_FAILURE); 1442 1443 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1444 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1445 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1446 return (DDI_FAILURE); 1447 1448 /* Add the interrupt handler */ 1449 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1450 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1451 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1452 return (DDI_FAILURE); 1453 break; 1454 case DDI_INTROP_DISABLE: 1455 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1456 return (DDI_FAILURE); 1457 1458 /* Call psm_ops() to translate irq with the dip */ 1459 if (psm_intr_ops == NULL) 1460 return (DDI_FAILURE); 1461 1462 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1463 (void) (*psm_intr_ops)(rdip, hdlp, 1464 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1465 1466 /* Remove the interrupt handler */ 1467 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1468 hdlp->ih_cb_func, hdlp->ih_vector); 1469 break; 1470 case DDI_INTROP_SETMASK: 1471 if (psm_intr_ops == NULL) 1472 return (DDI_FAILURE); 1473 1474 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1475 return (DDI_FAILURE); 1476 break; 1477 case DDI_INTROP_CLRMASK: 1478 if (psm_intr_ops == NULL) 1479 return (DDI_FAILURE); 1480 1481 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1482 return (DDI_FAILURE); 1483 break; 1484 case DDI_INTROP_GETPENDING: 1485 if (psm_intr_ops == NULL) 1486 return (DDI_FAILURE); 1487 1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1489 result)) { 1490 *(int *)result = 0; 1491 return (DDI_FAILURE); 1492 } 1493 break; 1494 case DDI_INTROP_NAVAIL: 1495 case DDI_INTROP_NINTRS: 1496 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1497 if (*(int *)result == 0) { 1498 /* 1499 * Special case for 'pcic' driver' only. This driver 1500 * driver is a child of 'isa' and 'rootnex' drivers. 1501 * 1502 * See detailed comments on this in the function 1503 * rootnex_get_ispec(). 1504 * 1505 * Children of 'pcic' send 'NINITR' request all the 1506 * way to rootnex driver. But, the 'pdp->par_nintr' 1507 * field may not initialized. So, we fake it here 1508 * to return 1 (a la what PCMCIA nexus does). 1509 */ 1510 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1511 *(int *)result = 1; 1512 else 1513 return (DDI_FAILURE); 1514 } 1515 break; 1516 case DDI_INTROP_SUPPORTED_TYPES: 1517 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1518 break; 1519 default: 1520 return (DDI_FAILURE); 1521 } 1522 1523 return (DDI_SUCCESS); 1524 } 1525 1526 1527 /* 1528 * rootnex_get_ispec() 1529 * convert an interrupt number to an interrupt specification. 1530 * The interrupt number determines which interrupt spec will be 1531 * returned if more than one exists. 1532 * 1533 * Look into the parent private data area of the 'rdip' to find out 1534 * the interrupt specification. First check to make sure there is 1535 * one that matchs "inumber" and then return a pointer to it. 1536 * 1537 * Return NULL if one could not be found. 1538 * 1539 * NOTE: This is needed for rootnex_intr_ops() 1540 */ 1541 static struct intrspec * 1542 rootnex_get_ispec(dev_info_t *rdip, int inum) 1543 { 1544 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1545 1546 /* 1547 * Special case handling for drivers that provide their own 1548 * intrspec structures instead of relying on the DDI framework. 1549 * 1550 * A broken hardware driver in ON could potentially provide its 1551 * own intrspec structure, instead of relying on the hardware. 1552 * If these drivers are children of 'rootnex' then we need to 1553 * continue to provide backward compatibility to them here. 1554 * 1555 * Following check is a special case for 'pcic' driver which 1556 * was found to have broken hardwre andby provides its own intrspec. 1557 * 1558 * Verbatim comments from this driver are shown here: 1559 * "Don't use the ddi_add_intr since we don't have a 1560 * default intrspec in all cases." 1561 * 1562 * Since an 'ispec' may not be always created for it, 1563 * check for that and create one if so. 1564 * 1565 * NOTE: Currently 'pcic' is the only driver found to do this. 1566 */ 1567 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1568 pdp->par_nintr = 1; 1569 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1570 pdp->par_nintr, KM_SLEEP); 1571 } 1572 1573 /* Validate the interrupt number */ 1574 if (inum >= pdp->par_nintr) 1575 return (NULL); 1576 1577 /* Get the interrupt structure pointer and return that */ 1578 return ((struct intrspec *)&pdp->par_intr[inum]); 1579 } 1580 1581 /* 1582 * Allocate interrupt vector for FIXED (legacy) type. 1583 */ 1584 static int 1585 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1586 void *result) 1587 { 1588 struct intrspec *ispec; 1589 ddi_intr_handle_impl_t info_hdl; 1590 int ret; 1591 int free_phdl = 0; 1592 apic_get_type_t type_info; 1593 1594 if (psm_intr_ops == NULL) 1595 return (DDI_FAILURE); 1596 1597 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1598 return (DDI_FAILURE); 1599 1600 /* 1601 * If the PSM module is "APIX" then pass the request for it 1602 * to allocate the vector now. 1603 */ 1604 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1605 info_hdl.ih_private = &type_info; 1606 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1607 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1608 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1609 free_phdl = 1; 1610 i_ddi_alloc_intr_phdl(hdlp); 1611 } 1612 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1613 ret = (*psm_intr_ops)(rdip, hdlp, 1614 PSM_INTR_OP_ALLOC_VECTORS, result); 1615 if (free_phdl) { /* free up the phdl structure */ 1616 free_phdl = 0; 1617 i_ddi_free_intr_phdl(hdlp); 1618 hdlp->ih_private = NULL; 1619 } 1620 } else { 1621 /* 1622 * No APIX module; fall back to the old scheme where the 1623 * interrupt vector is allocated during ddi_intr_enable() call. 1624 */ 1625 hdlp->ih_pri = ispec->intrspec_pri; 1626 *(int *)result = hdlp->ih_scratch1; 1627 ret = DDI_SUCCESS; 1628 } 1629 1630 return (ret); 1631 } 1632 1633 /* 1634 * Free up interrupt vector for FIXED (legacy) type. 1635 */ 1636 static int 1637 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1638 { 1639 struct intrspec *ispec; 1640 struct ddi_parent_private_data *pdp; 1641 ddi_intr_handle_impl_t info_hdl; 1642 int ret; 1643 apic_get_type_t type_info; 1644 1645 if (psm_intr_ops == NULL) 1646 return (DDI_FAILURE); 1647 1648 /* 1649 * If the PSM module is "APIX" then pass the request for it 1650 * to free up the vector now. 1651 */ 1652 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1653 info_hdl.ih_private = &type_info; 1654 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1655 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1656 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1657 return (DDI_FAILURE); 1658 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1659 ret = (*psm_intr_ops)(rdip, hdlp, 1660 PSM_INTR_OP_FREE_VECTORS, NULL); 1661 } else { 1662 /* 1663 * No APIX module; fall back to the old scheme where 1664 * the interrupt vector was already freed during 1665 * ddi_intr_disable() call. 1666 */ 1667 ret = DDI_SUCCESS; 1668 } 1669 1670 pdp = ddi_get_parent_data(rdip); 1671 1672 /* 1673 * Special case for 'pcic' driver' only. 1674 * If an intrspec was created for it, clean it up here 1675 * See detailed comments on this in the function 1676 * rootnex_get_ispec(). 1677 */ 1678 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1679 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1680 pdp->par_nintr); 1681 /* 1682 * Set it to zero; so that 1683 * DDI framework doesn't free it again 1684 */ 1685 pdp->par_intr = NULL; 1686 pdp->par_nintr = 0; 1687 } 1688 1689 return (ret); 1690 } 1691 1692 1693 /* 1694 * ****************** 1695 * dma related code 1696 * ****************** 1697 */ 1698 1699 /*ARGSUSED*/ 1700 static int 1701 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1702 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1703 ddi_dma_handle_t *handlep) 1704 { 1705 uint64_t maxsegmentsize_ll; 1706 uint_t maxsegmentsize; 1707 ddi_dma_impl_t *hp; 1708 rootnex_dma_t *dma; 1709 uint64_t count_max; 1710 uint64_t seg; 1711 int kmflag; 1712 int e; 1713 1714 1715 /* convert our sleep flags */ 1716 if (waitfp == DDI_DMA_SLEEP) { 1717 kmflag = KM_SLEEP; 1718 } else { 1719 kmflag = KM_NOSLEEP; 1720 } 1721 1722 /* 1723 * We try to do only one memory allocation here. We'll do a little 1724 * pointer manipulation later. If the bind ends up taking more than 1725 * our prealloc's space, we'll have to allocate more memory in the 1726 * bind operation. Not great, but much better than before and the 1727 * best we can do with the current bind interfaces. 1728 */ 1729 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1730 if (hp == NULL) 1731 return (DDI_DMA_NORESOURCES); 1732 1733 /* Do our pointer manipulation now, align the structures */ 1734 hp->dmai_private = (void *)(((uintptr_t)hp + 1735 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1736 dma = (rootnex_dma_t *)hp->dmai_private; 1737 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1738 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1739 1740 /* setup the handle */ 1741 rootnex_clean_dmahdl(hp); 1742 hp->dmai_error.err_fep = NULL; 1743 hp->dmai_error.err_cf = NULL; 1744 dma->dp_dip = rdip; 1745 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1746 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1747 1748 /* 1749 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1750 * is being used. Set the upper limit to the seg value. 1751 * There will be enough DVMA space to always get addresses 1752 * that will match the constraints. 1753 */ 1754 if (IOMMU_USED(rdip) && 1755 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1756 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1757 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1758 } else 1759 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1760 1761 hp->dmai_minxfer = attr->dma_attr_minxfer; 1762 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1763 hp->dmai_rdip = rdip; 1764 hp->dmai_attr = *attr; 1765 1766 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1767 dma->dp_sglinfo.si_cancross = B_FALSE; 1768 else 1769 dma->dp_sglinfo.si_cancross = B_TRUE; 1770 1771 /* we don't need to worry about the SPL since we do a tryenter */ 1772 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1773 1774 /* 1775 * Figure out our maximum segment size. If the segment size is greater 1776 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1777 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1778 * dma_attr_count_max are size-1 type values. 1779 * 1780 * Maximum segment size is the largest physically contiguous chunk of 1781 * memory that we can return from a bind (i.e. the maximum size of a 1782 * single cookie). 1783 */ 1784 1785 /* handle the rollover cases */ 1786 seg = attr->dma_attr_seg + 1; 1787 if (seg < attr->dma_attr_seg) { 1788 seg = attr->dma_attr_seg; 1789 } 1790 count_max = attr->dma_attr_count_max + 1; 1791 if (count_max < attr->dma_attr_count_max) { 1792 count_max = attr->dma_attr_count_max; 1793 } 1794 1795 /* 1796 * granularity may or may not be a power of two. If it isn't, we can't 1797 * use a simple mask. 1798 */ 1799 if (!ISP2(attr->dma_attr_granular)) { 1800 dma->dp_granularity_power_2 = B_FALSE; 1801 } else { 1802 dma->dp_granularity_power_2 = B_TRUE; 1803 } 1804 1805 /* 1806 * maxxfer should be a whole multiple of granularity. If we're going to 1807 * break up a window because we're greater than maxxfer, we might as 1808 * well make sure it's maxxfer is a whole multiple so we don't have to 1809 * worry about triming the window later on for this case. 1810 */ 1811 if (attr->dma_attr_granular > 1) { 1812 if (dma->dp_granularity_power_2) { 1813 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1814 (attr->dma_attr_maxxfer & 1815 (attr->dma_attr_granular - 1)); 1816 } else { 1817 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1818 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1819 } 1820 } else { 1821 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1822 } 1823 1824 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1825 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1826 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1827 maxsegmentsize = 0xFFFFFFFF; 1828 } else { 1829 maxsegmentsize = maxsegmentsize_ll; 1830 } 1831 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1832 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1833 1834 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1835 if (rootnex_alloc_check_parms) { 1836 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1837 if (e != DDI_SUCCESS) { 1838 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1839 (void) rootnex_dma_freehdl(dip, rdip, 1840 (ddi_dma_handle_t)hp); 1841 return (e); 1842 } 1843 } 1844 1845 *handlep = (ddi_dma_handle_t)hp; 1846 1847 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1848 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1849 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1850 1851 return (DDI_SUCCESS); 1852 } 1853 1854 1855 /* 1856 * rootnex_dma_allochdl() 1857 * called from ddi_dma_alloc_handle(). 1858 */ 1859 static int 1860 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1861 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1862 { 1863 int retval = DDI_SUCCESS; 1864 #if !defined(__xpv) 1865 1866 if (IOMMU_UNITIALIZED(rdip)) { 1867 retval = iommulib_nex_open(dip, rdip); 1868 1869 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1870 return (retval); 1871 } 1872 1873 if (IOMMU_UNUSED(rdip)) { 1874 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1875 handlep); 1876 } else { 1877 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1878 waitfp, arg, handlep); 1879 } 1880 #else 1881 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1882 handlep); 1883 #endif 1884 switch (retval) { 1885 case DDI_DMA_NORESOURCES: 1886 if (waitfp != DDI_DMA_DONTWAIT) { 1887 ddi_set_callback(waitfp, arg, 1888 &rootnex_state->r_dvma_call_list_id); 1889 } 1890 break; 1891 case DDI_SUCCESS: 1892 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1893 break; 1894 default: 1895 break; 1896 } 1897 return (retval); 1898 } 1899 1900 /*ARGSUSED*/ 1901 static int 1902 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1903 ddi_dma_handle_t handle) 1904 { 1905 ddi_dma_impl_t *hp; 1906 rootnex_dma_t *dma; 1907 1908 1909 hp = (ddi_dma_impl_t *)handle; 1910 dma = (rootnex_dma_t *)hp->dmai_private; 1911 1912 /* unbind should have been called first */ 1913 ASSERT(!dma->dp_inuse); 1914 1915 mutex_destroy(&dma->dp_mutex); 1916 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1917 1918 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1919 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1920 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1921 1922 return (DDI_SUCCESS); 1923 } 1924 1925 /* 1926 * rootnex_dma_freehdl() 1927 * called from ddi_dma_free_handle(). 1928 */ 1929 static int 1930 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1931 { 1932 int ret; 1933 1934 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1935 #if !defined(__xpv) 1936 if (IOMMU_USED(rdip)) 1937 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1938 else 1939 #endif 1940 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1941 1942 if (rootnex_state->r_dvma_call_list_id) 1943 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1944 1945 return (ret); 1946 } 1947 1948 /*ARGSUSED*/ 1949 static int 1950 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1951 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1952 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1953 { 1954 rootnex_sglinfo_t *sinfo; 1955 ddi_dma_obj_t *dmao; 1956 #if !defined(__xpv) 1957 struct dvmaseg *dvs; 1958 ddi_dma_cookie_t *cookie; 1959 #endif 1960 ddi_dma_attr_t *attr; 1961 ddi_dma_impl_t *hp; 1962 rootnex_dma_t *dma; 1963 int kmflag; 1964 int e; 1965 uint_t ncookies; 1966 1967 hp = (ddi_dma_impl_t *)handle; 1968 dma = (rootnex_dma_t *)hp->dmai_private; 1969 dmao = &dma->dp_dma; 1970 sinfo = &dma->dp_sglinfo; 1971 attr = &hp->dmai_attr; 1972 1973 /* convert the sleep flags */ 1974 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1975 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1976 } else { 1977 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1978 } 1979 1980 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1981 1982 /* 1983 * This is useful for debugging a driver. Not as useful in a production 1984 * system. The only time this will fail is if you have a driver bug. 1985 */ 1986 if (rootnex_bind_check_inuse) { 1987 /* 1988 * No one else should ever have this lock unless someone else 1989 * is trying to use this handle. So contention on the lock 1990 * is the same as inuse being set. 1991 */ 1992 e = mutex_tryenter(&dma->dp_mutex); 1993 if (e == 0) { 1994 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1995 return (DDI_DMA_INUSE); 1996 } 1997 if (dma->dp_inuse) { 1998 mutex_exit(&dma->dp_mutex); 1999 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2000 return (DDI_DMA_INUSE); 2001 } 2002 dma->dp_inuse = B_TRUE; 2003 mutex_exit(&dma->dp_mutex); 2004 } 2005 2006 /* save away the original bind info */ 2007 dma->dp_dma = dmareq->dmar_object; 2008 2009 #if !defined(__xpv) 2010 if (IOMMU_USED(rdip)) { 2011 dmao = &dma->dp_dvma; 2012 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2013 switch (e) { 2014 case DDI_SUCCESS: 2015 if (sinfo->si_cancross || 2016 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2017 dmao->dmao_size > sinfo->si_max_cookie_size) { 2018 dma->dp_dvma_used = B_TRUE; 2019 break; 2020 } 2021 sinfo->si_sgl_size = 1; 2022 hp->dmai_rflags |= DMP_NOSYNC; 2023 2024 dma->dp_dvma_used = B_TRUE; 2025 dma->dp_need_to_free_cookie = B_FALSE; 2026 2027 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2028 cookie = hp->dmai_cookie = dma->dp_cookies = 2029 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2030 cookie->dmac_laddress = dvs->dvs_start + 2031 dmao->dmao_obj.dvma_obj.dv_off; 2032 cookie->dmac_size = dvs->dvs_len; 2033 cookie->dmac_type = 0; 2034 2035 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2036 rdip); 2037 goto fast; 2038 case DDI_ENOTSUP: 2039 break; 2040 default: 2041 rootnex_clean_dmahdl(hp); 2042 return (e); 2043 } 2044 } 2045 #endif 2046 2047 /* 2048 * Figure out a rough estimate of what maximum number of pages 2049 * this buffer could use (a high estimate of course). 2050 */ 2051 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2052 2053 if (dma->dp_dvma_used) { 2054 /* 2055 * The number of physical pages is the worst case. 2056 * 2057 * For DVMA, the worst case is the length divided 2058 * by the maximum cookie length, plus 1. Add to that 2059 * the number of segment boundaries potentially crossed, and 2060 * the additional number of DVMA segments that was returned. 2061 * 2062 * In the normal case, for modern devices, si_cancross will 2063 * be false, and dv_nseg will be 1, and the fast path will 2064 * have been taken above. 2065 */ 2066 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2067 + 1; 2068 if (sinfo->si_cancross) 2069 ncookies += 2070 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2071 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2072 2073 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2074 } 2075 2076 /* 2077 * We'll use the pre-allocated cookies for any bind that will *always* 2078 * fit (more important to be consistent, we don't want to create 2079 * additional degenerate cases). 2080 */ 2081 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2082 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2083 dma->dp_need_to_free_cookie = B_FALSE; 2084 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2085 uint_t, sinfo->si_max_pages); 2086 2087 /* 2088 * For anything larger than that, we'll go ahead and allocate the 2089 * maximum number of pages we expect to see. Hopefuly, we won't be 2090 * seeing this path in the fast path for high performance devices very 2091 * frequently. 2092 * 2093 * a ddi bind interface that allowed the driver to provide storage to 2094 * the bind interface would speed this case up. 2095 */ 2096 } else { 2097 /* 2098 * Save away how much memory we allocated. If we're doing a 2099 * nosleep, the alloc could fail... 2100 */ 2101 dma->dp_cookie_size = sinfo->si_max_pages * 2102 sizeof (ddi_dma_cookie_t); 2103 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2104 if (dma->dp_cookies == NULL) { 2105 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2106 rootnex_clean_dmahdl(hp); 2107 return (DDI_DMA_NORESOURCES); 2108 } 2109 dma->dp_need_to_free_cookie = B_TRUE; 2110 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2111 uint_t, sinfo->si_max_pages); 2112 } 2113 hp->dmai_cookie = dma->dp_cookies; 2114 2115 /* 2116 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2117 * looking at the constraints in the dma structure. It will then put 2118 * some additional state about the sgl in the dma struct (i.e. is 2119 * the sgl clean, or do we need to do some munging; how many pages 2120 * need to be copied, etc.) 2121 */ 2122 if (dma->dp_dvma_used) 2123 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2124 else 2125 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2126 2127 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2128 /* if we don't need a copy buffer, we don't need to sync */ 2129 if (sinfo->si_copybuf_req == 0) { 2130 hp->dmai_rflags |= DMP_NOSYNC; 2131 } 2132 2133 /* 2134 * if we don't need the copybuf and we don't need to do a partial, we 2135 * hit the fast path. All the high performance devices should be trying 2136 * to hit this path. To hit this path, a device should be able to reach 2137 * all of memory, shouldn't try to bind more than it can transfer, and 2138 * the buffer shouldn't require more cookies than the driver/device can 2139 * handle [sgllen]). 2140 * 2141 * Note that negative values of dma_attr_sgllen are supposed 2142 * to mean unlimited, but we just cast them to mean a 2143 * "ridiculous large limit". This saves some extra checks on 2144 * hot paths. 2145 */ 2146 if ((sinfo->si_copybuf_req == 0) && 2147 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2148 (dmao->dmao_size <= dma->dp_maxxfer)) { 2149 #if !defined(__xpv) 2150 fast: 2151 #endif 2152 /* 2153 * If the driver supports FMA, insert the handle in the FMA DMA 2154 * handle cache. 2155 */ 2156 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2157 hp->dmai_error.err_cf = rootnex_dma_check; 2158 2159 /* 2160 * copy out the first cookie and ccountp, set the cookie 2161 * pointer to the second cookie. The first cookie is passed 2162 * back on the stack. Additional cookies are accessed via 2163 * ddi_dma_nextcookie() 2164 */ 2165 *cookiep = dma->dp_cookies[0]; 2166 *ccountp = sinfo->si_sgl_size; 2167 hp->dmai_cookie++; 2168 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2169 hp->dmai_ncookies = *ccountp; 2170 hp->dmai_curcookie = 1; 2171 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2172 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2173 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2174 uint_t, dmao->dmao_size, uint_t, *ccountp); 2175 2176 2177 return (DDI_DMA_MAPPED); 2178 } 2179 2180 /* 2181 * go to the slow path, we may need to alloc more memory, create 2182 * multiple windows, and munge up a sgl to make the device happy. 2183 */ 2184 2185 /* 2186 * With the IOMMU mapobject method used, we should never hit 2187 * the slow path. If we do, something is seriously wrong. 2188 * Clean up and return an error. 2189 */ 2190 2191 #if !defined(__xpv) 2192 2193 if (dma->dp_dvma_used) { 2194 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2195 &dma->dp_dvma); 2196 e = DDI_DMA_NOMAPPING; 2197 } else { 2198 #endif 2199 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2200 kmflag); 2201 #if !defined(__xpv) 2202 } 2203 #endif 2204 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2205 if (dma->dp_need_to_free_cookie) { 2206 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2207 } 2208 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2209 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2210 return (e); 2211 } 2212 2213 /* 2214 * If the driver supports FMA, insert the handle in the FMA DMA handle 2215 * cache. 2216 */ 2217 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2218 hp->dmai_error.err_cf = rootnex_dma_check; 2219 2220 /* if the first window uses the copy buffer, sync it for the device */ 2221 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2222 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2223 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2224 DDI_DMA_SYNC_FORDEV); 2225 } 2226 2227 /* 2228 * copy out the first cookie and ccountp, set the cookie pointer to the 2229 * second cookie. Make sure the partial flag is set/cleared correctly. 2230 * If we have a partial map (i.e. multiple windows), the number of 2231 * cookies we return is the number of cookies in the first window. 2232 */ 2233 if (e == DDI_DMA_MAPPED) { 2234 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2235 *ccountp = sinfo->si_sgl_size; 2236 hp->dmai_nwin = 1; 2237 } else { 2238 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2239 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2240 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2241 } 2242 *cookiep = dma->dp_cookies[0]; 2243 hp->dmai_cookie++; 2244 hp->dmai_ncookies = *ccountp; 2245 hp->dmai_curcookie = 1; 2246 2247 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2248 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2249 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2250 dmao->dmao_size, uint_t, *ccountp); 2251 return (e); 2252 } 2253 2254 /* 2255 * rootnex_dma_bindhdl() 2256 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2257 */ 2258 static int 2259 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2260 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2261 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2262 { 2263 int ret; 2264 #if !defined(__xpv) 2265 if (IOMMU_USED(rdip)) 2266 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2267 cookiep, ccountp); 2268 else 2269 #endif 2270 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2271 cookiep, ccountp); 2272 2273 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2274 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2275 &rootnex_state->r_dvma_call_list_id); 2276 } 2277 2278 return (ret); 2279 } 2280 2281 2282 2283 /*ARGSUSED*/ 2284 static int 2285 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2286 ddi_dma_handle_t handle) 2287 { 2288 ddi_dma_impl_t *hp; 2289 rootnex_dma_t *dma; 2290 int e; 2291 2292 hp = (ddi_dma_impl_t *)handle; 2293 dma = (rootnex_dma_t *)hp->dmai_private; 2294 2295 /* make sure the buffer wasn't free'd before calling unbind */ 2296 if (rootnex_unbind_verify_buffer) { 2297 e = rootnex_verify_buffer(dma); 2298 if (e != DDI_SUCCESS) { 2299 ASSERT(0); 2300 return (DDI_FAILURE); 2301 } 2302 } 2303 2304 /* sync the current window before unbinding the buffer */ 2305 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2306 (hp->dmai_rflags & DDI_DMA_READ)) { 2307 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2308 DDI_DMA_SYNC_FORCPU); 2309 } 2310 2311 /* 2312 * cleanup and copy buffer or window state. if we didn't use the copy 2313 * buffer or windows, there won't be much to do :-) 2314 */ 2315 rootnex_teardown_copybuf(dma); 2316 rootnex_teardown_windows(dma); 2317 2318 #if !defined(__xpv) 2319 if (IOMMU_USED(rdip) && dma->dp_dvma_used) 2320 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2321 &dma->dp_dvma); 2322 #endif 2323 2324 /* 2325 * If we had to allocate space to for the worse case sgl (it didn't 2326 * fit into our pre-allocate buffer), free that up now 2327 */ 2328 if (dma->dp_need_to_free_cookie) { 2329 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2330 } 2331 2332 /* 2333 * clean up the handle so it's ready for the next bind (i.e. if the 2334 * handle is reused). 2335 */ 2336 rootnex_clean_dmahdl(hp); 2337 hp->dmai_error.err_cf = NULL; 2338 2339 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2340 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2341 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2342 2343 return (DDI_SUCCESS); 2344 } 2345 2346 /* 2347 * rootnex_dma_unbindhdl() 2348 * called from ddi_dma_unbind_handle() 2349 */ 2350 /*ARGSUSED*/ 2351 static int 2352 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2353 ddi_dma_handle_t handle) 2354 { 2355 int ret; 2356 2357 #if !defined(__xpv) 2358 if (IOMMU_USED(rdip)) 2359 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2360 else 2361 #endif 2362 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2363 2364 if (rootnex_state->r_dvma_call_list_id) 2365 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2366 2367 return (ret); 2368 } 2369 2370 #if !defined(__xpv) 2371 2372 static int 2373 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2374 { 2375 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2376 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2377 2378 if (dma->dp_sleep_flags != KM_SLEEP && 2379 dma->dp_sleep_flags != KM_NOSLEEP) 2380 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2381 return (dma->dp_sleep_flags); 2382 } 2383 /*ARGSUSED*/ 2384 static void 2385 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2386 { 2387 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2388 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2389 rootnex_window_t *window; 2390 2391 if (dma->dp_window) { 2392 window = &dma->dp_window[dma->dp_current_win]; 2393 hp->dmai_cookie = window->wd_first_cookie; 2394 } else { 2395 hp->dmai_cookie = dma->dp_cookies; 2396 } 2397 hp->dmai_cookie++; 2398 hp->dmai_curcookie = 1; 2399 } 2400 2401 /*ARGSUSED*/ 2402 static int 2403 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2404 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2405 { 2406 int i; 2407 int km_flags; 2408 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2409 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2410 rootnex_window_t *window; 2411 ddi_dma_cookie_t *cp; 2412 ddi_dma_cookie_t *cookie; 2413 2414 ASSERT(*cookiepp == NULL); 2415 ASSERT(*ccountp == 0); 2416 2417 if (dma->dp_window) { 2418 window = &dma->dp_window[dma->dp_current_win]; 2419 cp = window->wd_first_cookie; 2420 *ccountp = window->wd_cookie_cnt; 2421 } else { 2422 cp = dma->dp_cookies; 2423 *ccountp = dma->dp_sglinfo.si_sgl_size; 2424 } 2425 2426 km_flags = rootnex_coredma_get_sleep_flags(handle); 2427 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2428 if (cookie == NULL) { 2429 return (DDI_DMA_NORESOURCES); 2430 } 2431 2432 for (i = 0; i < *ccountp; i++) { 2433 cookie[i].dmac_notused = cp[i].dmac_notused; 2434 cookie[i].dmac_type = cp[i].dmac_type; 2435 cookie[i].dmac_address = cp[i].dmac_address; 2436 cookie[i].dmac_size = cp[i].dmac_size; 2437 } 2438 2439 *cookiepp = cookie; 2440 2441 return (DDI_SUCCESS); 2442 } 2443 2444 /*ARGSUSED*/ 2445 static int 2446 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2447 ddi_dma_cookie_t *cookiep, uint_t ccount) 2448 { 2449 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2450 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2451 rootnex_window_t *window; 2452 ddi_dma_cookie_t *cur_cookiep; 2453 2454 ASSERT(cookiep); 2455 ASSERT(ccount != 0); 2456 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2457 2458 if (dma->dp_window) { 2459 window = &dma->dp_window[dma->dp_current_win]; 2460 dma->dp_saved_cookies = window->wd_first_cookie; 2461 window->wd_first_cookie = cookiep; 2462 ASSERT(ccount == window->wd_cookie_cnt); 2463 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2464 + window->wd_first_cookie; 2465 } else { 2466 dma->dp_saved_cookies = dma->dp_cookies; 2467 dma->dp_cookies = cookiep; 2468 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2469 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2470 + dma->dp_cookies; 2471 } 2472 2473 dma->dp_need_to_switch_cookies = B_TRUE; 2474 hp->dmai_cookie = cur_cookiep; 2475 2476 return (DDI_SUCCESS); 2477 } 2478 2479 /*ARGSUSED*/ 2480 static int 2481 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2482 { 2483 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2484 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2485 rootnex_window_t *window; 2486 ddi_dma_cookie_t *cur_cookiep; 2487 ddi_dma_cookie_t *cookie_array; 2488 uint_t ccount; 2489 2490 /* check if cookies have not been switched */ 2491 if (dma->dp_need_to_switch_cookies == B_FALSE) 2492 return (DDI_SUCCESS); 2493 2494 ASSERT(dma->dp_saved_cookies); 2495 2496 if (dma->dp_window) { 2497 window = &dma->dp_window[dma->dp_current_win]; 2498 cookie_array = window->wd_first_cookie; 2499 window->wd_first_cookie = dma->dp_saved_cookies; 2500 dma->dp_saved_cookies = NULL; 2501 ccount = window->wd_cookie_cnt; 2502 cur_cookiep = (hp->dmai_cookie - cookie_array) 2503 + window->wd_first_cookie; 2504 } else { 2505 cookie_array = dma->dp_cookies; 2506 dma->dp_cookies = dma->dp_saved_cookies; 2507 dma->dp_saved_cookies = NULL; 2508 ccount = dma->dp_sglinfo.si_sgl_size; 2509 cur_cookiep = (hp->dmai_cookie - cookie_array) 2510 + dma->dp_cookies; 2511 } 2512 2513 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2514 2515 hp->dmai_cookie = cur_cookiep; 2516 2517 dma->dp_need_to_switch_cookies = B_FALSE; 2518 2519 return (DDI_SUCCESS); 2520 } 2521 2522 #endif 2523 2524 static struct as * 2525 rootnex_get_as(ddi_dma_obj_t *dmao) 2526 { 2527 struct as *asp; 2528 2529 switch (dmao->dmao_type) { 2530 case DMA_OTYP_VADDR: 2531 case DMA_OTYP_BUFVADDR: 2532 asp = dmao->dmao_obj.virt_obj.v_as; 2533 if (asp == NULL) 2534 asp = &kas; 2535 break; 2536 default: 2537 asp = NULL; 2538 break; 2539 } 2540 return (asp); 2541 } 2542 2543 /* 2544 * rootnex_verify_buffer() 2545 * verify buffer wasn't free'd 2546 */ 2547 static int 2548 rootnex_verify_buffer(rootnex_dma_t *dma) 2549 { 2550 page_t **pplist; 2551 caddr_t vaddr; 2552 uint_t pcnt; 2553 uint_t poff; 2554 page_t *pp; 2555 char b; 2556 int i; 2557 2558 /* Figure out how many pages this buffer occupies */ 2559 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2560 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2561 } else { 2562 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2563 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2564 } 2565 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2566 2567 switch (dma->dp_dma.dmao_type) { 2568 case DMA_OTYP_PAGES: 2569 /* 2570 * for a linked list of pp's walk through them to make sure 2571 * they're locked and not free. 2572 */ 2573 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2574 for (i = 0; i < pcnt; i++) { 2575 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2576 return (DDI_FAILURE); 2577 } 2578 pp = pp->p_next; 2579 } 2580 break; 2581 2582 case DMA_OTYP_VADDR: 2583 case DMA_OTYP_BUFVADDR: 2584 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2585 /* 2586 * for an array of pp's walk through them to make sure they're 2587 * not free. It's possible that they may not be locked. 2588 */ 2589 if (pplist) { 2590 for (i = 0; i < pcnt; i++) { 2591 if (PP_ISFREE(pplist[i])) { 2592 return (DDI_FAILURE); 2593 } 2594 } 2595 2596 /* For a virtual address, try to peek at each page */ 2597 } else { 2598 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2599 for (i = 0; i < pcnt; i++) { 2600 if (ddi_peek8(NULL, vaddr, &b) == 2601 DDI_FAILURE) 2602 return (DDI_FAILURE); 2603 vaddr += MMU_PAGESIZE; 2604 } 2605 } 2606 } 2607 break; 2608 2609 default: 2610 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2611 break; 2612 } 2613 2614 return (DDI_SUCCESS); 2615 } 2616 2617 2618 /* 2619 * rootnex_clean_dmahdl() 2620 * Clean the dma handle. This should be called on a handle alloc and an 2621 * unbind handle. Set the handle state to the default settings. 2622 */ 2623 static void 2624 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2625 { 2626 rootnex_dma_t *dma; 2627 2628 2629 dma = (rootnex_dma_t *)hp->dmai_private; 2630 2631 hp->dmai_nwin = 0; 2632 dma->dp_current_cookie = 0; 2633 dma->dp_copybuf_size = 0; 2634 dma->dp_window = NULL; 2635 dma->dp_cbaddr = NULL; 2636 dma->dp_inuse = B_FALSE; 2637 dma->dp_dvma_used = B_FALSE; 2638 dma->dp_need_to_free_cookie = B_FALSE; 2639 dma->dp_need_to_switch_cookies = B_FALSE; 2640 dma->dp_saved_cookies = NULL; 2641 dma->dp_sleep_flags = KM_PANIC; 2642 dma->dp_need_to_free_window = B_FALSE; 2643 dma->dp_partial_required = B_FALSE; 2644 dma->dp_trim_required = B_FALSE; 2645 dma->dp_sglinfo.si_copybuf_req = 0; 2646 2647 /* FMA related initialization */ 2648 hp->dmai_fault = 0; 2649 hp->dmai_fault_check = NULL; 2650 hp->dmai_fault_notify = NULL; 2651 hp->dmai_error.err_ena = 0; 2652 hp->dmai_error.err_status = DDI_FM_OK; 2653 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2654 hp->dmai_error.err_ontrap = NULL; 2655 2656 /* Cookie tracking */ 2657 hp->dmai_ncookies = 0; 2658 hp->dmai_curcookie = 0; 2659 } 2660 2661 2662 /* 2663 * rootnex_valid_alloc_parms() 2664 * Called in ddi_dma_alloc_handle path to validate its parameters. 2665 */ 2666 static int 2667 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2668 { 2669 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2670 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2671 (attr->dma_attr_granular > MMU_PAGESIZE) || 2672 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2673 return (DDI_DMA_BADATTR); 2674 } 2675 2676 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2677 return (DDI_DMA_BADATTR); 2678 } 2679 2680 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2681 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2682 attr->dma_attr_sgllen == 0) { 2683 return (DDI_DMA_BADATTR); 2684 } 2685 2686 /* We should be able to DMA into every byte offset in a page */ 2687 if (maxsegmentsize < MMU_PAGESIZE) { 2688 return (DDI_DMA_BADATTR); 2689 } 2690 2691 /* if we're bouncing on seg, seg must be <= addr_hi */ 2692 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2693 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2694 return (DDI_DMA_BADATTR); 2695 } 2696 return (DDI_SUCCESS); 2697 } 2698 2699 /* 2700 * rootnex_need_bounce_seg() 2701 * check to see if the buffer lives on both side of the seg. 2702 */ 2703 static boolean_t 2704 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2705 { 2706 ddi_dma_atyp_t buftype; 2707 rootnex_addr_t raddr; 2708 boolean_t lower_addr; 2709 boolean_t upper_addr; 2710 uint64_t offset; 2711 page_t **pplist; 2712 uint64_t paddr; 2713 uint32_t psize; 2714 uint32_t size; 2715 caddr_t vaddr; 2716 uint_t pcnt; 2717 page_t *pp; 2718 2719 pp = NULL; 2720 /* shortcuts */ 2721 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2722 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2723 buftype = dmar_object->dmao_type; 2724 size = dmar_object->dmao_size; 2725 2726 lower_addr = B_FALSE; 2727 upper_addr = B_FALSE; 2728 pcnt = 0; 2729 2730 /* 2731 * Process the first page to handle the initial offset of the buffer. 2732 * We'll use the base address we get later when we loop through all 2733 * the pages. 2734 */ 2735 if (buftype == DMA_OTYP_PAGES) { 2736 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2737 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2738 MMU_PAGEOFFSET; 2739 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2740 psize = MIN(size, (MMU_PAGESIZE - offset)); 2741 pp = pp->p_next; 2742 sglinfo->si_asp = NULL; 2743 } else if (pplist != NULL) { 2744 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2745 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2746 if (sglinfo->si_asp == NULL) { 2747 sglinfo->si_asp = &kas; 2748 } 2749 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2750 paddr += offset; 2751 psize = MIN(size, (MMU_PAGESIZE - offset)); 2752 pcnt++; 2753 } else { 2754 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2755 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2756 if (sglinfo->si_asp == NULL) { 2757 sglinfo->si_asp = &kas; 2758 } 2759 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2760 paddr += offset; 2761 psize = MIN(size, (MMU_PAGESIZE - offset)); 2762 vaddr += psize; 2763 } 2764 2765 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2766 2767 if ((raddr + psize) > sglinfo->si_segmask) { 2768 upper_addr = B_TRUE; 2769 } else { 2770 lower_addr = B_TRUE; 2771 } 2772 size -= psize; 2773 2774 /* 2775 * Walk through the rest of the pages in the buffer. Track to see 2776 * if we have pages on both sides of the segment boundary. 2777 */ 2778 while (size > 0) { 2779 /* partial or full page */ 2780 psize = MIN(size, MMU_PAGESIZE); 2781 2782 if (buftype == DMA_OTYP_PAGES) { 2783 /* get the paddr from the page_t */ 2784 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2785 paddr = pfn_to_pa(pp->p_pagenum); 2786 pp = pp->p_next; 2787 } else if (pplist != NULL) { 2788 /* index into the array of page_t's to get the paddr */ 2789 ASSERT(!PP_ISFREE(pplist[pcnt])); 2790 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2791 pcnt++; 2792 } else { 2793 /* call into the VM to get the paddr */ 2794 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2795 vaddr)); 2796 vaddr += psize; 2797 } 2798 2799 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2800 2801 if ((raddr + psize) > sglinfo->si_segmask) { 2802 upper_addr = B_TRUE; 2803 } else { 2804 lower_addr = B_TRUE; 2805 } 2806 /* 2807 * if the buffer lives both above and below the segment 2808 * boundary, or the current page is the page immediately 2809 * after the segment, we will use a copy/bounce buffer for 2810 * all pages > seg. 2811 */ 2812 if ((lower_addr && upper_addr) || 2813 (raddr == (sglinfo->si_segmask + 1))) { 2814 return (B_TRUE); 2815 } 2816 2817 size -= psize; 2818 } 2819 2820 return (B_FALSE); 2821 } 2822 2823 /* 2824 * rootnex_get_sgl() 2825 * Called in bind fastpath to get the sgl. Most of this will be replaced 2826 * with a call to the vm layer when vm2.0 comes around... 2827 */ 2828 static void 2829 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2830 rootnex_sglinfo_t *sglinfo) 2831 { 2832 ddi_dma_atyp_t buftype; 2833 rootnex_addr_t raddr; 2834 uint64_t last_page; 2835 uint64_t offset; 2836 uint64_t addrhi; 2837 uint64_t addrlo; 2838 uint64_t maxseg; 2839 page_t **pplist; 2840 uint64_t paddr; 2841 uint32_t psize; 2842 uint32_t size; 2843 caddr_t vaddr; 2844 uint_t pcnt; 2845 page_t *pp; 2846 uint_t cnt; 2847 2848 pp = NULL; 2849 /* shortcuts */ 2850 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2851 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2852 maxseg = sglinfo->si_max_cookie_size; 2853 buftype = dmar_object->dmao_type; 2854 addrhi = sglinfo->si_max_addr; 2855 addrlo = sglinfo->si_min_addr; 2856 size = dmar_object->dmao_size; 2857 2858 pcnt = 0; 2859 cnt = 0; 2860 2861 2862 /* 2863 * check to see if we need to use the copy buffer for pages over 2864 * the segment attr. 2865 */ 2866 sglinfo->si_bounce_on_seg = B_FALSE; 2867 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2868 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2869 dmar_object, sglinfo); 2870 } 2871 2872 /* 2873 * if we were passed down a linked list of pages, i.e. pointer to 2874 * page_t, use this to get our physical address and buf offset. 2875 */ 2876 if (buftype == DMA_OTYP_PAGES) { 2877 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2878 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2879 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2880 MMU_PAGEOFFSET; 2881 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2882 psize = MIN(size, (MMU_PAGESIZE - offset)); 2883 pp = pp->p_next; 2884 sglinfo->si_asp = NULL; 2885 2886 /* 2887 * We weren't passed down a linked list of pages, but if we were passed 2888 * down an array of pages, use this to get our physical address and buf 2889 * offset. 2890 */ 2891 } else if (pplist != NULL) { 2892 ASSERT((buftype == DMA_OTYP_VADDR) || 2893 (buftype == DMA_OTYP_BUFVADDR)); 2894 2895 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2896 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2897 if (sglinfo->si_asp == NULL) { 2898 sglinfo->si_asp = &kas; 2899 } 2900 2901 ASSERT(!PP_ISFREE(pplist[pcnt])); 2902 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2903 paddr += offset; 2904 psize = MIN(size, (MMU_PAGESIZE - offset)); 2905 pcnt++; 2906 2907 /* 2908 * All we have is a virtual address, we'll need to call into the VM 2909 * to get the physical address. 2910 */ 2911 } else { 2912 ASSERT((buftype == DMA_OTYP_VADDR) || 2913 (buftype == DMA_OTYP_BUFVADDR)); 2914 2915 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2916 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2917 if (sglinfo->si_asp == NULL) { 2918 sglinfo->si_asp = &kas; 2919 } 2920 2921 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2922 paddr += offset; 2923 psize = MIN(size, (MMU_PAGESIZE - offset)); 2924 vaddr += psize; 2925 } 2926 2927 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2928 2929 /* 2930 * Setup the first cookie with the physical address of the page and the 2931 * size of the page (which takes into account the initial offset into 2932 * the page. 2933 */ 2934 sgl[cnt].dmac_laddress = raddr; 2935 sgl[cnt].dmac_size = psize; 2936 sgl[cnt].dmac_type = 0; 2937 2938 /* 2939 * Save away the buffer offset into the page. We'll need this later in 2940 * the copy buffer code to help figure out the page index within the 2941 * buffer and the offset into the current page. 2942 */ 2943 sglinfo->si_buf_offset = offset; 2944 2945 /* 2946 * If we are using the copy buffer for anything over the segment 2947 * boundary, and this page is over the segment boundary. 2948 * OR 2949 * if the DMA engine can't reach the physical address. 2950 */ 2951 if (((sglinfo->si_bounce_on_seg) && 2952 ((raddr + psize) > sglinfo->si_segmask)) || 2953 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2954 /* 2955 * Increase how much copy buffer we use. We always increase by 2956 * pagesize so we don't have to worry about converting offsets. 2957 * Set a flag in the cookies dmac_type to indicate that it uses 2958 * the copy buffer. If this isn't the last cookie, go to the 2959 * next cookie (since we separate each page which uses the copy 2960 * buffer in case the copy buffer is not physically contiguous. 2961 */ 2962 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2963 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2964 if ((cnt + 1) < sglinfo->si_max_pages) { 2965 cnt++; 2966 sgl[cnt].dmac_laddress = 0; 2967 sgl[cnt].dmac_size = 0; 2968 sgl[cnt].dmac_type = 0; 2969 } 2970 } 2971 2972 /* 2973 * save this page's physical address so we can figure out if the next 2974 * page is physically contiguous. Keep decrementing size until we are 2975 * done with the buffer. 2976 */ 2977 last_page = raddr & MMU_PAGEMASK; 2978 size -= psize; 2979 2980 while (size > 0) { 2981 /* Get the size for this page (i.e. partial or full page) */ 2982 psize = MIN(size, MMU_PAGESIZE); 2983 2984 if (buftype == DMA_OTYP_PAGES) { 2985 /* get the paddr from the page_t */ 2986 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2987 paddr = pfn_to_pa(pp->p_pagenum); 2988 pp = pp->p_next; 2989 } else if (pplist != NULL) { 2990 /* index into the array of page_t's to get the paddr */ 2991 ASSERT(!PP_ISFREE(pplist[pcnt])); 2992 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2993 pcnt++; 2994 } else { 2995 /* call into the VM to get the paddr */ 2996 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2997 vaddr)); 2998 vaddr += psize; 2999 } 3000 3001 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3002 3003 /* 3004 * If we are using the copy buffer for anything over the 3005 * segment boundary, and this page is over the segment 3006 * boundary. 3007 * OR 3008 * if the DMA engine can't reach the physical address. 3009 */ 3010 if (((sglinfo->si_bounce_on_seg) && 3011 ((raddr + psize) > sglinfo->si_segmask)) || 3012 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3013 3014 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3015 3016 /* 3017 * if there is something in the current cookie, go to 3018 * the next one. We only want one page in a cookie which 3019 * uses the copybuf since the copybuf doesn't have to 3020 * be physically contiguous. 3021 */ 3022 if (sgl[cnt].dmac_size != 0) { 3023 cnt++; 3024 } 3025 sgl[cnt].dmac_laddress = raddr; 3026 sgl[cnt].dmac_size = psize; 3027 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3028 /* if this isn't the last cookie, go to the next one */ 3029 if ((cnt + 1) < sglinfo->si_max_pages) { 3030 cnt++; 3031 sgl[cnt].dmac_laddress = 0; 3032 sgl[cnt].dmac_size = 0; 3033 sgl[cnt].dmac_type = 0; 3034 } 3035 3036 /* 3037 * this page didn't need the copy buffer, if it's not physically 3038 * contiguous, or it would put us over a segment boundary, or it 3039 * puts us over the max cookie size, or the current sgl doesn't 3040 * have anything in it. 3041 */ 3042 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3043 !(raddr & sglinfo->si_segmask) || 3044 ((sgl[cnt].dmac_size + psize) > maxseg) || 3045 (sgl[cnt].dmac_size == 0)) { 3046 /* 3047 * if we're not already in a new cookie, go to the next 3048 * cookie. 3049 */ 3050 if (sgl[cnt].dmac_size != 0) { 3051 cnt++; 3052 } 3053 3054 /* save the cookie information */ 3055 sgl[cnt].dmac_laddress = raddr; 3056 sgl[cnt].dmac_size = psize; 3057 sgl[cnt].dmac_type = 0; 3058 3059 /* 3060 * this page didn't need the copy buffer, it is physically 3061 * contiguous with the last page, and it's <= the max cookie 3062 * size. 3063 */ 3064 } else { 3065 sgl[cnt].dmac_size += psize; 3066 3067 /* 3068 * If this cookie is used up, and more cookies 3069 * are available, then move onto the next one. 3070 */ 3071 if ((sgl[cnt].dmac_size == maxseg) && 3072 ((cnt + 1) < sglinfo->si_max_pages)) { 3073 cnt++; 3074 sgl[cnt].dmac_laddress = 0; 3075 sgl[cnt].dmac_size = 0; 3076 sgl[cnt].dmac_type = 0; 3077 } 3078 } 3079 3080 /* 3081 * save this page's physical address so we can figure out if the 3082 * next page is physically contiguous. Keep decrementing size 3083 * until we are done with the buffer. 3084 */ 3085 last_page = raddr; 3086 size -= psize; 3087 } 3088 3089 /* we're done, save away how many cookies the sgl has */ 3090 if (sgl[cnt].dmac_size == 0) { 3091 ASSERT(cnt < sglinfo->si_max_pages); 3092 sglinfo->si_sgl_size = cnt; 3093 } else { 3094 sglinfo->si_sgl_size = cnt + 1; 3095 } 3096 } 3097 3098 static void 3099 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3100 rootnex_sglinfo_t *sglinfo) 3101 { 3102 uint64_t offset; 3103 uint64_t maxseg; 3104 uint64_t dvaddr; 3105 struct dvmaseg *dvs; 3106 uint64_t paddr; 3107 uint32_t psize, ssize; 3108 uint32_t size; 3109 uint_t cnt; 3110 int physcontig; 3111 3112 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3113 3114 /* shortcuts */ 3115 maxseg = sglinfo->si_max_cookie_size; 3116 size = dmar_object->dmao_size; 3117 3118 cnt = 0; 3119 sglinfo->si_bounce_on_seg = B_FALSE; 3120 3121 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3122 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3123 ssize = dvs->dvs_len; 3124 paddr = dvs->dvs_start; 3125 paddr += offset; 3126 psize = MIN(ssize, (maxseg - offset)); 3127 dvaddr = paddr + psize; 3128 ssize -= psize; 3129 3130 sgl[cnt].dmac_laddress = paddr; 3131 sgl[cnt].dmac_size = psize; 3132 sgl[cnt].dmac_type = 0; 3133 3134 size -= psize; 3135 while (size > 0) { 3136 if (ssize == 0) { 3137 dvs++; 3138 ssize = dvs->dvs_len; 3139 dvaddr = dvs->dvs_start; 3140 physcontig = 0; 3141 } else 3142 physcontig = 1; 3143 3144 paddr = dvaddr; 3145 psize = MIN(ssize, maxseg); 3146 dvaddr += psize; 3147 ssize -= psize; 3148 3149 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3150 ((sgl[cnt].dmac_size + psize) > maxseg) || 3151 (sgl[cnt].dmac_size == 0)) { 3152 /* 3153 * if we're not already in a new cookie, go to the next 3154 * cookie. 3155 */ 3156 if (sgl[cnt].dmac_size != 0) { 3157 cnt++; 3158 } 3159 3160 /* save the cookie information */ 3161 sgl[cnt].dmac_laddress = paddr; 3162 sgl[cnt].dmac_size = psize; 3163 sgl[cnt].dmac_type = 0; 3164 } else { 3165 sgl[cnt].dmac_size += psize; 3166 3167 /* 3168 * If this cookie is used up, and more cookies 3169 * are available, then move onto the next one. 3170 */ 3171 if ((sgl[cnt].dmac_size == maxseg) && 3172 ((cnt + 1) < sglinfo->si_max_pages)) { 3173 cnt++; 3174 sgl[cnt].dmac_laddress = 0; 3175 sgl[cnt].dmac_size = 0; 3176 sgl[cnt].dmac_type = 0; 3177 } 3178 } 3179 size -= psize; 3180 } 3181 3182 /* we're done, save away how many cookies the sgl has */ 3183 if (sgl[cnt].dmac_size == 0) { 3184 sglinfo->si_sgl_size = cnt; 3185 } else { 3186 sglinfo->si_sgl_size = cnt + 1; 3187 } 3188 } 3189 3190 /* 3191 * rootnex_bind_slowpath() 3192 * Call in the bind path if the calling driver can't use the sgl without 3193 * modifying it. We either need to use the copy buffer and/or we will end up 3194 * with a partial bind. 3195 */ 3196 static int 3197 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3198 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3199 { 3200 rootnex_sglinfo_t *sinfo; 3201 rootnex_window_t *window; 3202 ddi_dma_cookie_t *cookie; 3203 size_t copybuf_used; 3204 size_t dmac_size; 3205 boolean_t partial; 3206 off_t cur_offset; 3207 page_t *cur_pp; 3208 major_t mnum; 3209 int e; 3210 int i; 3211 3212 3213 sinfo = &dma->dp_sglinfo; 3214 copybuf_used = 0; 3215 partial = B_FALSE; 3216 3217 /* 3218 * If we're using the copybuf, set the copybuf state in dma struct. 3219 * Needs to be first since it sets the copy buffer size. 3220 */ 3221 if (sinfo->si_copybuf_req != 0) { 3222 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3223 if (e != DDI_SUCCESS) { 3224 return (e); 3225 } 3226 } else { 3227 dma->dp_copybuf_size = 0; 3228 } 3229 3230 /* 3231 * Figure out if we need to do a partial mapping. If so, figure out 3232 * if we need to trim the buffers when we munge the sgl. 3233 */ 3234 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3235 (dmao->dmao_size > dma->dp_maxxfer) || 3236 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3237 dma->dp_partial_required = B_TRUE; 3238 if (attr->dma_attr_granular != 1) { 3239 dma->dp_trim_required = B_TRUE; 3240 } 3241 } else { 3242 dma->dp_partial_required = B_FALSE; 3243 dma->dp_trim_required = B_FALSE; 3244 } 3245 3246 /* If we need to do a partial bind, make sure the driver supports it */ 3247 if (dma->dp_partial_required && 3248 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3249 3250 mnum = ddi_driver_major(dma->dp_dip); 3251 /* 3252 * patchable which allows us to print one warning per major 3253 * number. 3254 */ 3255 if ((rootnex_bind_warn) && 3256 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3257 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3258 cmn_err(CE_WARN, "!%s: coding error detected, the " 3259 "driver is using ddi_dma_attr(9S) incorrectly. " 3260 "There is a small risk of data corruption in " 3261 "particular with large I/Os. The driver should be " 3262 "replaced with a corrected version for proper " 3263 "system operation. To disable this warning, add " 3264 "'set rootnex:rootnex_bind_warn=0' to " 3265 "/etc/system(5).", ddi_driver_name(dma->dp_dip)); 3266 } 3267 return (DDI_DMA_TOOBIG); 3268 } 3269 3270 /* 3271 * we might need multiple windows, setup state to handle them. In this 3272 * code path, we will have at least one window. 3273 */ 3274 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3275 if (e != DDI_SUCCESS) { 3276 rootnex_teardown_copybuf(dma); 3277 return (e); 3278 } 3279 3280 window = &dma->dp_window[0]; 3281 cookie = &dma->dp_cookies[0]; 3282 cur_offset = 0; 3283 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3284 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3285 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3286 } 3287 3288 /* loop though all the cookies we got back from get_sgl() */ 3289 for (i = 0; i < sinfo->si_sgl_size; i++) { 3290 /* 3291 * If we're using the copy buffer, check this cookie and setup 3292 * its associated copy buffer state. If this cookie uses the 3293 * copy buffer, make sure we sync this window during dma_sync. 3294 */ 3295 if (dma->dp_copybuf_size > 0) { 3296 rootnex_setup_cookie(dmao, dma, cookie, 3297 cur_offset, ©buf_used, &cur_pp); 3298 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3299 window->wd_dosync = B_TRUE; 3300 } 3301 } 3302 3303 /* 3304 * save away the cookie size, since it could be modified in 3305 * the windowing code. 3306 */ 3307 dmac_size = cookie->dmac_size; 3308 3309 /* if we went over max copybuf size */ 3310 if (dma->dp_copybuf_size && 3311 (copybuf_used > dma->dp_copybuf_size)) { 3312 partial = B_TRUE; 3313 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3314 cookie, cur_offset, ©buf_used); 3315 if (e != DDI_SUCCESS) { 3316 rootnex_teardown_copybuf(dma); 3317 rootnex_teardown_windows(dma); 3318 return (e); 3319 } 3320 3321 /* 3322 * if the coookie uses the copy buffer, make sure the 3323 * new window we just moved to is set to sync. 3324 */ 3325 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3326 window->wd_dosync = B_TRUE; 3327 } 3328 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3329 dma->dp_dip); 3330 3331 /* if the cookie cnt == max sgllen, move to the next window */ 3332 } else if (window->wd_cookie_cnt >= 3333 (unsigned)attr->dma_attr_sgllen) { 3334 partial = B_TRUE; 3335 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3336 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3337 cookie, attr, cur_offset); 3338 if (e != DDI_SUCCESS) { 3339 rootnex_teardown_copybuf(dma); 3340 rootnex_teardown_windows(dma); 3341 return (e); 3342 } 3343 3344 /* 3345 * if the coookie uses the copy buffer, make sure the 3346 * new window we just moved to is set to sync. 3347 */ 3348 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3349 window->wd_dosync = B_TRUE; 3350 } 3351 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3352 dma->dp_dip); 3353 3354 /* else if we will be over maxxfer */ 3355 } else if ((window->wd_size + dmac_size) > 3356 dma->dp_maxxfer) { 3357 partial = B_TRUE; 3358 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3359 cookie); 3360 if (e != DDI_SUCCESS) { 3361 rootnex_teardown_copybuf(dma); 3362 rootnex_teardown_windows(dma); 3363 return (e); 3364 } 3365 3366 /* 3367 * if the coookie uses the copy buffer, make sure the 3368 * new window we just moved to is set to sync. 3369 */ 3370 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3371 window->wd_dosync = B_TRUE; 3372 } 3373 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3374 dma->dp_dip); 3375 3376 /* else this cookie fits in the current window */ 3377 } else { 3378 window->wd_cookie_cnt++; 3379 window->wd_size += dmac_size; 3380 } 3381 3382 /* track our offset into the buffer, go to the next cookie */ 3383 ASSERT(dmac_size <= dmao->dmao_size); 3384 ASSERT(cookie->dmac_size <= dmac_size); 3385 cur_offset += dmac_size; 3386 cookie++; 3387 } 3388 3389 /* if we ended up with a zero sized window in the end, clean it up */ 3390 if (window->wd_size == 0) { 3391 hp->dmai_nwin--; 3392 window--; 3393 } 3394 3395 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3396 3397 if (!partial) { 3398 return (DDI_DMA_MAPPED); 3399 } 3400 3401 ASSERT(dma->dp_partial_required); 3402 return (DDI_DMA_PARTIAL_MAP); 3403 } 3404 3405 /* 3406 * rootnex_setup_copybuf() 3407 * Called in bind slowpath. Figures out if we're going to use the copy 3408 * buffer, and if we do, sets up the basic state to handle it. 3409 */ 3410 static int 3411 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3412 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3413 { 3414 rootnex_sglinfo_t *sinfo; 3415 ddi_dma_attr_t lattr; 3416 size_t max_copybuf; 3417 int cansleep; 3418 int e; 3419 3420 ASSERT(!dma->dp_dvma_used); 3421 3422 sinfo = &dma->dp_sglinfo; 3423 3424 /* read this first so it's consistent through the routine */ 3425 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3426 3427 /* We need to call into the rootnex on ddi_dma_sync() */ 3428 hp->dmai_rflags &= ~DMP_NOSYNC; 3429 3430 /* make sure the copybuf size <= the max size */ 3431 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3432 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3433 3434 3435 /* convert the sleep flags */ 3436 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3437 cansleep = 1; 3438 } else { 3439 cansleep = 0; 3440 } 3441 3442 /* 3443 * Allocate the actual copy buffer. This needs to fit within the DMA 3444 * engine limits, so we can't use kmem_alloc... We don't need 3445 * contiguous memory (sgllen) since we will be forcing windows on 3446 * sgllen anyway. 3447 */ 3448 lattr = *attr; 3449 lattr.dma_attr_align = MMU_PAGESIZE; 3450 lattr.dma_attr_sgllen = -1; /* no limit */ 3451 /* 3452 * if we're using the copy buffer because of seg, use that for our 3453 * upper address limit. 3454 */ 3455 if (sinfo->si_bounce_on_seg) { 3456 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3457 } 3458 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3459 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3460 if (e != DDI_SUCCESS) { 3461 return (DDI_DMA_NORESOURCES); 3462 } 3463 3464 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3465 size_t, dma->dp_copybuf_size); 3466 3467 return (DDI_SUCCESS); 3468 } 3469 3470 3471 /* 3472 * rootnex_setup_windows() 3473 * Called in bind slowpath to setup the window state. We always have windows 3474 * in the slowpath. Even if the window count = 1. 3475 */ 3476 static int 3477 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3478 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3479 { 3480 rootnex_window_t *windowp; 3481 rootnex_sglinfo_t *sinfo; 3482 size_t copy_state_size; 3483 size_t win_state_size; 3484 size_t state_available; 3485 size_t space_needed; 3486 uint_t copybuf_win; 3487 uint_t maxxfer_win; 3488 size_t space_used; 3489 uint_t sglwin; 3490 3491 3492 sinfo = &dma->dp_sglinfo; 3493 3494 dma->dp_current_win = 0; 3495 hp->dmai_nwin = 0; 3496 3497 /* If we don't need to do a partial, we only have one window */ 3498 if (!dma->dp_partial_required) { 3499 dma->dp_max_win = 1; 3500 3501 /* 3502 * we need multiple windows, need to figure out the worse case number 3503 * of windows. 3504 */ 3505 } else { 3506 /* 3507 * if we need windows because we need more copy buffer that 3508 * we allow, the worse case number of windows we could need 3509 * here would be (copybuf space required / copybuf space that 3510 * we have) plus one for remainder, and plus 2 to handle the 3511 * extra pages on the trim for the first and last pages of the 3512 * buffer (a page is the minimum window size so under the right 3513 * attr settings, you could have a window for each page). 3514 * The last page will only be hit here if the size is not a 3515 * multiple of the granularity (which theoretically shouldn't 3516 * be the case but never has been enforced, so we could have 3517 * broken things without it). 3518 */ 3519 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3520 ASSERT(dma->dp_copybuf_size > 0); 3521 copybuf_win = (sinfo->si_copybuf_req / 3522 dma->dp_copybuf_size) + 1 + 2; 3523 } else { 3524 copybuf_win = 0; 3525 } 3526 3527 /* 3528 * if we need windows because we have more cookies than the H/W 3529 * can handle, the number of windows we would need here would 3530 * be (cookie count / cookies count H/W supports minus 1[for 3531 * trim]) plus one for remainder. 3532 */ 3533 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3534 sglwin = (sinfo->si_sgl_size / 3535 (attr->dma_attr_sgllen - 1)) + 1; 3536 } else { 3537 sglwin = 0; 3538 } 3539 3540 /* 3541 * if we need windows because we're binding more memory than the 3542 * H/W can transfer at once, the number of windows we would need 3543 * here would be (xfer count / max xfer H/W supports) plus one 3544 * for remainder, and plus 2 to handle the extra pages on the 3545 * trim (see above comment about trim) 3546 */ 3547 if (dmao->dmao_size > dma->dp_maxxfer) { 3548 maxxfer_win = (dmao->dmao_size / 3549 dma->dp_maxxfer) + 1 + 2; 3550 } else { 3551 maxxfer_win = 0; 3552 } 3553 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3554 ASSERT(dma->dp_max_win > 0); 3555 } 3556 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3557 3558 /* 3559 * Get space for window and potential copy buffer state. Before we 3560 * go and allocate memory, see if we can get away with using what's 3561 * left in the pre-allocted state or the dynamically allocated sgl. 3562 */ 3563 space_used = (uintptr_t)(sinfo->si_sgl_size * 3564 sizeof (ddi_dma_cookie_t)); 3565 3566 /* if we dynamically allocated space for the cookies */ 3567 if (dma->dp_need_to_free_cookie) { 3568 /* if we have more space in the pre-allocted buffer, use it */ 3569 ASSERT(space_used <= dma->dp_cookie_size); 3570 if ((dma->dp_cookie_size - space_used) <= 3571 rootnex_state->r_prealloc_size) { 3572 state_available = rootnex_state->r_prealloc_size; 3573 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3574 3575 /* 3576 * else, we have more free space in the dynamically allocated 3577 * buffer, i.e. the buffer wasn't worse case fragmented so we 3578 * didn't need a lot of cookies. 3579 */ 3580 } else { 3581 state_available = dma->dp_cookie_size - space_used; 3582 windowp = (rootnex_window_t *) 3583 &dma->dp_cookies[sinfo->si_sgl_size]; 3584 } 3585 3586 /* we used the pre-alloced buffer */ 3587 } else { 3588 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3589 state_available = rootnex_state->r_prealloc_size - space_used; 3590 windowp = (rootnex_window_t *) 3591 &dma->dp_cookies[sinfo->si_sgl_size]; 3592 } 3593 3594 /* 3595 * figure out how much state we need to track the copy buffer. Add an 3596 * addition 8 bytes for pointer alignemnt later. 3597 */ 3598 if (dma->dp_copybuf_size > 0) { 3599 copy_state_size = sinfo->si_max_pages * 3600 sizeof (rootnex_pgmap_t); 3601 } else { 3602 copy_state_size = 0; 3603 } 3604 /* add an additional 8 bytes for pointer alignment */ 3605 space_needed = win_state_size + copy_state_size + 0x8; 3606 3607 /* if we have enough space already, use it */ 3608 if (state_available >= space_needed) { 3609 dma->dp_window = windowp; 3610 dma->dp_need_to_free_window = B_FALSE; 3611 3612 /* not enough space, need to allocate more. */ 3613 } else { 3614 dma->dp_window = kmem_alloc(space_needed, kmflag); 3615 if (dma->dp_window == NULL) { 3616 return (DDI_DMA_NORESOURCES); 3617 } 3618 dma->dp_need_to_free_window = B_TRUE; 3619 dma->dp_window_size = space_needed; 3620 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3621 dma->dp_dip, size_t, space_needed); 3622 } 3623 3624 /* 3625 * we allocate copy buffer state and window state at the same time. 3626 * setup our copy buffer state pointers. Make sure it's aligned. 3627 */ 3628 if (dma->dp_copybuf_size > 0) { 3629 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3630 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3631 3632 } else { 3633 dma->dp_pgmap = NULL; 3634 } 3635 3636 return (DDI_SUCCESS); 3637 } 3638 3639 3640 /* 3641 * rootnex_teardown_copybuf() 3642 * cleans up after rootnex_setup_copybuf() 3643 */ 3644 static void 3645 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3646 { 3647 /* if we allocated a copy buffer, free it */ 3648 if (dma->dp_cbaddr != NULL) { 3649 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3650 } 3651 } 3652 3653 3654 /* 3655 * rootnex_teardown_windows() 3656 * cleans up after rootnex_setup_windows() 3657 */ 3658 static void 3659 rootnex_teardown_windows(rootnex_dma_t *dma) 3660 { 3661 /* 3662 * if we had to allocate window state on the last bind (because we 3663 * didn't have enough pre-allocated space in the handle), free it. 3664 */ 3665 if (dma->dp_need_to_free_window) { 3666 kmem_free(dma->dp_window, dma->dp_window_size); 3667 } 3668 } 3669 3670 3671 /* 3672 * rootnex_init_win() 3673 * Called in bind slow path during creation of a new window. Initializes 3674 * window state to default values. 3675 */ 3676 /*ARGSUSED*/ 3677 static void 3678 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3679 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3680 { 3681 hp->dmai_nwin++; 3682 window->wd_dosync = B_FALSE; 3683 window->wd_offset = cur_offset; 3684 window->wd_size = 0; 3685 window->wd_first_cookie = cookie; 3686 window->wd_cookie_cnt = 0; 3687 window->wd_trim.tr_trim_first = B_FALSE; 3688 window->wd_trim.tr_trim_last = B_FALSE; 3689 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3690 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3691 } 3692 3693 3694 /* 3695 * rootnex_setup_cookie() 3696 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3697 * the sgl uses the copy buffer, we need to go through each cookie, figure 3698 * out if it uses the copy buffer, and if it does, save away everything we'll 3699 * need during sync. 3700 */ 3701 static void 3702 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3703 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3704 page_t **cur_pp) 3705 { 3706 boolean_t copybuf_sz_power_2; 3707 rootnex_sglinfo_t *sinfo; 3708 paddr_t paddr; 3709 uint_t pidx; 3710 uint_t pcnt; 3711 off_t poff; 3712 pfn_t pfn; 3713 3714 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3715 3716 sinfo = &dma->dp_sglinfo; 3717 3718 /* 3719 * Calculate the page index relative to the start of the buffer. The 3720 * index to the current page for our buffer is the offset into the 3721 * first page of the buffer plus our current offset into the buffer 3722 * itself, shifted of course... 3723 */ 3724 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3725 ASSERT(pidx < sinfo->si_max_pages); 3726 3727 /* if this cookie uses the copy buffer */ 3728 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3729 /* 3730 * NOTE: we know that since this cookie uses the copy buffer, it 3731 * is <= MMU_PAGESIZE. 3732 */ 3733 3734 /* 3735 * get the offset into the page. For the 64-bit kernel, get the 3736 * pfn which we'll use with seg kpm. 3737 */ 3738 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3739 /* mfn_to_pfn() is a NOP on i86pc */ 3740 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3741 3742 /* figure out if the copybuf size is a power of 2 */ 3743 if (!ISP2(dma->dp_copybuf_size)) { 3744 copybuf_sz_power_2 = B_FALSE; 3745 } else { 3746 copybuf_sz_power_2 = B_TRUE; 3747 } 3748 3749 /* This page uses the copy buffer */ 3750 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3751 3752 /* 3753 * save the copy buffer KVA that we'll use with this page. 3754 * if we still fit within the copybuf, it's a simple add. 3755 * otherwise, we need to wrap over using & or % accordingly. 3756 */ 3757 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3758 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3759 *copybuf_used; 3760 } else { 3761 if (copybuf_sz_power_2) { 3762 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3763 (uintptr_t)dma->dp_cbaddr + 3764 (*copybuf_used & 3765 (dma->dp_copybuf_size - 1))); 3766 } else { 3767 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3768 (uintptr_t)dma->dp_cbaddr + 3769 (*copybuf_used % dma->dp_copybuf_size)); 3770 } 3771 } 3772 3773 /* 3774 * over write the cookie physical address with the address of 3775 * the physical address of the copy buffer page that we will 3776 * use. 3777 */ 3778 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3779 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3780 3781 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3782 3783 /* if we have a kernel VA, it's easy, just save that address */ 3784 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3785 (sinfo->si_asp == &kas)) { 3786 /* 3787 * save away the page aligned virtual address of the 3788 * driver buffer. Offsets are handled in the sync code. 3789 */ 3790 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3791 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3792 & MMU_PAGEMASK); 3793 3794 /* we don't have a kernel VA. We need one for the bcopy. */ 3795 } else { 3796 /* 3797 * for the 64-bit kernel, it's easy. We use seg kpm to 3798 * get a Kernel VA for the corresponding pfn. 3799 */ 3800 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3801 /* go to the next page_t */ 3802 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3803 *cur_pp = (*cur_pp)->p_next; 3804 } 3805 } 3806 3807 /* add to the copy buffer count */ 3808 *copybuf_used += MMU_PAGESIZE; 3809 3810 /* 3811 * This cookie doesn't use the copy buffer. Walk through the pages this 3812 * cookie occupies to reflect this. 3813 */ 3814 } else { 3815 /* 3816 * figure out how many pages the cookie occupies. We need to 3817 * use the original page offset of the buffer and the cookies 3818 * offset in the buffer to do this. 3819 */ 3820 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3821 pcnt = mmu_btopr(cookie->dmac_size + poff); 3822 3823 while (pcnt > 0) { 3824 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3825 3826 /* 3827 * we need to update pidx and cur_pp or we'll loose 3828 * track of where we are. 3829 */ 3830 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3831 *cur_pp = (*cur_pp)->p_next; 3832 } 3833 pidx++; 3834 pcnt--; 3835 } 3836 } 3837 } 3838 3839 3840 /* 3841 * rootnex_sgllen_window_boundary() 3842 * Called in the bind slow path when the next cookie causes us to exceed (in 3843 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3844 * length supported by the DMA H/W. 3845 */ 3846 static int 3847 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3848 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3849 off_t cur_offset) 3850 { 3851 off_t new_offset; 3852 size_t trim_sz; 3853 off_t coffset; 3854 3855 3856 /* 3857 * if we know we'll never have to trim, it's pretty easy. Just move to 3858 * the next window and init it. We're done. 3859 */ 3860 if (!dma->dp_trim_required) { 3861 (*windowp)++; 3862 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3863 (*windowp)->wd_cookie_cnt++; 3864 (*windowp)->wd_size = cookie->dmac_size; 3865 return (DDI_SUCCESS); 3866 } 3867 3868 /* figure out how much we need to trim from the window */ 3869 ASSERT(attr->dma_attr_granular != 0); 3870 if (dma->dp_granularity_power_2) { 3871 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3872 } else { 3873 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3874 } 3875 3876 /* The window's a whole multiple of granularity. We're done */ 3877 if (trim_sz == 0) { 3878 (*windowp)++; 3879 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3880 (*windowp)->wd_cookie_cnt++; 3881 (*windowp)->wd_size = cookie->dmac_size; 3882 return (DDI_SUCCESS); 3883 } 3884 3885 /* 3886 * The window's not a whole multiple of granularity, since we know this 3887 * is due to the sgllen, we need to go back to the last cookie and trim 3888 * that one, add the left over part of the old cookie into the new 3889 * window, and then add in the new cookie into the new window. 3890 */ 3891 3892 /* 3893 * make sure the driver isn't making us do something bad... Trimming and 3894 * sgllen == 1 don't go together. 3895 */ 3896 if (attr->dma_attr_sgllen == 1) { 3897 return (DDI_DMA_NOMAPPING); 3898 } 3899 3900 /* 3901 * first, setup the current window to account for the trim. Need to go 3902 * back to the last cookie for this. 3903 */ 3904 cookie--; 3905 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3906 (*windowp)->wd_trim.tr_last_cookie = cookie; 3907 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3908 ASSERT(cookie->dmac_size > trim_sz); 3909 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3910 (*windowp)->wd_size -= trim_sz; 3911 3912 /* save the buffer offsets for the next window */ 3913 coffset = cookie->dmac_size - trim_sz; 3914 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3915 3916 /* 3917 * set this now in case this is the first window. all other cases are 3918 * set in dma_win() 3919 */ 3920 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3921 3922 /* 3923 * initialize the next window using what's left over in the previous 3924 * cookie. 3925 */ 3926 (*windowp)++; 3927 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3928 (*windowp)->wd_cookie_cnt++; 3929 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3930 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3931 (*windowp)->wd_trim.tr_first_size = trim_sz; 3932 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3933 (*windowp)->wd_dosync = B_TRUE; 3934 } 3935 3936 /* 3937 * now go back to the current cookie and add it to the new window. set 3938 * the new window size to the what was left over from the previous 3939 * cookie and what's in the current cookie. 3940 */ 3941 cookie++; 3942 (*windowp)->wd_cookie_cnt++; 3943 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3944 3945 /* 3946 * trim plus the next cookie could put us over maxxfer (a cookie can be 3947 * a max size of maxxfer). Handle that case. 3948 */ 3949 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3950 /* 3951 * maxxfer is already a whole multiple of granularity, and this 3952 * trim will be <= the previous trim (since a cookie can't be 3953 * larger than maxxfer). Make things simple here. 3954 */ 3955 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3956 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3957 (*windowp)->wd_trim.tr_last_cookie = cookie; 3958 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3959 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3960 (*windowp)->wd_size -= trim_sz; 3961 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3962 3963 /* save the buffer offsets for the next window */ 3964 coffset = cookie->dmac_size - trim_sz; 3965 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3966 3967 /* setup the next window */ 3968 (*windowp)++; 3969 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3970 (*windowp)->wd_cookie_cnt++; 3971 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3972 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3973 coffset; 3974 (*windowp)->wd_trim.tr_first_size = trim_sz; 3975 } 3976 3977 return (DDI_SUCCESS); 3978 } 3979 3980 3981 /* 3982 * rootnex_copybuf_window_boundary() 3983 * Called in bind slowpath when we get to a window boundary because we used 3984 * up all the copy buffer that we have. 3985 */ 3986 static int 3987 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3988 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3989 size_t *copybuf_used) 3990 { 3991 rootnex_sglinfo_t *sinfo; 3992 off_t new_offset; 3993 size_t trim_sz; 3994 paddr_t paddr; 3995 off_t coffset; 3996 uint_t pidx; 3997 off_t poff; 3998 3999 pidx = 0; 4000 sinfo = &dma->dp_sglinfo; 4001 4002 /* 4003 * the copy buffer should be a whole multiple of page size. We know that 4004 * this cookie is <= MMU_PAGESIZE. 4005 */ 4006 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4007 4008 /* reset copybuf used */ 4009 *copybuf_used = 0; 4010 4011 /* 4012 * if we don't have to trim (since granularity is set to 1), go to the 4013 * next window and add the current cookie to it. We know the current 4014 * cookie uses the copy buffer since we're in this code path. 4015 */ 4016 if (!dma->dp_trim_required) { 4017 (*windowp)++; 4018 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4019 4020 /* Add this cookie to the new window */ 4021 (*windowp)->wd_cookie_cnt++; 4022 (*windowp)->wd_size += cookie->dmac_size; 4023 *copybuf_used += MMU_PAGESIZE; 4024 return (DDI_SUCCESS); 4025 } 4026 4027 /* 4028 * *** may need to trim, figure it out. 4029 */ 4030 4031 /* figure out how much we need to trim from the window */ 4032 if (dma->dp_granularity_power_2) { 4033 trim_sz = (*windowp)->wd_size & 4034 (hp->dmai_attr.dma_attr_granular - 1); 4035 } else { 4036 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4037 } 4038 4039 /* 4040 * if the window's a whole multiple of granularity, go to the next 4041 * window, init it, then add in the current cookie. We know the current 4042 * cookie uses the copy buffer since we're in this code path. 4043 */ 4044 if (trim_sz == 0) { 4045 (*windowp)++; 4046 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4047 4048 /* Add this cookie to the new window */ 4049 (*windowp)->wd_cookie_cnt++; 4050 (*windowp)->wd_size += cookie->dmac_size; 4051 *copybuf_used += MMU_PAGESIZE; 4052 return (DDI_SUCCESS); 4053 } 4054 4055 /* 4056 * *** We figured it out, we definitly need to trim 4057 */ 4058 4059 /* 4060 * make sure the driver isn't making us do something bad... 4061 * Trimming and sgllen == 1 don't go together. 4062 */ 4063 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4064 return (DDI_DMA_NOMAPPING); 4065 } 4066 4067 /* 4068 * first, setup the current window to account for the trim. Need to go 4069 * back to the last cookie for this. Some of the last cookie will be in 4070 * the current window, and some of the last cookie will be in the new 4071 * window. All of the current cookie will be in the new window. 4072 */ 4073 cookie--; 4074 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4075 (*windowp)->wd_trim.tr_last_cookie = cookie; 4076 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4077 ASSERT(cookie->dmac_size > trim_sz); 4078 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4079 (*windowp)->wd_size -= trim_sz; 4080 4081 /* 4082 * we're trimming the last cookie (not the current cookie). So that 4083 * last cookie may have or may not have been using the copy buffer ( 4084 * we know the cookie passed in uses the copy buffer since we're in 4085 * this code path). 4086 * 4087 * If the last cookie doesn't use the copy buffer, nothing special to 4088 * do. However, if it does uses the copy buffer, it will be both the 4089 * last page in the current window and the first page in the next 4090 * window. Since we are reusing the copy buffer (and KVA space on the 4091 * 32-bit kernel), this page will use the end of the copy buffer in the 4092 * current window, and the start of the copy buffer in the next window. 4093 * Track that info... The cookie physical address was already set to 4094 * the copy buffer physical address in setup_cookie.. 4095 */ 4096 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4097 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4098 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4099 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4100 (*windowp)->wd_trim.tr_last_pidx = pidx; 4101 (*windowp)->wd_trim.tr_last_cbaddr = 4102 dma->dp_pgmap[pidx].pm_cbaddr; 4103 } 4104 4105 /* save the buffer offsets for the next window */ 4106 coffset = cookie->dmac_size - trim_sz; 4107 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4108 4109 /* 4110 * set this now in case this is the first window. all other cases are 4111 * set in dma_win() 4112 */ 4113 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4114 4115 /* 4116 * initialize the next window using what's left over in the previous 4117 * cookie. 4118 */ 4119 (*windowp)++; 4120 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4121 (*windowp)->wd_cookie_cnt++; 4122 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4123 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4124 (*windowp)->wd_trim.tr_first_size = trim_sz; 4125 4126 /* 4127 * again, we're tracking if the last cookie uses the copy buffer. 4128 * read the comment above for more info on why we need to track 4129 * additional state. 4130 * 4131 * For the first cookie in the new window, we need reset the physical 4132 * address to DMA into to the start of the copy buffer plus any 4133 * initial page offset which may be present. 4134 */ 4135 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4136 (*windowp)->wd_dosync = B_TRUE; 4137 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4138 (*windowp)->wd_trim.tr_first_pidx = pidx; 4139 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4140 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4141 4142 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4143 poff; 4144 (*windowp)->wd_trim.tr_first_paddr = 4145 ROOTNEX_PADDR_TO_RBASE(paddr); 4146 4147 /* account for the cookie copybuf usage in the new window */ 4148 *copybuf_used += MMU_PAGESIZE; 4149 4150 /* 4151 * every piece of code has to have a hack, and here is this 4152 * ones :-) 4153 * 4154 * There is a complex interaction between setup_cookie and the 4155 * copybuf window boundary. The complexity had to be in either 4156 * the maxxfer window, or the copybuf window, and I chose the 4157 * copybuf code. 4158 * 4159 * So in this code path, we have taken the last cookie, 4160 * virtually broken it in half due to the trim, and it happens 4161 * to use the copybuf which further complicates life. At the 4162 * same time, we have already setup the current cookie, which 4163 * is now wrong. More background info: the current cookie uses 4164 * the copybuf, so it is only a page long max. So we need to 4165 * fix the current cookies copy buffer address, physical 4166 * address, and kva for the 32-bit kernel. We due this by 4167 * bumping them by page size (of course, we can't due this on 4168 * the physical address since the copy buffer may not be 4169 * physically contiguous). 4170 */ 4171 cookie++; 4172 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4173 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4174 4175 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4176 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4177 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4178 4179 } else { 4180 /* go back to the current cookie */ 4181 cookie++; 4182 } 4183 4184 /* 4185 * add the current cookie to the new window. set the new window size to 4186 * the what was left over from the previous cookie and what's in the 4187 * current cookie. 4188 */ 4189 (*windowp)->wd_cookie_cnt++; 4190 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4191 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4192 4193 /* 4194 * we know that the cookie passed in always uses the copy buffer. We 4195 * wouldn't be here if it didn't. 4196 */ 4197 *copybuf_used += MMU_PAGESIZE; 4198 4199 return (DDI_SUCCESS); 4200 } 4201 4202 4203 /* 4204 * rootnex_maxxfer_window_boundary() 4205 * Called in bind slowpath when we get to a window boundary because we will 4206 * go over maxxfer. 4207 */ 4208 static int 4209 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4210 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4211 { 4212 size_t dmac_size; 4213 off_t new_offset; 4214 size_t trim_sz; 4215 off_t coffset; 4216 4217 4218 /* 4219 * calculate how much we have to trim off of the current cookie to equal 4220 * maxxfer. We don't have to account for granularity here since our 4221 * maxxfer already takes that into account. 4222 */ 4223 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4224 ASSERT(trim_sz <= cookie->dmac_size); 4225 ASSERT(trim_sz <= dma->dp_maxxfer); 4226 4227 /* save cookie size since we need it later and we might change it */ 4228 dmac_size = cookie->dmac_size; 4229 4230 /* 4231 * if we're not trimming the entire cookie, setup the current window to 4232 * account for the trim. 4233 */ 4234 if (trim_sz < cookie->dmac_size) { 4235 (*windowp)->wd_cookie_cnt++; 4236 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4237 (*windowp)->wd_trim.tr_last_cookie = cookie; 4238 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4239 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4240 (*windowp)->wd_size = dma->dp_maxxfer; 4241 4242 /* 4243 * set the adjusted cookie size now in case this is the first 4244 * window. All other windows are taken care of in get win 4245 */ 4246 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4247 } 4248 4249 /* 4250 * coffset is the current offset within the cookie, new_offset is the 4251 * current offset with the entire buffer. 4252 */ 4253 coffset = dmac_size - trim_sz; 4254 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4255 4256 /* initialize the next window */ 4257 (*windowp)++; 4258 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4259 (*windowp)->wd_cookie_cnt++; 4260 (*windowp)->wd_size = trim_sz; 4261 if (trim_sz < dmac_size) { 4262 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4263 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4264 coffset; 4265 (*windowp)->wd_trim.tr_first_size = trim_sz; 4266 } 4267 4268 return (DDI_SUCCESS); 4269 } 4270 4271 4272 /*ARGSUSED*/ 4273 static int 4274 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4275 off_t off, size_t len, uint_t cache_flags) 4276 { 4277 rootnex_sglinfo_t *sinfo; 4278 rootnex_pgmap_t *cbpage; 4279 rootnex_window_t *win; 4280 ddi_dma_impl_t *hp; 4281 rootnex_dma_t *dma; 4282 caddr_t fromaddr; 4283 caddr_t toaddr; 4284 uint_t psize; 4285 off_t offset; 4286 uint_t pidx; 4287 size_t size; 4288 off_t poff; 4289 int e; 4290 4291 4292 hp = (ddi_dma_impl_t *)handle; 4293 dma = (rootnex_dma_t *)hp->dmai_private; 4294 sinfo = &dma->dp_sglinfo; 4295 4296 /* 4297 * if we don't have any windows, we don't need to sync. A copybuf 4298 * will cause us to have at least one window. 4299 */ 4300 if (dma->dp_window == NULL) { 4301 return (DDI_SUCCESS); 4302 } 4303 4304 /* This window may not need to be sync'd */ 4305 win = &dma->dp_window[dma->dp_current_win]; 4306 if (!win->wd_dosync) { 4307 return (DDI_SUCCESS); 4308 } 4309 4310 /* handle off and len special cases */ 4311 if ((off == 0) || (rootnex_sync_ignore_params)) { 4312 offset = win->wd_offset; 4313 } else { 4314 offset = off; 4315 } 4316 if ((len == 0) || (rootnex_sync_ignore_params)) { 4317 size = win->wd_size; 4318 } else { 4319 size = len; 4320 } 4321 4322 /* check the sync args to make sure they make a little sense */ 4323 if (rootnex_sync_check_parms) { 4324 e = rootnex_valid_sync_parms(hp, win, offset, size, 4325 cache_flags); 4326 if (e != DDI_SUCCESS) { 4327 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4328 return (DDI_FAILURE); 4329 } 4330 } 4331 4332 /* 4333 * special case the first page to handle the offset into the page. The 4334 * offset to the current page for our buffer is the offset into the 4335 * first page of the buffer plus our current offset into the buffer 4336 * itself, masked of course. 4337 */ 4338 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4339 psize = MIN((MMU_PAGESIZE - poff), size); 4340 4341 /* go through all the pages that we want to sync */ 4342 while (size > 0) { 4343 /* 4344 * Calculate the page index relative to the start of the buffer. 4345 * The index to the current page for our buffer is the offset 4346 * into the first page of the buffer plus our current offset 4347 * into the buffer itself, shifted of course... 4348 */ 4349 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4350 ASSERT(pidx < sinfo->si_max_pages); 4351 4352 /* 4353 * if this page uses the copy buffer, we need to sync it, 4354 * otherwise, go on to the next page. 4355 */ 4356 cbpage = &dma->dp_pgmap[pidx]; 4357 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4358 (cbpage->pm_uses_copybuf == B_FALSE)); 4359 if (cbpage->pm_uses_copybuf) { 4360 /* cbaddr and kaddr should be page aligned */ 4361 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4362 MMU_PAGEOFFSET) == 0); 4363 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4364 MMU_PAGEOFFSET) == 0); 4365 4366 /* 4367 * if we're copying for the device, we are going to 4368 * copy from the drivers buffer and to the rootnex 4369 * allocated copy buffer. 4370 */ 4371 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4372 fromaddr = cbpage->pm_kaddr + poff; 4373 toaddr = cbpage->pm_cbaddr + poff; 4374 ROOTNEX_DPROBE2(rootnex__sync__dev, 4375 dev_info_t *, dma->dp_dip, size_t, psize); 4376 4377 /* 4378 * if we're copying for the cpu/kernel, we are going to 4379 * copy from the rootnex allocated copy buffer to the 4380 * drivers buffer. 4381 */ 4382 } else { 4383 fromaddr = cbpage->pm_cbaddr + poff; 4384 toaddr = cbpage->pm_kaddr + poff; 4385 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4386 dev_info_t *, dma->dp_dip, size_t, psize); 4387 } 4388 4389 bcopy(fromaddr, toaddr, psize); 4390 } 4391 4392 /* 4393 * decrement size until we're done, update our offset into the 4394 * buffer, and get the next page size. 4395 */ 4396 size -= psize; 4397 offset += psize; 4398 psize = MIN(MMU_PAGESIZE, size); 4399 4400 /* page offset is zero for the rest of this loop */ 4401 poff = 0; 4402 } 4403 4404 return (DDI_SUCCESS); 4405 } 4406 4407 /* 4408 * rootnex_dma_sync() 4409 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4410 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4411 * is set, ddi_dma_sync() returns immediately passing back success. 4412 */ 4413 /*ARGSUSED*/ 4414 static int 4415 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4416 off_t off, size_t len, uint_t cache_flags) 4417 { 4418 #if !defined(__xpv) 4419 if (IOMMU_USED(rdip)) { 4420 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4421 cache_flags)); 4422 } 4423 #endif 4424 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4425 cache_flags)); 4426 } 4427 4428 /* 4429 * rootnex_valid_sync_parms() 4430 * checks the parameters passed to sync to verify they are correct. 4431 */ 4432 static int 4433 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4434 off_t offset, size_t size, uint_t cache_flags) 4435 { 4436 off_t woffset; 4437 4438 4439 /* 4440 * the first part of the test to make sure the offset passed in is 4441 * within the window. 4442 */ 4443 if (offset < win->wd_offset) { 4444 return (DDI_FAILURE); 4445 } 4446 4447 /* 4448 * second and last part of the test to make sure the offset and length 4449 * passed in is within the window. 4450 */ 4451 woffset = offset - win->wd_offset; 4452 if ((woffset + size) > win->wd_size) { 4453 return (DDI_FAILURE); 4454 } 4455 4456 /* 4457 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4458 * be set too. 4459 */ 4460 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4461 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4462 return (DDI_SUCCESS); 4463 } 4464 4465 /* 4466 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4467 * should be set. Also DDI_DMA_READ should be set in the flags. 4468 */ 4469 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4470 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4471 (hp->dmai_rflags & DDI_DMA_READ)) { 4472 return (DDI_SUCCESS); 4473 } 4474 4475 return (DDI_FAILURE); 4476 } 4477 4478 4479 /*ARGSUSED*/ 4480 static int 4481 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4482 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4483 uint_t *ccountp) 4484 { 4485 rootnex_window_t *window; 4486 rootnex_trim_t *trim; 4487 ddi_dma_impl_t *hp; 4488 rootnex_dma_t *dma; 4489 ddi_dma_obj_t *dmao; 4490 4491 4492 hp = (ddi_dma_impl_t *)handle; 4493 dma = (rootnex_dma_t *)hp->dmai_private; 4494 4495 /* If we try and get a window which doesn't exist, return failure */ 4496 if (win >= hp->dmai_nwin) { 4497 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4498 return (DDI_FAILURE); 4499 } 4500 4501 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4502 4503 /* 4504 * if we don't have any windows, and they're asking for the first 4505 * window, setup the cookie pointer to the first cookie in the bind. 4506 * setup our return values, then increment the cookie since we return 4507 * the first cookie on the stack. 4508 */ 4509 if (dma->dp_window == NULL) { 4510 if (win != 0) { 4511 ROOTNEX_DPROF_INC( 4512 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4513 return (DDI_FAILURE); 4514 } 4515 hp->dmai_cookie = dma->dp_cookies; 4516 *offp = 0; 4517 *lenp = dmao->dmao_size; 4518 *ccountp = dma->dp_sglinfo.si_sgl_size; 4519 *cookiep = hp->dmai_cookie[0]; 4520 hp->dmai_cookie++; 4521 hp->dmai_ncookies = *ccountp; 4522 hp->dmai_curcookie = 1; 4523 return (DDI_SUCCESS); 4524 } 4525 4526 /* sync the old window before moving on to the new one */ 4527 window = &dma->dp_window[dma->dp_current_win]; 4528 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4529 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4530 DDI_DMA_SYNC_FORCPU); 4531 } 4532 4533 4534 /* 4535 * Move to the new window. 4536 * NOTE: current_win must be set for sync to work right 4537 */ 4538 dma->dp_current_win = win; 4539 window = &dma->dp_window[win]; 4540 4541 /* if needed, adjust the first and/or last cookies for trim */ 4542 trim = &window->wd_trim; 4543 if (trim->tr_trim_first) { 4544 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4545 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4546 if (trim->tr_first_copybuf_win) { 4547 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4548 trim->tr_first_cbaddr; 4549 } 4550 } 4551 if (trim->tr_trim_last) { 4552 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4553 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4554 if (trim->tr_last_copybuf_win) { 4555 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4556 trim->tr_last_cbaddr; 4557 } 4558 } 4559 4560 /* 4561 * setup the cookie pointer to the first cookie in the window. setup 4562 * our return values, then increment the cookie since we return the 4563 * first cookie on the stack. 4564 */ 4565 hp->dmai_cookie = window->wd_first_cookie; 4566 *offp = window->wd_offset; 4567 *lenp = window->wd_size; 4568 *ccountp = window->wd_cookie_cnt; 4569 *cookiep = hp->dmai_cookie[0]; 4570 hp->dmai_ncookies = *ccountp; 4571 hp->dmai_curcookie = 1; 4572 hp->dmai_cookie++; 4573 4574 4575 /* if the new window uses the copy buffer, sync it for the device */ 4576 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4577 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4578 DDI_DMA_SYNC_FORDEV); 4579 } 4580 4581 return (DDI_SUCCESS); 4582 } 4583 4584 /* 4585 * rootnex_dma_win() 4586 * called from ddi_dma_getwin() 4587 */ 4588 /*ARGSUSED*/ 4589 static int 4590 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4591 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4592 uint_t *ccountp) 4593 { 4594 #if !defined(__xpv) 4595 if (IOMMU_USED(rdip)) { 4596 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4597 cookiep, ccountp)); 4598 } 4599 #endif 4600 4601 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4602 cookiep, ccountp)); 4603 } 4604 4605 #if !defined(__xpv) 4606 /*ARGSUSED*/ 4607 static int 4608 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4609 ddi_dma_handle_t handle, void *v) 4610 { 4611 ddi_dma_impl_t *hp; 4612 rootnex_dma_t *dma; 4613 4614 hp = (ddi_dma_impl_t *)handle; 4615 dma = (rootnex_dma_t *)hp->dmai_private; 4616 dma->dp_iommu_private = v; 4617 4618 return (DDI_SUCCESS); 4619 } 4620 4621 /*ARGSUSED*/ 4622 static void * 4623 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4624 ddi_dma_handle_t handle) 4625 { 4626 ddi_dma_impl_t *hp; 4627 rootnex_dma_t *dma; 4628 4629 hp = (ddi_dma_impl_t *)handle; 4630 dma = (rootnex_dma_t *)hp->dmai_private; 4631 4632 return (dma->dp_iommu_private); 4633 } 4634 #endif 4635 4636 /* 4637 * ************************ 4638 * obsoleted dma routines 4639 * ************************ 4640 */ 4641 4642 /* 4643 * rootnex_dma_mctl() 4644 * 4645 * We don't support this legacy interface any more on x86. 4646 */ 4647 /* ARGSUSED */ 4648 static int 4649 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4650 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4651 uint_t cache_flags) 4652 { 4653 /* 4654 * The only thing dma_mctl is usef for anymore is legacy SPARC 4655 * dvma and sbus-specific routines. 4656 */ 4657 return (DDI_FAILURE); 4658 } 4659 4660 /* 4661 * ********* 4662 * FMA Code 4663 * ********* 4664 */ 4665 4666 /* 4667 * rootnex_fm_init() 4668 * FMA init busop 4669 */ 4670 /* ARGSUSED */ 4671 static int 4672 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4673 ddi_iblock_cookie_t *ibc) 4674 { 4675 *ibc = rootnex_state->r_err_ibc; 4676 4677 return (ddi_system_fmcap); 4678 } 4679 4680 /* 4681 * rootnex_dma_check() 4682 * Function called after a dma fault occurred to find out whether the 4683 * fault address is associated with a driver that is able to handle faults 4684 * and recover from faults. 4685 */ 4686 /* ARGSUSED */ 4687 static int 4688 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4689 const void *not_used) 4690 { 4691 rootnex_window_t *window; 4692 uint64_t start_addr; 4693 uint64_t fault_addr; 4694 ddi_dma_impl_t *hp; 4695 rootnex_dma_t *dma; 4696 uint64_t end_addr; 4697 size_t csize; 4698 int i; 4699 int j; 4700 4701 4702 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4703 hp = (ddi_dma_impl_t *)handle; 4704 ASSERT(hp); 4705 4706 dma = (rootnex_dma_t *)hp->dmai_private; 4707 4708 /* Get the address that we need to search for */ 4709 fault_addr = *(uint64_t *)addr; 4710 4711 /* 4712 * if we don't have any windows, we can just walk through all the 4713 * cookies. 4714 */ 4715 if (dma->dp_window == NULL) { 4716 /* for each cookie */ 4717 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4718 /* 4719 * if the faulted address is within the physical address 4720 * range of the cookie, return DDI_FM_NONFATAL. 4721 */ 4722 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4723 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4724 dma->dp_cookies[i].dmac_size))) { 4725 return (DDI_FM_NONFATAL); 4726 } 4727 } 4728 4729 /* fault_addr not within this DMA handle */ 4730 return (DDI_FM_UNKNOWN); 4731 } 4732 4733 /* we have mutiple windows, walk through each window */ 4734 for (i = 0; i < hp->dmai_nwin; i++) { 4735 window = &dma->dp_window[i]; 4736 4737 /* Go through all the cookies in the window */ 4738 for (j = 0; j < window->wd_cookie_cnt; j++) { 4739 4740 start_addr = window->wd_first_cookie[j].dmac_laddress; 4741 csize = window->wd_first_cookie[j].dmac_size; 4742 4743 /* 4744 * if we are trimming the first cookie in the window, 4745 * and this is the first cookie, adjust the start 4746 * address and size of the cookie to account for the 4747 * trim. 4748 */ 4749 if (window->wd_trim.tr_trim_first && (j == 0)) { 4750 start_addr = window->wd_trim.tr_first_paddr; 4751 csize = window->wd_trim.tr_first_size; 4752 } 4753 4754 /* 4755 * if we are trimming the last cookie in the window, 4756 * and this is the last cookie, adjust the start 4757 * address and size of the cookie to account for the 4758 * trim. 4759 */ 4760 if (window->wd_trim.tr_trim_last && 4761 (j == (window->wd_cookie_cnt - 1))) { 4762 start_addr = window->wd_trim.tr_last_paddr; 4763 csize = window->wd_trim.tr_last_size; 4764 } 4765 4766 end_addr = start_addr + csize; 4767 4768 /* 4769 * if the faulted address is within the physical 4770 * address of the cookie, return DDI_FM_NONFATAL. 4771 */ 4772 if ((fault_addr >= start_addr) && 4773 (fault_addr <= end_addr)) { 4774 return (DDI_FM_NONFATAL); 4775 } 4776 } 4777 } 4778 4779 /* fault_addr not within this DMA handle */ 4780 return (DDI_FM_UNKNOWN); 4781 } 4782 4783 /*ARGSUSED*/ 4784 static int 4785 rootnex_quiesce(dev_info_t *dip) 4786 { 4787 #if !defined(__xpv) 4788 return (immu_quiesce()); 4789 #else 4790 return (DDI_SUCCESS); 4791 #endif 4792 } 4793 4794 #if defined(__xpv) 4795 void 4796 immu_init(void) 4797 { 4798 ; 4799 } 4800 4801 void 4802 immu_startup(void) 4803 { 4804 ; 4805 } 4806 /*ARGSUSED*/ 4807 void 4808 immu_physmem_update(uint64_t addr, uint64_t size) 4809 { 4810 ; 4811 } 4812 #endif 4813