1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 /* 25 * Copyright 2011 Nexenta Systems, Inc. All rights reserved. 26 * Copyright (c) 2011 Bayard G. Bell. All rights reserved. 27 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved. 28 * Copyright 2017 Joyent, Inc. 29 * Copyright 2020 Ryan Zezeski 30 */ 31 32 /* 33 * x86 root nexus driver 34 */ 35 36 #include <sys/sysmacros.h> 37 #include <sys/conf.h> 38 #include <sys/autoconf.h> 39 #include <sys/sysmacros.h> 40 #include <sys/debug.h> 41 #include <sys/psw.h> 42 #include <sys/ddidmareq.h> 43 #include <sys/promif.h> 44 #include <sys/devops.h> 45 #include <sys/kmem.h> 46 #include <sys/cmn_err.h> 47 #include <vm/seg.h> 48 #include <vm/seg_kmem.h> 49 #include <vm/seg_dev.h> 50 #include <sys/vmem.h> 51 #include <sys/mman.h> 52 #include <vm/hat.h> 53 #include <vm/as.h> 54 #include <vm/page.h> 55 #include <sys/avintr.h> 56 #include <sys/errno.h> 57 #include <sys/modctl.h> 58 #include <sys/ddi_impldefs.h> 59 #include <sys/sunddi.h> 60 #include <sys/sunndi.h> 61 #include <sys/mach_intr.h> 62 #include <sys/psm.h> 63 #include <sys/ontrap.h> 64 #include <sys/atomic.h> 65 #include <sys/sdt.h> 66 #include <sys/rootnex.h> 67 #include <vm/hat_i86.h> 68 #include <sys/ddifm.h> 69 #include <sys/ddi_isa.h> 70 #include <sys/apic.h> 71 72 #ifdef __xpv 73 #include <sys/bootinfo.h> 74 #include <sys/hypervisor.h> 75 #include <sys/bootconf.h> 76 #include <vm/kboot_mmu.h> 77 #endif 78 79 #if !defined(__xpv) 80 #include <sys/immu.h> 81 #endif 82 83 84 /* 85 * enable/disable extra checking of function parameters. Useful for debugging 86 * drivers. 87 */ 88 #ifdef DEBUG 89 int rootnex_alloc_check_parms = 1; 90 int rootnex_bind_check_inuse = 1; 91 int rootnex_unbind_verify_buffer = 0; 92 int rootnex_sync_check_parms = 1; 93 #else 94 int rootnex_alloc_check_parms = 0; 95 int rootnex_bind_check_inuse = 0; 96 int rootnex_unbind_verify_buffer = 0; 97 int rootnex_sync_check_parms = 0; 98 #endif 99 100 boolean_t rootnex_dmar_not_setup; 101 102 /* Master Abort and Target Abort panic flag */ 103 int rootnex_fm_ma_ta_panic_flag = 0; 104 105 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 106 int rootnex_bind_fail = 1; 107 int rootnex_bind_warn = 1; 108 uint8_t *rootnex_warn_list; 109 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 110 #define ROOTNEX_BIND_WARNING (0x1 << 0) 111 112 /* 113 * revert back to old broken behavior of always sync'ing entire copy buffer. 114 * This is useful if be have a buggy driver which doesn't correctly pass in 115 * the offset and size into ddi_dma_sync(). 116 */ 117 int rootnex_sync_ignore_params = 0; 118 119 /* 120 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 121 * page for alignment. Allocate enough windows to handle a 256K buffer w/ at 122 * least 65 sgllen DMA engine, and enough copybuf buffer state pages to handle 123 * 2 pages (< 8K). We will still need to allocate the copy buffer during bind 124 * though (if we need one). These can only be modified in /etc/system before 125 * rootnex attach. 126 */ 127 int rootnex_prealloc_cookies = 65; 128 int rootnex_prealloc_windows = 4; 129 int rootnex_prealloc_copybuf = 2; 130 131 /* driver global state */ 132 static rootnex_state_t *rootnex_state; 133 134 #ifdef DEBUG 135 /* shortcut to rootnex counters */ 136 static uint64_t *rootnex_cnt; 137 #endif 138 139 /* 140 * XXX - does x86 even need these or are they left over from the SPARC days? 141 */ 142 /* statically defined integer/boolean properties for the root node */ 143 static rootnex_intprop_t rootnex_intprp[] = { 144 { "PAGESIZE", PAGESIZE }, 145 { "MMU_PAGESIZE", MMU_PAGESIZE }, 146 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 147 { DDI_RELATIVE_ADDRESSING, 1 }, 148 }; 149 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 150 151 /* 152 * If we're dom0, we're using a real device so we need to load 153 * the cookies with MFNs instead of PFNs. 154 */ 155 #ifdef __xpv 156 typedef maddr_t rootnex_addr_t; 157 #define ROOTNEX_PADDR_TO_RBASE(pa) \ 158 (DOMAIN_IS_INITDOMAIN(xen_info) ? pa_to_ma(pa) : (pa)) 159 #else 160 typedef paddr_t rootnex_addr_t; 161 #define ROOTNEX_PADDR_TO_RBASE(pa) (pa) 162 #endif 163 164 static struct cb_ops rootnex_cb_ops = { 165 nodev, /* open */ 166 nodev, /* close */ 167 nodev, /* strategy */ 168 nodev, /* print */ 169 nodev, /* dump */ 170 nodev, /* read */ 171 nodev, /* write */ 172 nodev, /* ioctl */ 173 nodev, /* devmap */ 174 nodev, /* mmap */ 175 nodev, /* segmap */ 176 nochpoll, /* chpoll */ 177 ddi_prop_op, /* cb_prop_op */ 178 NULL, /* struct streamtab */ 179 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 180 CB_REV, /* Rev */ 181 nodev, /* cb_aread */ 182 nodev /* cb_awrite */ 183 }; 184 185 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 186 off_t offset, off_t len, caddr_t *vaddrp); 187 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 188 struct hat *hat, struct seg *seg, caddr_t addr, 189 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 190 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 191 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 192 ddi_dma_handle_t *handlep); 193 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 194 ddi_dma_handle_t handle); 195 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 196 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 197 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 198 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_handle_t handle); 200 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 202 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 203 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 204 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 205 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 206 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 207 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 208 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 209 ddi_ctl_enum_t ctlop, void *arg, void *result); 210 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 211 ddi_iblock_cookie_t *ibc); 212 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 213 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 214 static int rootnex_alloc_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *, 215 void *); 216 static int rootnex_free_intr_fixed(dev_info_t *, ddi_intr_handle_impl_t *); 217 218 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 219 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 220 ddi_dma_handle_t *handlep); 221 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle); 223 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 224 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 225 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 226 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 227 ddi_dma_handle_t handle); 228 #if !defined(__xpv) 229 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 230 ddi_dma_handle_t handle); 231 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 232 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 233 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 234 ddi_dma_cookie_t *cookiep, uint_t ccount); 235 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 236 ddi_dma_handle_t handle); 237 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 238 #endif 239 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 240 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 241 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 242 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 243 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 244 245 #if !defined(__xpv) 246 static int rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 247 ddi_dma_handle_t handle, void *v); 248 static void *rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 249 ddi_dma_handle_t handle); 250 #endif 251 252 253 static struct bus_ops rootnex_bus_ops = { 254 BUSO_REV, 255 rootnex_map, 256 NULL, 257 NULL, 258 NULL, 259 rootnex_map_fault, 260 0, 261 rootnex_dma_allochdl, 262 rootnex_dma_freehdl, 263 rootnex_dma_bindhdl, 264 rootnex_dma_unbindhdl, 265 rootnex_dma_sync, 266 rootnex_dma_win, 267 rootnex_dma_mctl, 268 rootnex_ctlops, 269 ddi_bus_prop_op, 270 i_ddi_rootnex_get_eventcookie, 271 i_ddi_rootnex_add_eventcall, 272 i_ddi_rootnex_remove_eventcall, 273 i_ddi_rootnex_post_event, 274 0, /* bus_intr_ctl */ 275 0, /* bus_config */ 276 0, /* bus_unconfig */ 277 rootnex_fm_init, /* bus_fm_init */ 278 NULL, /* bus_fm_fini */ 279 NULL, /* bus_fm_access_enter */ 280 NULL, /* bus_fm_access_exit */ 281 NULL, /* bus_powr */ 282 rootnex_intr_ops /* bus_intr_op */ 283 }; 284 285 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 286 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 287 static int rootnex_quiesce(dev_info_t *dip); 288 289 static struct dev_ops rootnex_ops = { 290 DEVO_REV, 291 0, 292 ddi_no_info, 293 nulldev, 294 nulldev, 295 rootnex_attach, 296 rootnex_detach, 297 nulldev, 298 &rootnex_cb_ops, 299 &rootnex_bus_ops, 300 NULL, 301 rootnex_quiesce, /* quiesce */ 302 }; 303 304 static struct modldrv rootnex_modldrv = { 305 &mod_driverops, 306 "i86pc root nexus", 307 &rootnex_ops 308 }; 309 310 static struct modlinkage rootnex_modlinkage = { 311 MODREV_1, 312 (void *)&rootnex_modldrv, 313 NULL 314 }; 315 316 #if !defined(__xpv) 317 static iommulib_nexops_t iommulib_nexops = { 318 IOMMU_NEXOPS_VERSION, 319 "Rootnex IOMMU ops Vers 1.1", 320 NULL, 321 rootnex_coredma_allochdl, 322 rootnex_coredma_freehdl, 323 rootnex_coredma_bindhdl, 324 rootnex_coredma_unbindhdl, 325 rootnex_coredma_reset_cookies, 326 rootnex_coredma_get_cookies, 327 rootnex_coredma_set_cookies, 328 rootnex_coredma_clear_cookies, 329 rootnex_coredma_get_sleep_flags, 330 rootnex_coredma_sync, 331 rootnex_coredma_win, 332 rootnex_coredma_hdl_setprivate, 333 rootnex_coredma_hdl_getprivate 334 }; 335 #endif 336 337 /* 338 * extern hacks 339 */ 340 extern struct seg_ops segdev_ops; 341 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 342 #ifdef DDI_MAP_DEBUG 343 extern int ddi_map_debug_flag; 344 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 345 #endif 346 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 347 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 348 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 349 psm_intr_op_t, int *); 350 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 351 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 352 353 /* 354 * Use device arena to use for device control register mappings. 355 * Various kernel memory walkers (debugger, dtrace) need to know 356 * to avoid this address range to prevent undesired device activity. 357 */ 358 extern void *device_arena_alloc(size_t size, int vm_flag); 359 extern void device_arena_free(void * vaddr, size_t size); 360 361 362 /* 363 * Internal functions 364 */ 365 static int rootnex_dma_init(); 366 static void rootnex_add_props(dev_info_t *); 367 static int rootnex_ctl_reportdev(dev_info_t *dip); 368 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 369 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 370 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 371 static int rootnex_map_handle(ddi_map_req_t *mp); 372 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 373 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 374 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 375 rootnex_sglinfo_t *sglinfo); 376 static void rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, 377 ddi_dma_cookie_t *sgl, rootnex_sglinfo_t *sglinfo); 378 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 379 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 380 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 381 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 382 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 383 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 384 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag); 385 static void rootnex_teardown_windows(rootnex_dma_t *dma); 386 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 387 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 388 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 389 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 390 size_t *copybuf_used, page_t **cur_pp); 391 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 392 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 393 ddi_dma_attr_t *attr, off_t cur_offset); 394 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 395 rootnex_dma_t *dma, rootnex_window_t **windowp, 396 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 397 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 398 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 399 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 400 off_t offset, size_t size, uint_t cache_flags); 401 static int rootnex_verify_buffer(rootnex_dma_t *dma); 402 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 403 const void *comp_addr, const void *not_used); 404 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 405 rootnex_sglinfo_t *sglinfo); 406 static struct as *rootnex_get_as(ddi_dma_obj_t *dmar_object); 407 408 /* 409 * _init() 410 * 411 */ 412 int 413 _init(void) 414 { 415 416 rootnex_state = NULL; 417 return (mod_install(&rootnex_modlinkage)); 418 } 419 420 421 /* 422 * _info() 423 * 424 */ 425 int 426 _info(struct modinfo *modinfop) 427 { 428 return (mod_info(&rootnex_modlinkage, modinfop)); 429 } 430 431 432 /* 433 * _fini() 434 * 435 */ 436 int 437 _fini(void) 438 { 439 return (EBUSY); 440 } 441 442 443 /* 444 * rootnex_attach() 445 * 446 */ 447 static int 448 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 449 { 450 int fmcap; 451 int e; 452 453 switch (cmd) { 454 case DDI_ATTACH: 455 break; 456 case DDI_RESUME: 457 #if !defined(__xpv) 458 return (immu_unquiesce()); 459 #else 460 return (DDI_SUCCESS); 461 #endif 462 default: 463 return (DDI_FAILURE); 464 } 465 466 /* 467 * We should only have one instance of rootnex. Save it away since we 468 * don't have an easy way to get it back later. 469 */ 470 ASSERT(rootnex_state == NULL); 471 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 472 473 rootnex_state->r_dip = dip; 474 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 475 rootnex_state->r_reserved_msg_printed = B_FALSE; 476 #ifdef DEBUG 477 rootnex_cnt = &rootnex_state->r_counters[0]; 478 #endif 479 480 /* 481 * Set minimum fm capability level for i86pc platforms and then 482 * initialize error handling. Since we're the rootnex, we don't 483 * care what's returned in the fmcap field. 484 */ 485 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 486 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 487 fmcap = ddi_system_fmcap; 488 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 489 490 /* initialize DMA related state */ 491 e = rootnex_dma_init(); 492 if (e != DDI_SUCCESS) { 493 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 494 return (DDI_FAILURE); 495 } 496 497 /* Add static root node properties */ 498 rootnex_add_props(dip); 499 500 /* since we can't call ddi_report_dev() */ 501 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 502 503 /* Initialize rootnex event handle */ 504 i_ddi_rootnex_init_events(dip); 505 506 #if !defined(__xpv) 507 e = iommulib_nexus_register(dip, &iommulib_nexops, 508 &rootnex_state->r_iommulib_handle); 509 510 ASSERT(e == DDI_SUCCESS); 511 #endif 512 513 return (DDI_SUCCESS); 514 } 515 516 517 /* 518 * rootnex_detach() 519 * 520 */ 521 /*ARGSUSED*/ 522 static int 523 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 524 { 525 switch (cmd) { 526 case DDI_SUSPEND: 527 #if !defined(__xpv) 528 return (immu_quiesce()); 529 #else 530 return (DDI_SUCCESS); 531 #endif 532 default: 533 return (DDI_FAILURE); 534 } 535 /*NOTREACHED*/ 536 537 } 538 539 540 /* 541 * rootnex_dma_init() 542 * 543 */ 544 /*ARGSUSED*/ 545 static int 546 rootnex_dma_init() 547 { 548 size_t bufsize; 549 550 551 /* 552 * size of our cookie/window/copybuf state needed in dma bind that we 553 * pre-alloc in dma_alloc_handle 554 */ 555 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 556 rootnex_state->r_prealloc_size = 557 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 558 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 559 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 560 561 /* 562 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 563 * allocate 16 extra bytes for struct pointer alignment 564 * (p->dmai_private & dma->dp_prealloc_buffer) 565 */ 566 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 567 rootnex_state->r_prealloc_size + 0x10; 568 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 569 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 570 if (rootnex_state->r_dmahdl_cache == NULL) { 571 return (DDI_FAILURE); 572 } 573 574 /* 575 * allocate array to track which major numbers we have printed warnings 576 * for. 577 */ 578 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 579 KM_SLEEP); 580 581 return (DDI_SUCCESS); 582 } 583 584 585 /* 586 * rootnex_add_props() 587 * 588 */ 589 static void 590 rootnex_add_props(dev_info_t *dip) 591 { 592 rootnex_intprop_t *rpp; 593 int i; 594 595 /* Add static integer/boolean properties to the root node */ 596 rpp = rootnex_intprp; 597 for (i = 0; i < NROOT_INTPROPS; i++) { 598 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 599 rpp[i].prop_name, rpp[i].prop_value); 600 } 601 } 602 603 604 605 /* 606 * ************************* 607 * ctlops related routines 608 * ************************* 609 */ 610 611 /* 612 * rootnex_ctlops() 613 * 614 */ 615 /*ARGSUSED*/ 616 static int 617 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 618 void *arg, void *result) 619 { 620 int n, *ptr; 621 struct ddi_parent_private_data *pdp; 622 623 switch (ctlop) { 624 case DDI_CTLOPS_DMAPMAPC: 625 /* 626 * Return 'partial' to indicate that dma mapping 627 * has to be done in the main MMU. 628 */ 629 return (DDI_DMA_PARTIAL); 630 631 case DDI_CTLOPS_BTOP: 632 /* 633 * Convert byte count input to physical page units. 634 * (byte counts that are not a page-size multiple 635 * are rounded down) 636 */ 637 *(ulong_t *)result = btop(*(ulong_t *)arg); 638 return (DDI_SUCCESS); 639 640 case DDI_CTLOPS_PTOB: 641 /* 642 * Convert size in physical pages to bytes 643 */ 644 *(ulong_t *)result = ptob(*(ulong_t *)arg); 645 return (DDI_SUCCESS); 646 647 case DDI_CTLOPS_BTOPR: 648 /* 649 * Convert byte count input to physical page units 650 * (byte counts that are not a page-size multiple 651 * are rounded up) 652 */ 653 *(ulong_t *)result = btopr(*(ulong_t *)arg); 654 return (DDI_SUCCESS); 655 656 case DDI_CTLOPS_INITCHILD: 657 return (impl_ddi_sunbus_initchild(arg)); 658 659 case DDI_CTLOPS_UNINITCHILD: 660 impl_ddi_sunbus_removechild(arg); 661 return (DDI_SUCCESS); 662 663 case DDI_CTLOPS_REPORTDEV: 664 return (rootnex_ctl_reportdev(rdip)); 665 666 case DDI_CTLOPS_IOMIN: 667 /* 668 * Nothing to do here but reflect back.. 669 */ 670 return (DDI_SUCCESS); 671 672 case DDI_CTLOPS_REGSIZE: 673 case DDI_CTLOPS_NREGS: 674 break; 675 676 case DDI_CTLOPS_SIDDEV: 677 if (ndi_dev_is_prom_node(rdip)) 678 return (DDI_SUCCESS); 679 if (ndi_dev_is_persistent_node(rdip)) 680 return (DDI_SUCCESS); 681 return (DDI_FAILURE); 682 683 case DDI_CTLOPS_POWER: 684 return ((*pm_platform_power)((power_req_t *)arg)); 685 686 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 687 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 688 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 689 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 690 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 691 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 692 if (!rootnex_state->r_reserved_msg_printed) { 693 rootnex_state->r_reserved_msg_printed = B_TRUE; 694 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 695 "1 or more reserved/obsolete operations."); 696 } 697 return (DDI_FAILURE); 698 699 default: 700 return (DDI_FAILURE); 701 } 702 /* 703 * The rest are for "hardware" properties 704 */ 705 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 706 return (DDI_FAILURE); 707 708 if (ctlop == DDI_CTLOPS_NREGS) { 709 ptr = (int *)result; 710 *ptr = pdp->par_nreg; 711 } else { 712 off_t *size = (off_t *)result; 713 714 ptr = (int *)arg; 715 n = *ptr; 716 if (n >= pdp->par_nreg) { 717 return (DDI_FAILURE); 718 } 719 *size = (off_t)pdp->par_reg[n].regspec_size; 720 } 721 return (DDI_SUCCESS); 722 } 723 724 725 /* 726 * rootnex_ctl_reportdev() 727 * 728 */ 729 static int 730 rootnex_ctl_reportdev(dev_info_t *dev) 731 { 732 int i, n, len, f_len = 0; 733 char *buf; 734 735 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 736 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 737 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 738 len = strlen(buf); 739 740 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 741 742 struct regspec *rp = sparc_pd_getreg(dev, i); 743 744 if (i == 0) 745 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 746 ": "); 747 else 748 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 749 " and "); 750 len = strlen(buf); 751 752 switch (rp->regspec_bustype) { 753 754 case BTEISA: 755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 756 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 757 break; 758 759 case BTISA: 760 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 761 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 762 break; 763 764 default: 765 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 766 "space %x offset %x", 767 rp->regspec_bustype, rp->regspec_addr); 768 break; 769 } 770 len = strlen(buf); 771 } 772 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 773 int pri; 774 775 if (i != 0) { 776 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 777 ","); 778 len = strlen(buf); 779 } 780 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 781 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 782 " sparc ipl %d", pri); 783 len = strlen(buf); 784 } 785 #ifdef DEBUG 786 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 787 cmn_err(CE_NOTE, "next message is truncated: " 788 "printed length 1024, real length %d", f_len); 789 } 790 #endif /* DEBUG */ 791 cmn_err(CE_CONT, "?%s\n", buf); 792 kmem_free(buf, REPORTDEV_BUFSIZE); 793 return (DDI_SUCCESS); 794 } 795 796 797 /* 798 * ****************** 799 * map related code 800 * ****************** 801 */ 802 803 /* 804 * rootnex_map() 805 * 806 */ 807 static int 808 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 809 off_t len, caddr_t *vaddrp) 810 { 811 struct regspec *orp = NULL; 812 struct regspec64 rp = { 0 }; 813 ddi_map_req_t mr = *mp; /* Get private copy of request */ 814 815 mp = &mr; 816 817 switch (mp->map_op) { 818 case DDI_MO_MAP_LOCKED: 819 case DDI_MO_UNMAP: 820 case DDI_MO_MAP_HANDLE: 821 break; 822 default: 823 #ifdef DDI_MAP_DEBUG 824 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 825 mp->map_op); 826 #endif /* DDI_MAP_DEBUG */ 827 return (DDI_ME_UNIMPLEMENTED); 828 } 829 830 if (mp->map_flags & DDI_MF_USER_MAPPING) { 831 #ifdef DDI_MAP_DEBUG 832 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 833 #endif /* DDI_MAP_DEBUG */ 834 return (DDI_ME_UNIMPLEMENTED); 835 } 836 837 /* 838 * First, we need to get the original regspec out before we convert it 839 * to the extended format. If we have a register number, then we need to 840 * convert that to a regspec. 841 */ 842 if (mp->map_type == DDI_MT_RNUMBER) { 843 844 int rnumber = mp->map_obj.rnumber; 845 #ifdef DDI_MAP_DEBUG 846 static char *out_of_range = 847 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 848 #endif /* DDI_MAP_DEBUG */ 849 850 orp = i_ddi_rnumber_to_regspec(rdip, rnumber); 851 if (orp == NULL) { 852 #ifdef DDI_MAP_DEBUG 853 cmn_err(CE_WARN, out_of_range, rnumber, 854 ddi_get_name(rdip)); 855 #endif /* DDI_MAP_DEBUG */ 856 return (DDI_ME_RNUMBER_RANGE); 857 } 858 } else if (!(mp->map_flags & DDI_MF_EXT_REGSPEC)) { 859 orp = mp->map_obj.rp; 860 } 861 862 /* 863 * Ensure that we are always using a 64-bit extended regspec regardless 864 * of what was passed into us. If the child driver is using a 64-bit 865 * regspec, then we need to make sure that we copy this to the local 866 * regspec64, rp. 867 */ 868 if (orp != NULL) { 869 rp.regspec_bustype = orp->regspec_bustype; 870 rp.regspec_addr = orp->regspec_addr; 871 rp.regspec_size = orp->regspec_size; 872 } else { 873 struct regspec64 *rp64; 874 rp64 = (struct regspec64 *)mp->map_obj.rp; 875 rp = *rp64; 876 } 877 878 mp->map_type = DDI_MT_REGSPEC; 879 mp->map_flags |= DDI_MF_EXT_REGSPEC; 880 mp->map_obj.rp = (struct regspec *)&rp; 881 882 /* 883 * Adjust offset and length correspnding to called values... 884 * XXX: A non-zero length means override the one in the regspec 885 * XXX: (regardless of what's in the parent's range?) 886 */ 887 888 #ifdef DDI_MAP_DEBUG 889 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 890 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 891 rp.regspec_bustype, rp.regspec_addr, rp.regspec_size, offset, 892 len, mp->map_handlep); 893 #endif /* DDI_MAP_DEBUG */ 894 895 /* 896 * I/O or memory mapping: 897 * 898 * <bustype=0, addr=x, len=x>: memory 899 * <bustype=1, addr=x, len=x>: i/o 900 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 901 */ 902 903 if (rp.regspec_bustype > 1 && rp.regspec_addr != 0) { 904 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 905 " <0x%" PRIx64 ", 0x%" PRIx64 ", 0x%" PRIx64 ">", 906 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 907 rp.regspec_addr, rp.regspec_size); 908 return (DDI_ME_INVAL); 909 } 910 911 if (rp.regspec_bustype > 1 && rp.regspec_addr == 0) { 912 /* 913 * compatibility i/o mapping 914 */ 915 rp.regspec_bustype += offset; 916 } else { 917 /* 918 * Normal memory or i/o mapping 919 */ 920 rp.regspec_addr += offset; 921 } 922 923 if (len != 0) 924 rp.regspec_size = len; 925 926 #ifdef DDI_MAP_DEBUG 927 cmn_err(CE_CONT, " <%s,%s> <0x%" PRIx64 ", 0x%" PRIx64 928 ", 0x%" PRId64 "> offset %d len %d handle 0x%x\n", 929 ddi_get_name(dip), ddi_get_name(rdip), rp.regspec_bustype, 930 rp.regspec_addr, rp.regspec_size, offset, len, mp->map_handlep); 931 #endif /* DDI_MAP_DEBUG */ 932 933 934 /* 935 * The x86 root nexus does not have any notion of valid ranges of 936 * addresses. Its children have valid ranges, but because there are none 937 * for the nexus, we don't need to call i_ddi_apply_range(). Verify 938 * that is the case. 939 */ 940 ASSERT0(sparc_pd_getnrng(dip)); 941 942 switch (mp->map_op) { 943 case DDI_MO_MAP_LOCKED: 944 945 /* 946 * Set up the locked down kernel mapping to the regspec... 947 */ 948 949 return (rootnex_map_regspec(mp, vaddrp)); 950 951 case DDI_MO_UNMAP: 952 953 /* 954 * Release mapping... 955 */ 956 957 return (rootnex_unmap_regspec(mp, vaddrp)); 958 959 case DDI_MO_MAP_HANDLE: 960 961 return (rootnex_map_handle(mp)); 962 963 default: 964 return (DDI_ME_UNIMPLEMENTED); 965 } 966 } 967 968 969 /* 970 * rootnex_map_fault() 971 * 972 * fault in mappings for requestors 973 */ 974 /*ARGSUSED*/ 975 static int 976 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 977 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 978 uint_t lock) 979 { 980 981 #ifdef DDI_MAP_DEBUG 982 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 983 ddi_map_debug(" Seg <%s>\n", 984 seg->s_ops == &segdev_ops ? "segdev" : 985 seg == &kvseg ? "segkmem" : "NONE!"); 986 #endif /* DDI_MAP_DEBUG */ 987 988 /* 989 * This is all terribly broken, but it is a start 990 * 991 * XXX Note that this test means that segdev_ops 992 * must be exported from seg_dev.c. 993 * XXX What about devices with their own segment drivers? 994 */ 995 if (seg->s_ops == &segdev_ops) { 996 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 997 998 if (hat == NULL) { 999 /* 1000 * This is one plausible interpretation of 1001 * a null hat i.e. use the first hat on the 1002 * address space hat list which by convention is 1003 * the hat of the system MMU. At alternative 1004 * would be to panic .. this might well be better .. 1005 */ 1006 ASSERT(AS_READ_HELD(seg->s_as)); 1007 hat = seg->s_as->a_hat; 1008 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 1009 } 1010 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 1011 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 1012 } else if (seg == &kvseg && dp == NULL) { 1013 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 1014 HAT_LOAD_LOCK); 1015 } else 1016 return (DDI_FAILURE); 1017 return (DDI_SUCCESS); 1018 } 1019 1020 1021 static int 1022 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1023 { 1024 rootnex_addr_t rbase; 1025 void *cvaddr; 1026 uint64_t npages, pgoffset; 1027 struct regspec64 *rp; 1028 ddi_acc_hdl_t *hp; 1029 ddi_acc_impl_t *ap; 1030 uint_t hat_acc_flags; 1031 paddr_t pbase; 1032 1033 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1034 rp = (struct regspec64 *)mp->map_obj.rp; 1035 hp = mp->map_handlep; 1036 1037 #ifdef DDI_MAP_DEBUG 1038 ddi_map_debug( 1039 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1040 rp->regspec_bustype, rp->regspec_addr, 1041 rp->regspec_size, mp->map_handlep); 1042 #endif /* DDI_MAP_DEBUG */ 1043 1044 /* 1045 * I/O or memory mapping 1046 * 1047 * <bustype=0, addr=x, len=x>: memory 1048 * <bustype=1, addr=x, len=x>: i/o 1049 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1050 */ 1051 1052 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1053 cmn_err(CE_WARN, "rootnex: invalid register spec" 1054 " <0x%" PRIx64 ", 0x%" PRIx64", 0x%" PRIx64">", 1055 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size); 1056 return (DDI_FAILURE); 1057 } 1058 1059 if (rp->regspec_bustype != 0) { 1060 /* 1061 * I/O space - needs a handle. 1062 */ 1063 if (hp == NULL) { 1064 return (DDI_FAILURE); 1065 } 1066 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1067 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1068 impl_acc_hdl_init(hp); 1069 1070 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1071 #ifdef DDI_MAP_DEBUG 1072 ddi_map_debug("rootnex_map_regspec: mmap() " 1073 "to I/O space is not supported.\n"); 1074 #endif /* DDI_MAP_DEBUG */ 1075 return (DDI_ME_INVAL); 1076 } else { 1077 /* 1078 * 1275-compliant vs. compatibility i/o mapping 1079 */ 1080 *vaddrp = 1081 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1082 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1083 ((caddr_t)(uintptr_t)rp->regspec_addr); 1084 #ifdef __xpv 1085 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1086 hp->ah_pfn = xen_assign_pfn( 1087 mmu_btop((ulong_t)rp->regspec_addr & 1088 MMU_PAGEMASK)); 1089 } else { 1090 hp->ah_pfn = mmu_btop( 1091 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1092 } 1093 #else 1094 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1095 MMU_PAGEMASK); 1096 #endif 1097 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1098 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1099 } 1100 1101 #ifdef DDI_MAP_DEBUG 1102 ddi_map_debug( 1103 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1104 rp->regspec_size, *vaddrp); 1105 #endif /* DDI_MAP_DEBUG */ 1106 return (DDI_SUCCESS); 1107 } 1108 1109 /* 1110 * Memory space 1111 */ 1112 1113 if (hp != NULL) { 1114 /* 1115 * hat layer ignores 1116 * hp->ah_acc.devacc_attr_endian_flags. 1117 */ 1118 switch (hp->ah_acc.devacc_attr_dataorder) { 1119 case DDI_STRICTORDER_ACC: 1120 hat_acc_flags = HAT_STRICTORDER; 1121 break; 1122 case DDI_UNORDERED_OK_ACC: 1123 hat_acc_flags = HAT_UNORDERED_OK; 1124 break; 1125 case DDI_MERGING_OK_ACC: 1126 hat_acc_flags = HAT_MERGING_OK; 1127 break; 1128 case DDI_LOADCACHING_OK_ACC: 1129 hat_acc_flags = HAT_LOADCACHING_OK; 1130 break; 1131 case DDI_STORECACHING_OK_ACC: 1132 hat_acc_flags = HAT_STORECACHING_OK; 1133 break; 1134 default: 1135 return (DDI_ME_INVAL); 1136 } 1137 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1138 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1139 impl_acc_hdl_init(hp); 1140 hp->ah_hat_flags = hat_acc_flags; 1141 } else { 1142 hat_acc_flags = HAT_STRICTORDER; 1143 } 1144 1145 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1146 #ifdef __xpv 1147 /* 1148 * If we're dom0, we're using a real device so we need to translate 1149 * the MA to a PA. 1150 */ 1151 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1152 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1153 } else { 1154 pbase = rbase; 1155 } 1156 #else 1157 pbase = rbase; 1158 #endif 1159 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1160 1161 if (rp->regspec_size == 0) { 1162 #ifdef DDI_MAP_DEBUG 1163 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1164 #endif /* DDI_MAP_DEBUG */ 1165 return (DDI_ME_INVAL); 1166 } 1167 1168 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1169 /* extra cast to make gcc happy */ 1170 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1171 } else { 1172 npages = mmu_btopr(rp->regspec_size + pgoffset); 1173 1174 #ifdef DDI_MAP_DEBUG 1175 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1176 "physical %llx", npages, pbase); 1177 #endif /* DDI_MAP_DEBUG */ 1178 1179 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1180 if (cvaddr == NULL) 1181 return (DDI_ME_NORESOURCES); 1182 1183 /* 1184 * Now map in the pages we've allocated... 1185 */ 1186 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1187 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1188 HAT_LOAD_LOCK); 1189 *vaddrp = (caddr_t)cvaddr + pgoffset; 1190 1191 /* save away pfn and npages for FMA */ 1192 hp = mp->map_handlep; 1193 if (hp) { 1194 hp->ah_pfn = mmu_btop(pbase); 1195 hp->ah_pnum = npages; 1196 } 1197 } 1198 1199 #ifdef DDI_MAP_DEBUG 1200 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1201 #endif /* DDI_MAP_DEBUG */ 1202 return (DDI_SUCCESS); 1203 } 1204 1205 1206 static int 1207 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1208 { 1209 caddr_t addr = (caddr_t)*vaddrp; 1210 uint64_t npages, pgoffset; 1211 struct regspec64 *rp; 1212 1213 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1214 return (0); 1215 1216 ASSERT(mp->map_flags & DDI_MF_EXT_REGSPEC); 1217 rp = (struct regspec64 *)mp->map_obj.rp; 1218 1219 if (rp->regspec_size == 0) { 1220 #ifdef DDI_MAP_DEBUG 1221 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1222 #endif /* DDI_MAP_DEBUG */ 1223 return (DDI_ME_INVAL); 1224 } 1225 1226 /* 1227 * I/O or memory mapping: 1228 * 1229 * <bustype=0, addr=x, len=x>: memory 1230 * <bustype=1, addr=x, len=x>: i/o 1231 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1232 */ 1233 if (rp->regspec_bustype != 0) { 1234 /* 1235 * This is I/O space, which requires no particular 1236 * processing on unmap since it isn't mapped in the 1237 * first place. 1238 */ 1239 return (DDI_SUCCESS); 1240 } 1241 1242 /* 1243 * Memory space 1244 */ 1245 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1246 npages = mmu_btopr(rp->regspec_size + pgoffset); 1247 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1248 device_arena_free(addr - pgoffset, ptob(npages)); 1249 1250 /* 1251 * Destroy the pointer - the mapping has logically gone 1252 */ 1253 *vaddrp = NULL; 1254 1255 return (DDI_SUCCESS); 1256 } 1257 1258 static int 1259 rootnex_map_handle(ddi_map_req_t *mp) 1260 { 1261 rootnex_addr_t rbase; 1262 ddi_acc_hdl_t *hp; 1263 uint64_t pgoffset; 1264 struct regspec64 *rp; 1265 paddr_t pbase; 1266 1267 rp = (struct regspec64 *)mp->map_obj.rp; 1268 1269 #ifdef DDI_MAP_DEBUG 1270 ddi_map_debug( 1271 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1272 rp->regspec_bustype, rp->regspec_addr, 1273 rp->regspec_size, mp->map_handlep); 1274 #endif /* DDI_MAP_DEBUG */ 1275 1276 /* 1277 * I/O or memory mapping: 1278 * 1279 * <bustype=0, addr=x, len=x>: memory 1280 * <bustype=1, addr=x, len=x>: i/o 1281 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1282 */ 1283 if (rp->regspec_bustype != 0) { 1284 /* 1285 * This refers to I/O space, and we don't support "mapping" 1286 * I/O space to a user. 1287 */ 1288 return (DDI_FAILURE); 1289 } 1290 1291 /* 1292 * Set up the hat_flags for the mapping. 1293 */ 1294 hp = mp->map_handlep; 1295 1296 switch (hp->ah_acc.devacc_attr_endian_flags) { 1297 case DDI_NEVERSWAP_ACC: 1298 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1299 break; 1300 case DDI_STRUCTURE_LE_ACC: 1301 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1302 break; 1303 case DDI_STRUCTURE_BE_ACC: 1304 return (DDI_FAILURE); 1305 default: 1306 return (DDI_REGS_ACC_CONFLICT); 1307 } 1308 1309 switch (hp->ah_acc.devacc_attr_dataorder) { 1310 case DDI_STRICTORDER_ACC: 1311 break; 1312 case DDI_UNORDERED_OK_ACC: 1313 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1314 break; 1315 case DDI_MERGING_OK_ACC: 1316 hp->ah_hat_flags |= HAT_MERGING_OK; 1317 break; 1318 case DDI_LOADCACHING_OK_ACC: 1319 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1320 break; 1321 case DDI_STORECACHING_OK_ACC: 1322 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1323 break; 1324 default: 1325 return (DDI_FAILURE); 1326 } 1327 1328 rbase = (rootnex_addr_t)rp->regspec_addr & 1329 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1330 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1331 1332 if (rp->regspec_size == 0) 1333 return (DDI_ME_INVAL); 1334 1335 #ifdef __xpv 1336 /* 1337 * If we're dom0, we're using a real device so we need to translate 1338 * the MA to a PA. 1339 */ 1340 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1341 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1342 (rbase & MMU_PAGEOFFSET); 1343 } else { 1344 pbase = rbase; 1345 } 1346 #else 1347 pbase = rbase; 1348 #endif 1349 1350 hp->ah_pfn = mmu_btop(pbase); 1351 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1352 1353 return (DDI_SUCCESS); 1354 } 1355 1356 1357 1358 /* 1359 * ************************ 1360 * interrupt related code 1361 * ************************ 1362 */ 1363 1364 /* 1365 * rootnex_intr_ops() 1366 * bus_intr_op() function for interrupt support 1367 */ 1368 /* ARGSUSED */ 1369 static int 1370 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1371 ddi_intr_handle_impl_t *hdlp, void *result) 1372 { 1373 struct intrspec *ispec; 1374 1375 DDI_INTR_NEXDBG((CE_CONT, 1376 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1377 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1378 1379 /* Process the interrupt operation */ 1380 switch (intr_op) { 1381 case DDI_INTROP_GETCAP: 1382 /* First check with pcplusmp */ 1383 if (psm_intr_ops == NULL) 1384 return (DDI_FAILURE); 1385 1386 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1387 *(int *)result = 0; 1388 return (DDI_FAILURE); 1389 } 1390 break; 1391 case DDI_INTROP_SETCAP: 1392 if (psm_intr_ops == NULL) 1393 return (DDI_FAILURE); 1394 1395 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1396 return (DDI_FAILURE); 1397 break; 1398 case DDI_INTROP_ALLOC: 1399 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1400 return (rootnex_alloc_intr_fixed(rdip, hdlp, result)); 1401 case DDI_INTROP_FREE: 1402 ASSERT(hdlp->ih_type == DDI_INTR_TYPE_FIXED); 1403 return (rootnex_free_intr_fixed(rdip, hdlp)); 1404 case DDI_INTROP_GETPRI: 1405 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1406 return (DDI_FAILURE); 1407 *(int *)result = ispec->intrspec_pri; 1408 break; 1409 case DDI_INTROP_SETPRI: 1410 /* Validate the interrupt priority passed to us */ 1411 if (*(int *)result > LOCK_LEVEL) 1412 return (DDI_FAILURE); 1413 1414 /* Ensure that PSM is all initialized and ispec is ok */ 1415 if ((psm_intr_ops == NULL) || 1416 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1417 return (DDI_FAILURE); 1418 1419 /* Change the priority */ 1420 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1421 PSM_FAILURE) 1422 return (DDI_FAILURE); 1423 1424 /* update the ispec with the new priority */ 1425 ispec->intrspec_pri = *(int *)result; 1426 break; 1427 case DDI_INTROP_ADDISR: 1428 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1429 return (DDI_FAILURE); 1430 ispec->intrspec_func = hdlp->ih_cb_func; 1431 break; 1432 case DDI_INTROP_REMISR: 1433 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1434 return (DDI_FAILURE); 1435 ispec->intrspec_func = (uint_t (*)()) 0; 1436 break; 1437 case DDI_INTROP_ENABLE: 1438 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1439 return (DDI_FAILURE); 1440 1441 /* Call psmi to translate irq with the dip */ 1442 if (psm_intr_ops == NULL) 1443 return (DDI_FAILURE); 1444 1445 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1446 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1447 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1448 return (DDI_FAILURE); 1449 1450 /* Add the interrupt handler */ 1451 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1452 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1453 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1454 return (DDI_FAILURE); 1455 break; 1456 case DDI_INTROP_DISABLE: 1457 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1458 return (DDI_FAILURE); 1459 1460 /* Call psm_ops() to translate irq with the dip */ 1461 if (psm_intr_ops == NULL) 1462 return (DDI_FAILURE); 1463 1464 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1465 (void) (*psm_intr_ops)(rdip, hdlp, 1466 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1467 1468 /* Remove the interrupt handler */ 1469 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1470 hdlp->ih_cb_func, hdlp->ih_vector); 1471 break; 1472 case DDI_INTROP_SETMASK: 1473 if (psm_intr_ops == NULL) 1474 return (DDI_FAILURE); 1475 1476 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1477 return (DDI_FAILURE); 1478 break; 1479 case DDI_INTROP_CLRMASK: 1480 if (psm_intr_ops == NULL) 1481 return (DDI_FAILURE); 1482 1483 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1484 return (DDI_FAILURE); 1485 break; 1486 case DDI_INTROP_GETPENDING: 1487 if (psm_intr_ops == NULL) 1488 return (DDI_FAILURE); 1489 1490 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1491 result)) { 1492 *(int *)result = 0; 1493 return (DDI_FAILURE); 1494 } 1495 break; 1496 case DDI_INTROP_NAVAIL: 1497 case DDI_INTROP_NINTRS: 1498 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1499 if (*(int *)result == 0) { 1500 /* 1501 * Special case for 'pcic' driver' only. This driver 1502 * driver is a child of 'isa' and 'rootnex' drivers. 1503 * 1504 * See detailed comments on this in the function 1505 * rootnex_get_ispec(). 1506 * 1507 * Children of 'pcic' send 'NINITR' request all the 1508 * way to rootnex driver. But, the 'pdp->par_nintr' 1509 * field may not initialized. So, we fake it here 1510 * to return 1 (a la what PCMCIA nexus does). 1511 */ 1512 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1513 *(int *)result = 1; 1514 else 1515 return (DDI_FAILURE); 1516 } 1517 break; 1518 case DDI_INTROP_SUPPORTED_TYPES: 1519 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1520 break; 1521 default: 1522 return (DDI_FAILURE); 1523 } 1524 1525 return (DDI_SUCCESS); 1526 } 1527 1528 1529 /* 1530 * rootnex_get_ispec() 1531 * convert an interrupt number to an interrupt specification. 1532 * The interrupt number determines which interrupt spec will be 1533 * returned if more than one exists. 1534 * 1535 * Look into the parent private data area of the 'rdip' to find out 1536 * the interrupt specification. First check to make sure there is 1537 * one that matchs "inumber" and then return a pointer to it. 1538 * 1539 * Return NULL if one could not be found. 1540 * 1541 * NOTE: This is needed for rootnex_intr_ops() 1542 */ 1543 static struct intrspec * 1544 rootnex_get_ispec(dev_info_t *rdip, int inum) 1545 { 1546 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1547 1548 /* 1549 * Special case handling for drivers that provide their own 1550 * intrspec structures instead of relying on the DDI framework. 1551 * 1552 * A broken hardware driver in ON could potentially provide its 1553 * own intrspec structure, instead of relying on the hardware. 1554 * If these drivers are children of 'rootnex' then we need to 1555 * continue to provide backward compatibility to them here. 1556 * 1557 * Following check is a special case for 'pcic' driver which 1558 * was found to have broken hardwre andby provides its own intrspec. 1559 * 1560 * Verbatim comments from this driver are shown here: 1561 * "Don't use the ddi_add_intr since we don't have a 1562 * default intrspec in all cases." 1563 * 1564 * Since an 'ispec' may not be always created for it, 1565 * check for that and create one if so. 1566 * 1567 * NOTE: Currently 'pcic' is the only driver found to do this. 1568 */ 1569 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1570 pdp->par_nintr = 1; 1571 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1572 pdp->par_nintr, KM_SLEEP); 1573 } 1574 1575 /* Validate the interrupt number */ 1576 if (inum >= pdp->par_nintr) 1577 return (NULL); 1578 1579 /* Get the interrupt structure pointer and return that */ 1580 return ((struct intrspec *)&pdp->par_intr[inum]); 1581 } 1582 1583 /* 1584 * Allocate interrupt vector for FIXED (legacy) type. 1585 */ 1586 static int 1587 rootnex_alloc_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp, 1588 void *result) 1589 { 1590 struct intrspec *ispec; 1591 ddi_intr_handle_impl_t info_hdl; 1592 int ret; 1593 int free_phdl = 0; 1594 apic_get_type_t type_info; 1595 1596 if (psm_intr_ops == NULL) 1597 return (DDI_FAILURE); 1598 1599 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1600 return (DDI_FAILURE); 1601 1602 /* 1603 * If the PSM module is "APIX" then pass the request for it 1604 * to allocate the vector now. 1605 */ 1606 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1607 info_hdl.ih_private = &type_info; 1608 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1609 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1610 if (hdlp->ih_private == NULL) { /* allocate phdl structure */ 1611 free_phdl = 1; 1612 i_ddi_alloc_intr_phdl(hdlp); 1613 } 1614 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1615 ret = (*psm_intr_ops)(rdip, hdlp, 1616 PSM_INTR_OP_ALLOC_VECTORS, result); 1617 if (free_phdl) { /* free up the phdl structure */ 1618 free_phdl = 0; 1619 i_ddi_free_intr_phdl(hdlp); 1620 hdlp->ih_private = NULL; 1621 } 1622 } else { 1623 /* 1624 * No APIX module; fall back to the old scheme where the 1625 * interrupt vector is allocated during ddi_enable_intr() call. 1626 */ 1627 hdlp->ih_pri = ispec->intrspec_pri; 1628 *(int *)result = hdlp->ih_scratch1; 1629 ret = DDI_SUCCESS; 1630 } 1631 1632 return (ret); 1633 } 1634 1635 /* 1636 * Free up interrupt vector for FIXED (legacy) type. 1637 */ 1638 static int 1639 rootnex_free_intr_fixed(dev_info_t *rdip, ddi_intr_handle_impl_t *hdlp) 1640 { 1641 struct intrspec *ispec; 1642 struct ddi_parent_private_data *pdp; 1643 ddi_intr_handle_impl_t info_hdl; 1644 int ret; 1645 apic_get_type_t type_info; 1646 1647 if (psm_intr_ops == NULL) 1648 return (DDI_FAILURE); 1649 1650 /* 1651 * If the PSM module is "APIX" then pass the request for it 1652 * to free up the vector now. 1653 */ 1654 bzero(&info_hdl, sizeof (ddi_intr_handle_impl_t)); 1655 info_hdl.ih_private = &type_info; 1656 if ((*psm_intr_ops)(NULL, &info_hdl, PSM_INTR_OP_APIC_TYPE, NULL) == 1657 PSM_SUCCESS && strcmp(type_info.avgi_type, APIC_APIX_NAME) == 0) { 1658 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1659 return (DDI_FAILURE); 1660 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1661 ret = (*psm_intr_ops)(rdip, hdlp, 1662 PSM_INTR_OP_FREE_VECTORS, NULL); 1663 } else { 1664 /* 1665 * No APIX module; fall back to the old scheme where 1666 * the interrupt vector was already freed during 1667 * ddi_disable_intr() call. 1668 */ 1669 ret = DDI_SUCCESS; 1670 } 1671 1672 pdp = ddi_get_parent_data(rdip); 1673 1674 /* 1675 * Special case for 'pcic' driver' only. 1676 * If an intrspec was created for it, clean it up here 1677 * See detailed comments on this in the function 1678 * rootnex_get_ispec(). 1679 */ 1680 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1681 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1682 pdp->par_nintr); 1683 /* 1684 * Set it to zero; so that 1685 * DDI framework doesn't free it again 1686 */ 1687 pdp->par_intr = NULL; 1688 pdp->par_nintr = 0; 1689 } 1690 1691 return (ret); 1692 } 1693 1694 1695 /* 1696 * ****************** 1697 * dma related code 1698 * ****************** 1699 */ 1700 1701 /*ARGSUSED*/ 1702 static int 1703 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1704 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1705 ddi_dma_handle_t *handlep) 1706 { 1707 uint64_t maxsegmentsize_ll; 1708 uint_t maxsegmentsize; 1709 ddi_dma_impl_t *hp; 1710 rootnex_dma_t *dma; 1711 uint64_t count_max; 1712 uint64_t seg; 1713 int kmflag; 1714 int e; 1715 1716 1717 /* convert our sleep flags */ 1718 if (waitfp == DDI_DMA_SLEEP) { 1719 kmflag = KM_SLEEP; 1720 } else { 1721 kmflag = KM_NOSLEEP; 1722 } 1723 1724 /* 1725 * We try to do only one memory allocation here. We'll do a little 1726 * pointer manipulation later. If the bind ends up taking more than 1727 * our prealloc's space, we'll have to allocate more memory in the 1728 * bind operation. Not great, but much better than before and the 1729 * best we can do with the current bind interfaces. 1730 */ 1731 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1732 if (hp == NULL) 1733 return (DDI_DMA_NORESOURCES); 1734 1735 /* Do our pointer manipulation now, align the structures */ 1736 hp->dmai_private = (void *)(((uintptr_t)hp + 1737 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1738 dma = (rootnex_dma_t *)hp->dmai_private; 1739 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1740 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1741 1742 /* setup the handle */ 1743 rootnex_clean_dmahdl(hp); 1744 hp->dmai_error.err_fep = NULL; 1745 hp->dmai_error.err_cf = NULL; 1746 dma->dp_dip = rdip; 1747 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1748 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1749 1750 /* 1751 * The BOUNCE_ON_SEG workaround is not needed when an IOMMU 1752 * is being used. Set the upper limit to the seg value. 1753 * There will be enough DVMA space to always get addresses 1754 * that will match the constraints. 1755 */ 1756 if (IOMMU_USED(rdip) && 1757 (attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG)) { 1758 dma->dp_sglinfo.si_max_addr = attr->dma_attr_seg; 1759 dma->dp_sglinfo.si_flags &= ~_DDI_DMA_BOUNCE_ON_SEG; 1760 } else 1761 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1762 1763 hp->dmai_minxfer = attr->dma_attr_minxfer; 1764 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1765 hp->dmai_rdip = rdip; 1766 hp->dmai_attr = *attr; 1767 1768 if (attr->dma_attr_seg >= dma->dp_sglinfo.si_max_addr) 1769 dma->dp_sglinfo.si_cancross = B_FALSE; 1770 else 1771 dma->dp_sglinfo.si_cancross = B_TRUE; 1772 1773 /* we don't need to worry about the SPL since we do a tryenter */ 1774 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1775 1776 /* 1777 * Figure out our maximum segment size. If the segment size is greater 1778 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1779 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1780 * dma_attr_count_max are size-1 type values. 1781 * 1782 * Maximum segment size is the largest physically contiguous chunk of 1783 * memory that we can return from a bind (i.e. the maximum size of a 1784 * single cookie). 1785 */ 1786 1787 /* handle the rollover cases */ 1788 seg = attr->dma_attr_seg + 1; 1789 if (seg < attr->dma_attr_seg) { 1790 seg = attr->dma_attr_seg; 1791 } 1792 count_max = attr->dma_attr_count_max + 1; 1793 if (count_max < attr->dma_attr_count_max) { 1794 count_max = attr->dma_attr_count_max; 1795 } 1796 1797 /* 1798 * granularity may or may not be a power of two. If it isn't, we can't 1799 * use a simple mask. 1800 */ 1801 if (!ISP2(attr->dma_attr_granular)) { 1802 dma->dp_granularity_power_2 = B_FALSE; 1803 } else { 1804 dma->dp_granularity_power_2 = B_TRUE; 1805 } 1806 1807 /* 1808 * maxxfer should be a whole multiple of granularity. If we're going to 1809 * break up a window because we're greater than maxxfer, we might as 1810 * well make sure it's maxxfer is a whole multiple so we don't have to 1811 * worry about triming the window later on for this case. 1812 */ 1813 if (attr->dma_attr_granular > 1) { 1814 if (dma->dp_granularity_power_2) { 1815 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1816 (attr->dma_attr_maxxfer & 1817 (attr->dma_attr_granular - 1)); 1818 } else { 1819 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1820 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1821 } 1822 } else { 1823 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1824 } 1825 1826 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1827 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1828 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1829 maxsegmentsize = 0xFFFFFFFF; 1830 } else { 1831 maxsegmentsize = maxsegmentsize_ll; 1832 } 1833 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1834 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1835 1836 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1837 if (rootnex_alloc_check_parms) { 1838 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1839 if (e != DDI_SUCCESS) { 1840 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1841 (void) rootnex_dma_freehdl(dip, rdip, 1842 (ddi_dma_handle_t)hp); 1843 return (e); 1844 } 1845 } 1846 1847 *handlep = (ddi_dma_handle_t)hp; 1848 1849 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1850 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1851 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1852 1853 return (DDI_SUCCESS); 1854 } 1855 1856 1857 /* 1858 * rootnex_dma_allochdl() 1859 * called from ddi_dma_alloc_handle(). 1860 */ 1861 static int 1862 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1863 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1864 { 1865 int retval = DDI_SUCCESS; 1866 #if !defined(__xpv) 1867 1868 if (IOMMU_UNITIALIZED(rdip)) { 1869 retval = iommulib_nex_open(dip, rdip); 1870 1871 if (retval != DDI_SUCCESS && retval != DDI_ENOTSUP) 1872 return (retval); 1873 } 1874 1875 if (IOMMU_UNUSED(rdip)) { 1876 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1877 handlep); 1878 } else { 1879 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1880 waitfp, arg, handlep); 1881 } 1882 #else 1883 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1884 handlep); 1885 #endif 1886 switch (retval) { 1887 case DDI_DMA_NORESOURCES: 1888 if (waitfp != DDI_DMA_DONTWAIT) { 1889 ddi_set_callback(waitfp, arg, 1890 &rootnex_state->r_dvma_call_list_id); 1891 } 1892 break; 1893 case DDI_SUCCESS: 1894 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1895 break; 1896 default: 1897 break; 1898 } 1899 return (retval); 1900 } 1901 1902 /*ARGSUSED*/ 1903 static int 1904 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1905 ddi_dma_handle_t handle) 1906 { 1907 ddi_dma_impl_t *hp; 1908 rootnex_dma_t *dma; 1909 1910 1911 hp = (ddi_dma_impl_t *)handle; 1912 dma = (rootnex_dma_t *)hp->dmai_private; 1913 1914 /* unbind should have been called first */ 1915 ASSERT(!dma->dp_inuse); 1916 1917 mutex_destroy(&dma->dp_mutex); 1918 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1919 1920 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1921 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1922 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1923 1924 return (DDI_SUCCESS); 1925 } 1926 1927 /* 1928 * rootnex_dma_freehdl() 1929 * called from ddi_dma_free_handle(). 1930 */ 1931 static int 1932 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1933 { 1934 int ret; 1935 1936 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1937 #if !defined(__xpv) 1938 if (IOMMU_USED(rdip)) 1939 ret = iommulib_nexdma_freehdl(dip, rdip, handle); 1940 else 1941 #endif 1942 ret = rootnex_coredma_freehdl(dip, rdip, handle); 1943 1944 if (rootnex_state->r_dvma_call_list_id) 1945 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1946 1947 return (ret); 1948 } 1949 1950 /*ARGSUSED*/ 1951 static int 1952 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1953 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1954 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1955 { 1956 rootnex_sglinfo_t *sinfo; 1957 ddi_dma_obj_t *dmao; 1958 #if !defined(__xpv) 1959 struct dvmaseg *dvs; 1960 ddi_dma_cookie_t *cookie; 1961 #endif 1962 ddi_dma_attr_t *attr; 1963 ddi_dma_impl_t *hp; 1964 rootnex_dma_t *dma; 1965 int kmflag; 1966 int e; 1967 uint_t ncookies; 1968 1969 hp = (ddi_dma_impl_t *)handle; 1970 dma = (rootnex_dma_t *)hp->dmai_private; 1971 dmao = &dma->dp_dma; 1972 sinfo = &dma->dp_sglinfo; 1973 attr = &hp->dmai_attr; 1974 1975 /* convert the sleep flags */ 1976 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1977 dma->dp_sleep_flags = kmflag = KM_SLEEP; 1978 } else { 1979 dma->dp_sleep_flags = kmflag = KM_NOSLEEP; 1980 } 1981 1982 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1983 1984 /* 1985 * This is useful for debugging a driver. Not as useful in a production 1986 * system. The only time this will fail is if you have a driver bug. 1987 */ 1988 if (rootnex_bind_check_inuse) { 1989 /* 1990 * No one else should ever have this lock unless someone else 1991 * is trying to use this handle. So contention on the lock 1992 * is the same as inuse being set. 1993 */ 1994 e = mutex_tryenter(&dma->dp_mutex); 1995 if (e == 0) { 1996 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1997 return (DDI_DMA_INUSE); 1998 } 1999 if (dma->dp_inuse) { 2000 mutex_exit(&dma->dp_mutex); 2001 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2002 return (DDI_DMA_INUSE); 2003 } 2004 dma->dp_inuse = B_TRUE; 2005 mutex_exit(&dma->dp_mutex); 2006 } 2007 2008 /* save away the original bind info */ 2009 dma->dp_dma = dmareq->dmar_object; 2010 2011 #if !defined(__xpv) 2012 if (IOMMU_USED(rdip)) { 2013 dmao = &dma->dp_dvma; 2014 e = iommulib_nexdma_mapobject(dip, rdip, handle, dmareq, dmao); 2015 switch (e) { 2016 case DDI_SUCCESS: 2017 if (sinfo->si_cancross || 2018 dmao->dmao_obj.dvma_obj.dv_nseg != 1 || 2019 dmao->dmao_size > sinfo->si_max_cookie_size) { 2020 dma->dp_dvma_used = B_TRUE; 2021 break; 2022 } 2023 sinfo->si_sgl_size = 1; 2024 hp->dmai_rflags |= DMP_NOSYNC; 2025 2026 dma->dp_dvma_used = B_TRUE; 2027 dma->dp_need_to_free_cookie = B_FALSE; 2028 2029 dvs = &dmao->dmao_obj.dvma_obj.dv_seg[0]; 2030 cookie = hp->dmai_cookie = dma->dp_cookies = 2031 (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2032 cookie->dmac_laddress = dvs->dvs_start + 2033 dmao->dmao_obj.dvma_obj.dv_off; 2034 cookie->dmac_size = dvs->dvs_len; 2035 cookie->dmac_type = 0; 2036 2037 ROOTNEX_DPROBE1(rootnex__bind__dvmafast, dev_info_t *, 2038 rdip); 2039 goto fast; 2040 case DDI_ENOTSUP: 2041 break; 2042 default: 2043 rootnex_clean_dmahdl(hp); 2044 return (e); 2045 } 2046 } 2047 #endif 2048 2049 /* 2050 * Figure out a rough estimate of what maximum number of pages 2051 * this buffer could use (a high estimate of course). 2052 */ 2053 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 2054 2055 if (dma->dp_dvma_used) { 2056 /* 2057 * The number of physical pages is the worst case. 2058 * 2059 * For DVMA, the worst case is the length divided 2060 * by the maximum cookie length, plus 1. Add to that 2061 * the number of segment boundaries potentially crossed, and 2062 * the additional number of DVMA segments that was returned. 2063 * 2064 * In the normal case, for modern devices, si_cancross will 2065 * be false, and dv_nseg will be 1, and the fast path will 2066 * have been taken above. 2067 */ 2068 ncookies = (dma->dp_dma.dmao_size / sinfo->si_max_cookie_size) 2069 + 1; 2070 if (sinfo->si_cancross) 2071 ncookies += 2072 (dma->dp_dma.dmao_size / attr->dma_attr_seg) + 1; 2073 ncookies += (dmao->dmao_obj.dvma_obj.dv_nseg - 1); 2074 2075 sinfo->si_max_pages = MIN(sinfo->si_max_pages, ncookies); 2076 } 2077 2078 /* 2079 * We'll use the pre-allocated cookies for any bind that will *always* 2080 * fit (more important to be consistent, we don't want to create 2081 * additional degenerate cases). 2082 */ 2083 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 2084 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 2085 dma->dp_need_to_free_cookie = B_FALSE; 2086 ROOTNEX_DPROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 2087 uint_t, sinfo->si_max_pages); 2088 2089 /* 2090 * For anything larger than that, we'll go ahead and allocate the 2091 * maximum number of pages we expect to see. Hopefuly, we won't be 2092 * seeing this path in the fast path for high performance devices very 2093 * frequently. 2094 * 2095 * a ddi bind interface that allowed the driver to provide storage to 2096 * the bind interface would speed this case up. 2097 */ 2098 } else { 2099 /* 2100 * Save away how much memory we allocated. If we're doing a 2101 * nosleep, the alloc could fail... 2102 */ 2103 dma->dp_cookie_size = sinfo->si_max_pages * 2104 sizeof (ddi_dma_cookie_t); 2105 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 2106 if (dma->dp_cookies == NULL) { 2107 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2108 rootnex_clean_dmahdl(hp); 2109 return (DDI_DMA_NORESOURCES); 2110 } 2111 dma->dp_need_to_free_cookie = B_TRUE; 2112 ROOTNEX_DPROBE2(rootnex__bind__alloc, dev_info_t *, rdip, 2113 uint_t, sinfo->si_max_pages); 2114 } 2115 hp->dmai_cookie = dma->dp_cookies; 2116 2117 /* 2118 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 2119 * looking at the constraints in the dma structure. It will then put 2120 * some additional state about the sgl in the dma struct (i.e. is 2121 * the sgl clean, or do we need to do some munging; how many pages 2122 * need to be copied, etc.) 2123 */ 2124 if (dma->dp_dvma_used) 2125 rootnex_dvma_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2126 else 2127 rootnex_get_sgl(dmao, dma->dp_cookies, &dma->dp_sglinfo); 2128 2129 out: 2130 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 2131 /* if we don't need a copy buffer, we don't need to sync */ 2132 if (sinfo->si_copybuf_req == 0) { 2133 hp->dmai_rflags |= DMP_NOSYNC; 2134 } 2135 2136 /* 2137 * if we don't need the copybuf and we don't need to do a partial, we 2138 * hit the fast path. All the high performance devices should be trying 2139 * to hit this path. To hit this path, a device should be able to reach 2140 * all of memory, shouldn't try to bind more than it can transfer, and 2141 * the buffer shouldn't require more cookies than the driver/device can 2142 * handle [sgllen]). 2143 * 2144 * Note that negative values of dma_attr_sgllen are supposed 2145 * to mean unlimited, but we just cast them to mean a 2146 * "ridiculous large limit". This saves some extra checks on 2147 * hot paths. 2148 */ 2149 if ((sinfo->si_copybuf_req == 0) && 2150 (sinfo->si_sgl_size <= (unsigned)attr->dma_attr_sgllen) && 2151 (dmao->dmao_size <= dma->dp_maxxfer)) { 2152 fast: 2153 /* 2154 * If the driver supports FMA, insert the handle in the FMA DMA 2155 * handle cache. 2156 */ 2157 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2158 hp->dmai_error.err_cf = rootnex_dma_check; 2159 2160 /* 2161 * copy out the first cookie and ccountp, set the cookie 2162 * pointer to the second cookie. The first cookie is passed 2163 * back on the stack. Additional cookies are accessed via 2164 * ddi_dma_nextcookie() 2165 */ 2166 *cookiep = dma->dp_cookies[0]; 2167 *ccountp = sinfo->si_sgl_size; 2168 hp->dmai_cookie++; 2169 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2170 hp->dmai_ncookies = *ccountp; 2171 hp->dmai_curcookie = 1; 2172 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2173 ROOTNEX_DPROBE4(rootnex__bind__fast, dev_info_t *, rdip, 2174 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2175 uint_t, dmao->dmao_size, uint_t, *ccountp); 2176 2177 2178 return (DDI_DMA_MAPPED); 2179 } 2180 2181 /* 2182 * go to the slow path, we may need to alloc more memory, create 2183 * multiple windows, and munge up a sgl to make the device happy. 2184 */ 2185 2186 /* 2187 * With the IOMMU mapobject method used, we should never hit 2188 * the slow path. If we do, something is seriously wrong. 2189 * Clean up and return an error. 2190 */ 2191 2192 #if !defined(__xpv) 2193 2194 if (dma->dp_dvma_used) { 2195 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2196 &dma->dp_dvma); 2197 e = DDI_DMA_NOMAPPING; 2198 } else { 2199 #endif 2200 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, &dma->dp_dma, 2201 kmflag); 2202 #if !defined(__xpv) 2203 } 2204 #endif 2205 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2206 if (dma->dp_need_to_free_cookie) { 2207 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2208 } 2209 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2210 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2211 return (e); 2212 } 2213 2214 /* 2215 * If the driver supports FMA, insert the handle in the FMA DMA handle 2216 * cache. 2217 */ 2218 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2219 hp->dmai_error.err_cf = rootnex_dma_check; 2220 2221 /* if the first window uses the copy buffer, sync it for the device */ 2222 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2223 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2224 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2225 DDI_DMA_SYNC_FORDEV); 2226 } 2227 2228 /* 2229 * copy out the first cookie and ccountp, set the cookie pointer to the 2230 * second cookie. Make sure the partial flag is set/cleared correctly. 2231 * If we have a partial map (i.e. multiple windows), the number of 2232 * cookies we return is the number of cookies in the first window. 2233 */ 2234 if (e == DDI_DMA_MAPPED) { 2235 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2236 *ccountp = sinfo->si_sgl_size; 2237 hp->dmai_nwin = 1; 2238 } else { 2239 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2240 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2241 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2242 } 2243 *cookiep = dma->dp_cookies[0]; 2244 hp->dmai_cookie++; 2245 hp->dmai_ncookies = *ccountp; 2246 hp->dmai_curcookie = 1; 2247 2248 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2249 ROOTNEX_DPROBE4(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2250 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2251 dmao->dmao_size, uint_t, *ccountp); 2252 return (e); 2253 } 2254 2255 /* 2256 * rootnex_dma_bindhdl() 2257 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2258 */ 2259 static int 2260 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2261 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2262 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2263 { 2264 int ret; 2265 #if !defined(__xpv) 2266 if (IOMMU_USED(rdip)) 2267 ret = iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2268 cookiep, ccountp); 2269 else 2270 #endif 2271 ret = rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2272 cookiep, ccountp); 2273 2274 if (ret == DDI_DMA_NORESOURCES && dmareq->dmar_fp != DDI_DMA_DONTWAIT) { 2275 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg, 2276 &rootnex_state->r_dvma_call_list_id); 2277 } 2278 2279 return (ret); 2280 } 2281 2282 2283 2284 /*ARGSUSED*/ 2285 static int 2286 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2287 ddi_dma_handle_t handle) 2288 { 2289 ddi_dma_impl_t *hp; 2290 rootnex_dma_t *dma; 2291 int e; 2292 2293 hp = (ddi_dma_impl_t *)handle; 2294 dma = (rootnex_dma_t *)hp->dmai_private; 2295 2296 /* make sure the buffer wasn't free'd before calling unbind */ 2297 if (rootnex_unbind_verify_buffer) { 2298 e = rootnex_verify_buffer(dma); 2299 if (e != DDI_SUCCESS) { 2300 ASSERT(0); 2301 return (DDI_FAILURE); 2302 } 2303 } 2304 2305 /* sync the current window before unbinding the buffer */ 2306 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2307 (hp->dmai_rflags & DDI_DMA_READ)) { 2308 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2309 DDI_DMA_SYNC_FORCPU); 2310 } 2311 2312 /* 2313 * cleanup and copy buffer or window state. if we didn't use the copy 2314 * buffer or windows, there won't be much to do :-) 2315 */ 2316 rootnex_teardown_copybuf(dma); 2317 rootnex_teardown_windows(dma); 2318 2319 #if !defined(__xpv) 2320 if (IOMMU_USED(rdip) && dma->dp_dvma_used) 2321 (void) iommulib_nexdma_unmapobject(dip, rdip, handle, 2322 &dma->dp_dvma); 2323 #endif 2324 2325 /* 2326 * If we had to allocate space to for the worse case sgl (it didn't 2327 * fit into our pre-allocate buffer), free that up now 2328 */ 2329 if (dma->dp_need_to_free_cookie) { 2330 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2331 } 2332 2333 /* 2334 * clean up the handle so it's ready for the next bind (i.e. if the 2335 * handle is reused). 2336 */ 2337 rootnex_clean_dmahdl(hp); 2338 hp->dmai_error.err_cf = NULL; 2339 2340 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2341 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2342 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2343 2344 return (DDI_SUCCESS); 2345 } 2346 2347 /* 2348 * rootnex_dma_unbindhdl() 2349 * called from ddi_dma_unbind_handle() 2350 */ 2351 /*ARGSUSED*/ 2352 static int 2353 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2354 ddi_dma_handle_t handle) 2355 { 2356 int ret; 2357 2358 #if !defined(__xpv) 2359 if (IOMMU_USED(rdip)) 2360 ret = iommulib_nexdma_unbindhdl(dip, rdip, handle); 2361 else 2362 #endif 2363 ret = rootnex_coredma_unbindhdl(dip, rdip, handle); 2364 2365 if (rootnex_state->r_dvma_call_list_id) 2366 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2367 2368 return (ret); 2369 } 2370 2371 #if !defined(__xpv) 2372 2373 static int 2374 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2375 { 2376 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2377 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2378 2379 if (dma->dp_sleep_flags != KM_SLEEP && 2380 dma->dp_sleep_flags != KM_NOSLEEP) 2381 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2382 return (dma->dp_sleep_flags); 2383 } 2384 /*ARGSUSED*/ 2385 static void 2386 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2387 { 2388 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2389 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2390 rootnex_window_t *window; 2391 2392 if (dma->dp_window) { 2393 window = &dma->dp_window[dma->dp_current_win]; 2394 hp->dmai_cookie = window->wd_first_cookie; 2395 } else { 2396 hp->dmai_cookie = dma->dp_cookies; 2397 } 2398 hp->dmai_cookie++; 2399 hp->dmai_curcookie = 1; 2400 } 2401 2402 /*ARGSUSED*/ 2403 static int 2404 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2405 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2406 { 2407 int i; 2408 int km_flags; 2409 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2410 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2411 rootnex_window_t *window; 2412 ddi_dma_cookie_t *cp; 2413 ddi_dma_cookie_t *cookie; 2414 2415 ASSERT(*cookiepp == NULL); 2416 ASSERT(*ccountp == 0); 2417 2418 if (dma->dp_window) { 2419 window = &dma->dp_window[dma->dp_current_win]; 2420 cp = window->wd_first_cookie; 2421 *ccountp = window->wd_cookie_cnt; 2422 } else { 2423 cp = dma->dp_cookies; 2424 *ccountp = dma->dp_sglinfo.si_sgl_size; 2425 } 2426 2427 km_flags = rootnex_coredma_get_sleep_flags(handle); 2428 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2429 if (cookie == NULL) { 2430 return (DDI_DMA_NORESOURCES); 2431 } 2432 2433 for (i = 0; i < *ccountp; i++) { 2434 cookie[i].dmac_notused = cp[i].dmac_notused; 2435 cookie[i].dmac_type = cp[i].dmac_type; 2436 cookie[i].dmac_address = cp[i].dmac_address; 2437 cookie[i].dmac_size = cp[i].dmac_size; 2438 } 2439 2440 *cookiepp = cookie; 2441 2442 return (DDI_SUCCESS); 2443 } 2444 2445 /*ARGSUSED*/ 2446 static int 2447 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2448 ddi_dma_cookie_t *cookiep, uint_t ccount) 2449 { 2450 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2451 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2452 rootnex_window_t *window; 2453 ddi_dma_cookie_t *cur_cookiep; 2454 2455 ASSERT(cookiep); 2456 ASSERT(ccount != 0); 2457 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2458 2459 if (dma->dp_window) { 2460 window = &dma->dp_window[dma->dp_current_win]; 2461 dma->dp_saved_cookies = window->wd_first_cookie; 2462 window->wd_first_cookie = cookiep; 2463 ASSERT(ccount == window->wd_cookie_cnt); 2464 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2465 + window->wd_first_cookie; 2466 } else { 2467 dma->dp_saved_cookies = dma->dp_cookies; 2468 dma->dp_cookies = cookiep; 2469 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2470 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2471 + dma->dp_cookies; 2472 } 2473 2474 dma->dp_need_to_switch_cookies = B_TRUE; 2475 hp->dmai_cookie = cur_cookiep; 2476 2477 return (DDI_SUCCESS); 2478 } 2479 2480 /*ARGSUSED*/ 2481 static int 2482 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2483 { 2484 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2485 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2486 rootnex_window_t *window; 2487 ddi_dma_cookie_t *cur_cookiep; 2488 ddi_dma_cookie_t *cookie_array; 2489 uint_t ccount; 2490 2491 /* check if cookies have not been switched */ 2492 if (dma->dp_need_to_switch_cookies == B_FALSE) 2493 return (DDI_SUCCESS); 2494 2495 ASSERT(dma->dp_saved_cookies); 2496 2497 if (dma->dp_window) { 2498 window = &dma->dp_window[dma->dp_current_win]; 2499 cookie_array = window->wd_first_cookie; 2500 window->wd_first_cookie = dma->dp_saved_cookies; 2501 dma->dp_saved_cookies = NULL; 2502 ccount = window->wd_cookie_cnt; 2503 cur_cookiep = (hp->dmai_cookie - cookie_array) 2504 + window->wd_first_cookie; 2505 } else { 2506 cookie_array = dma->dp_cookies; 2507 dma->dp_cookies = dma->dp_saved_cookies; 2508 dma->dp_saved_cookies = NULL; 2509 ccount = dma->dp_sglinfo.si_sgl_size; 2510 cur_cookiep = (hp->dmai_cookie - cookie_array) 2511 + dma->dp_cookies; 2512 } 2513 2514 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2515 2516 hp->dmai_cookie = cur_cookiep; 2517 2518 dma->dp_need_to_switch_cookies = B_FALSE; 2519 2520 return (DDI_SUCCESS); 2521 } 2522 2523 #endif 2524 2525 static struct as * 2526 rootnex_get_as(ddi_dma_obj_t *dmao) 2527 { 2528 struct as *asp; 2529 2530 switch (dmao->dmao_type) { 2531 case DMA_OTYP_VADDR: 2532 case DMA_OTYP_BUFVADDR: 2533 asp = dmao->dmao_obj.virt_obj.v_as; 2534 if (asp == NULL) 2535 asp = &kas; 2536 break; 2537 default: 2538 asp = NULL; 2539 break; 2540 } 2541 return (asp); 2542 } 2543 2544 /* 2545 * rootnex_verify_buffer() 2546 * verify buffer wasn't free'd 2547 */ 2548 static int 2549 rootnex_verify_buffer(rootnex_dma_t *dma) 2550 { 2551 page_t **pplist; 2552 caddr_t vaddr; 2553 uint_t pcnt; 2554 uint_t poff; 2555 page_t *pp; 2556 char b; 2557 int i; 2558 2559 /* Figure out how many pages this buffer occupies */ 2560 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2561 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2562 } else { 2563 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2564 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2565 } 2566 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2567 2568 switch (dma->dp_dma.dmao_type) { 2569 case DMA_OTYP_PAGES: 2570 /* 2571 * for a linked list of pp's walk through them to make sure 2572 * they're locked and not free. 2573 */ 2574 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2575 for (i = 0; i < pcnt; i++) { 2576 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2577 return (DDI_FAILURE); 2578 } 2579 pp = pp->p_next; 2580 } 2581 break; 2582 2583 case DMA_OTYP_VADDR: 2584 case DMA_OTYP_BUFVADDR: 2585 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2586 /* 2587 * for an array of pp's walk through them to make sure they're 2588 * not free. It's possible that they may not be locked. 2589 */ 2590 if (pplist) { 2591 for (i = 0; i < pcnt; i++) { 2592 if (PP_ISFREE(pplist[i])) { 2593 return (DDI_FAILURE); 2594 } 2595 } 2596 2597 /* For a virtual address, try to peek at each page */ 2598 } else { 2599 if (rootnex_get_as(&dma->dp_dma) == &kas) { 2600 for (i = 0; i < pcnt; i++) { 2601 if (ddi_peek8(NULL, vaddr, &b) == 2602 DDI_FAILURE) 2603 return (DDI_FAILURE); 2604 vaddr += MMU_PAGESIZE; 2605 } 2606 } 2607 } 2608 break; 2609 2610 default: 2611 cmn_err(CE_PANIC, "rootnex_verify_buffer: bad DMA object"); 2612 break; 2613 } 2614 2615 return (DDI_SUCCESS); 2616 } 2617 2618 2619 /* 2620 * rootnex_clean_dmahdl() 2621 * Clean the dma handle. This should be called on a handle alloc and an 2622 * unbind handle. Set the handle state to the default settings. 2623 */ 2624 static void 2625 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2626 { 2627 rootnex_dma_t *dma; 2628 2629 2630 dma = (rootnex_dma_t *)hp->dmai_private; 2631 2632 hp->dmai_nwin = 0; 2633 dma->dp_current_cookie = 0; 2634 dma->dp_copybuf_size = 0; 2635 dma->dp_window = NULL; 2636 dma->dp_cbaddr = NULL; 2637 dma->dp_inuse = B_FALSE; 2638 dma->dp_dvma_used = B_FALSE; 2639 dma->dp_need_to_free_cookie = B_FALSE; 2640 dma->dp_need_to_switch_cookies = B_FALSE; 2641 dma->dp_saved_cookies = NULL; 2642 dma->dp_sleep_flags = KM_PANIC; 2643 dma->dp_need_to_free_window = B_FALSE; 2644 dma->dp_partial_required = B_FALSE; 2645 dma->dp_trim_required = B_FALSE; 2646 dma->dp_sglinfo.si_copybuf_req = 0; 2647 2648 /* FMA related initialization */ 2649 hp->dmai_fault = 0; 2650 hp->dmai_fault_check = NULL; 2651 hp->dmai_fault_notify = NULL; 2652 hp->dmai_error.err_ena = 0; 2653 hp->dmai_error.err_status = DDI_FM_OK; 2654 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2655 hp->dmai_error.err_ontrap = NULL; 2656 2657 /* Cookie tracking */ 2658 hp->dmai_ncookies = 0; 2659 hp->dmai_curcookie = 0; 2660 } 2661 2662 2663 /* 2664 * rootnex_valid_alloc_parms() 2665 * Called in ddi_dma_alloc_handle path to validate its parameters. 2666 */ 2667 static int 2668 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2669 { 2670 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2671 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2672 (attr->dma_attr_granular > MMU_PAGESIZE) || 2673 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2674 return (DDI_DMA_BADATTR); 2675 } 2676 2677 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2678 return (DDI_DMA_BADATTR); 2679 } 2680 2681 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2682 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2683 attr->dma_attr_sgllen == 0) { 2684 return (DDI_DMA_BADATTR); 2685 } 2686 2687 /* We should be able to DMA into every byte offset in a page */ 2688 if (maxsegmentsize < MMU_PAGESIZE) { 2689 return (DDI_DMA_BADATTR); 2690 } 2691 2692 /* if we're bouncing on seg, seg must be <= addr_hi */ 2693 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2694 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2695 return (DDI_DMA_BADATTR); 2696 } 2697 return (DDI_SUCCESS); 2698 } 2699 2700 /* 2701 * rootnex_need_bounce_seg() 2702 * check to see if the buffer lives on both side of the seg. 2703 */ 2704 static boolean_t 2705 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2706 { 2707 ddi_dma_atyp_t buftype; 2708 rootnex_addr_t raddr; 2709 boolean_t lower_addr; 2710 boolean_t upper_addr; 2711 uint64_t offset; 2712 page_t **pplist; 2713 uint64_t paddr; 2714 uint32_t psize; 2715 uint32_t size; 2716 caddr_t vaddr; 2717 uint_t pcnt; 2718 page_t *pp; 2719 2720 pp = NULL; 2721 /* shortcuts */ 2722 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2723 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2724 buftype = dmar_object->dmao_type; 2725 size = dmar_object->dmao_size; 2726 2727 lower_addr = B_FALSE; 2728 upper_addr = B_FALSE; 2729 pcnt = 0; 2730 2731 /* 2732 * Process the first page to handle the initial offset of the buffer. 2733 * We'll use the base address we get later when we loop through all 2734 * the pages. 2735 */ 2736 if (buftype == DMA_OTYP_PAGES) { 2737 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2738 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2739 MMU_PAGEOFFSET; 2740 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2741 psize = MIN(size, (MMU_PAGESIZE - offset)); 2742 pp = pp->p_next; 2743 sglinfo->si_asp = NULL; 2744 } else if (pplist != NULL) { 2745 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2746 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2747 if (sglinfo->si_asp == NULL) { 2748 sglinfo->si_asp = &kas; 2749 } 2750 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2751 paddr += offset; 2752 psize = MIN(size, (MMU_PAGESIZE - offset)); 2753 pcnt++; 2754 } else { 2755 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2756 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2757 if (sglinfo->si_asp == NULL) { 2758 sglinfo->si_asp = &kas; 2759 } 2760 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2761 paddr += offset; 2762 psize = MIN(size, (MMU_PAGESIZE - offset)); 2763 vaddr += psize; 2764 } 2765 2766 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2767 2768 if ((raddr + psize) > sglinfo->si_segmask) { 2769 upper_addr = B_TRUE; 2770 } else { 2771 lower_addr = B_TRUE; 2772 } 2773 size -= psize; 2774 2775 /* 2776 * Walk through the rest of the pages in the buffer. Track to see 2777 * if we have pages on both sides of the segment boundary. 2778 */ 2779 while (size > 0) { 2780 /* partial or full page */ 2781 psize = MIN(size, MMU_PAGESIZE); 2782 2783 if (buftype == DMA_OTYP_PAGES) { 2784 /* get the paddr from the page_t */ 2785 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2786 paddr = pfn_to_pa(pp->p_pagenum); 2787 pp = pp->p_next; 2788 } else if (pplist != NULL) { 2789 /* index into the array of page_t's to get the paddr */ 2790 ASSERT(!PP_ISFREE(pplist[pcnt])); 2791 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2792 pcnt++; 2793 } else { 2794 /* call into the VM to get the paddr */ 2795 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2796 vaddr)); 2797 vaddr += psize; 2798 } 2799 2800 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2801 2802 if ((raddr + psize) > sglinfo->si_segmask) { 2803 upper_addr = B_TRUE; 2804 } else { 2805 lower_addr = B_TRUE; 2806 } 2807 /* 2808 * if the buffer lives both above and below the segment 2809 * boundary, or the current page is the page immediately 2810 * after the segment, we will use a copy/bounce buffer for 2811 * all pages > seg. 2812 */ 2813 if ((lower_addr && upper_addr) || 2814 (raddr == (sglinfo->si_segmask + 1))) { 2815 return (B_TRUE); 2816 } 2817 2818 size -= psize; 2819 } 2820 2821 return (B_FALSE); 2822 } 2823 2824 /* 2825 * rootnex_get_sgl() 2826 * Called in bind fastpath to get the sgl. Most of this will be replaced 2827 * with a call to the vm layer when vm2.0 comes around... 2828 */ 2829 static void 2830 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2831 rootnex_sglinfo_t *sglinfo) 2832 { 2833 ddi_dma_atyp_t buftype; 2834 rootnex_addr_t raddr; 2835 uint64_t last_page; 2836 uint64_t offset; 2837 uint64_t addrhi; 2838 uint64_t addrlo; 2839 uint64_t maxseg; 2840 page_t **pplist; 2841 uint64_t paddr; 2842 uint32_t psize; 2843 uint32_t size; 2844 caddr_t vaddr; 2845 uint_t pcnt; 2846 page_t *pp; 2847 uint_t cnt; 2848 2849 pp = NULL; 2850 /* shortcuts */ 2851 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2852 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2853 maxseg = sglinfo->si_max_cookie_size; 2854 buftype = dmar_object->dmao_type; 2855 addrhi = sglinfo->si_max_addr; 2856 addrlo = sglinfo->si_min_addr; 2857 size = dmar_object->dmao_size; 2858 2859 pcnt = 0; 2860 cnt = 0; 2861 2862 2863 /* 2864 * check to see if we need to use the copy buffer for pages over 2865 * the segment attr. 2866 */ 2867 sglinfo->si_bounce_on_seg = B_FALSE; 2868 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2869 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2870 dmar_object, sglinfo); 2871 } 2872 2873 /* 2874 * if we were passed down a linked list of pages, i.e. pointer to 2875 * page_t, use this to get our physical address and buf offset. 2876 */ 2877 if (buftype == DMA_OTYP_PAGES) { 2878 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2879 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2880 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2881 MMU_PAGEOFFSET; 2882 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2883 psize = MIN(size, (MMU_PAGESIZE - offset)); 2884 pp = pp->p_next; 2885 sglinfo->si_asp = NULL; 2886 2887 /* 2888 * We weren't passed down a linked list of pages, but if we were passed 2889 * down an array of pages, use this to get our physical address and buf 2890 * offset. 2891 */ 2892 } else if (pplist != NULL) { 2893 ASSERT((buftype == DMA_OTYP_VADDR) || 2894 (buftype == DMA_OTYP_BUFVADDR)); 2895 2896 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2897 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2898 if (sglinfo->si_asp == NULL) { 2899 sglinfo->si_asp = &kas; 2900 } 2901 2902 ASSERT(!PP_ISFREE(pplist[pcnt])); 2903 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2904 paddr += offset; 2905 psize = MIN(size, (MMU_PAGESIZE - offset)); 2906 pcnt++; 2907 2908 /* 2909 * All we have is a virtual address, we'll need to call into the VM 2910 * to get the physical address. 2911 */ 2912 } else { 2913 ASSERT((buftype == DMA_OTYP_VADDR) || 2914 (buftype == DMA_OTYP_BUFVADDR)); 2915 2916 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2917 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2918 if (sglinfo->si_asp == NULL) { 2919 sglinfo->si_asp = &kas; 2920 } 2921 2922 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2923 paddr += offset; 2924 psize = MIN(size, (MMU_PAGESIZE - offset)); 2925 vaddr += psize; 2926 } 2927 2928 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 2929 2930 /* 2931 * Setup the first cookie with the physical address of the page and the 2932 * size of the page (which takes into account the initial offset into 2933 * the page. 2934 */ 2935 sgl[cnt].dmac_laddress = raddr; 2936 sgl[cnt].dmac_size = psize; 2937 sgl[cnt].dmac_type = 0; 2938 2939 /* 2940 * Save away the buffer offset into the page. We'll need this later in 2941 * the copy buffer code to help figure out the page index within the 2942 * buffer and the offset into the current page. 2943 */ 2944 sglinfo->si_buf_offset = offset; 2945 2946 /* 2947 * If we are using the copy buffer for anything over the segment 2948 * boundary, and this page is over the segment boundary. 2949 * OR 2950 * if the DMA engine can't reach the physical address. 2951 */ 2952 if (((sglinfo->si_bounce_on_seg) && 2953 ((raddr + psize) > sglinfo->si_segmask)) || 2954 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2955 /* 2956 * Increase how much copy buffer we use. We always increase by 2957 * pagesize so we don't have to worry about converting offsets. 2958 * Set a flag in the cookies dmac_type to indicate that it uses 2959 * the copy buffer. If this isn't the last cookie, go to the 2960 * next cookie (since we separate each page which uses the copy 2961 * buffer in case the copy buffer is not physically contiguous. 2962 */ 2963 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2964 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2965 if ((cnt + 1) < sglinfo->si_max_pages) { 2966 cnt++; 2967 sgl[cnt].dmac_laddress = 0; 2968 sgl[cnt].dmac_size = 0; 2969 sgl[cnt].dmac_type = 0; 2970 } 2971 } 2972 2973 /* 2974 * save this page's physical address so we can figure out if the next 2975 * page is physically contiguous. Keep decrementing size until we are 2976 * done with the buffer. 2977 */ 2978 last_page = raddr & MMU_PAGEMASK; 2979 size -= psize; 2980 2981 while (size > 0) { 2982 /* Get the size for this page (i.e. partial or full page) */ 2983 psize = MIN(size, MMU_PAGESIZE); 2984 2985 if (buftype == DMA_OTYP_PAGES) { 2986 /* get the paddr from the page_t */ 2987 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2988 paddr = pfn_to_pa(pp->p_pagenum); 2989 pp = pp->p_next; 2990 } else if (pplist != NULL) { 2991 /* index into the array of page_t's to get the paddr */ 2992 ASSERT(!PP_ISFREE(pplist[pcnt])); 2993 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2994 pcnt++; 2995 } else { 2996 /* call into the VM to get the paddr */ 2997 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2998 vaddr)); 2999 vaddr += psize; 3000 } 3001 3002 raddr = ROOTNEX_PADDR_TO_RBASE(paddr); 3003 3004 /* 3005 * If we are using the copy buffer for anything over the 3006 * segment boundary, and this page is over the segment 3007 * boundary. 3008 * OR 3009 * if the DMA engine can't reach the physical address. 3010 */ 3011 if (((sglinfo->si_bounce_on_seg) && 3012 ((raddr + psize) > sglinfo->si_segmask)) || 3013 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 3014 3015 sglinfo->si_copybuf_req += MMU_PAGESIZE; 3016 3017 /* 3018 * if there is something in the current cookie, go to 3019 * the next one. We only want one page in a cookie which 3020 * uses the copybuf since the copybuf doesn't have to 3021 * be physically contiguous. 3022 */ 3023 if (sgl[cnt].dmac_size != 0) { 3024 cnt++; 3025 } 3026 sgl[cnt].dmac_laddress = raddr; 3027 sgl[cnt].dmac_size = psize; 3028 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 3029 /* if this isn't the last cookie, go to the next one */ 3030 if ((cnt + 1) < sglinfo->si_max_pages) { 3031 cnt++; 3032 sgl[cnt].dmac_laddress = 0; 3033 sgl[cnt].dmac_size = 0; 3034 sgl[cnt].dmac_type = 0; 3035 } 3036 3037 /* 3038 * this page didn't need the copy buffer, if it's not physically 3039 * contiguous, or it would put us over a segment boundary, or it 3040 * puts us over the max cookie size, or the current sgl doesn't 3041 * have anything in it. 3042 */ 3043 } else if (((last_page + MMU_PAGESIZE) != raddr) || 3044 !(raddr & sglinfo->si_segmask) || 3045 ((sgl[cnt].dmac_size + psize) > maxseg) || 3046 (sgl[cnt].dmac_size == 0)) { 3047 /* 3048 * if we're not already in a new cookie, go to the next 3049 * cookie. 3050 */ 3051 if (sgl[cnt].dmac_size != 0) { 3052 cnt++; 3053 } 3054 3055 /* save the cookie information */ 3056 sgl[cnt].dmac_laddress = raddr; 3057 sgl[cnt].dmac_size = psize; 3058 sgl[cnt].dmac_type = 0; 3059 3060 /* 3061 * this page didn't need the copy buffer, it is physically 3062 * contiguous with the last page, and it's <= the max cookie 3063 * size. 3064 */ 3065 } else { 3066 sgl[cnt].dmac_size += psize; 3067 3068 /* 3069 * If this cookie is used up, and more cookies 3070 * are available, then move onto the next one. 3071 */ 3072 if ((sgl[cnt].dmac_size == maxseg) && 3073 ((cnt + 1) < sglinfo->si_max_pages)) { 3074 cnt++; 3075 sgl[cnt].dmac_laddress = 0; 3076 sgl[cnt].dmac_size = 0; 3077 sgl[cnt].dmac_type = 0; 3078 } 3079 } 3080 3081 /* 3082 * save this page's physical address so we can figure out if the 3083 * next page is physically contiguous. Keep decrementing size 3084 * until we are done with the buffer. 3085 */ 3086 last_page = raddr; 3087 size -= psize; 3088 } 3089 3090 /* we're done, save away how many cookies the sgl has */ 3091 if (sgl[cnt].dmac_size == 0) { 3092 ASSERT(cnt < sglinfo->si_max_pages); 3093 sglinfo->si_sgl_size = cnt; 3094 } else { 3095 sglinfo->si_sgl_size = cnt + 1; 3096 } 3097 } 3098 3099 static void 3100 rootnex_dvma_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 3101 rootnex_sglinfo_t *sglinfo) 3102 { 3103 uint64_t offset; 3104 uint64_t maxseg; 3105 uint64_t dvaddr; 3106 struct dvmaseg *dvs; 3107 uint64_t paddr; 3108 uint32_t psize, ssize; 3109 uint32_t size; 3110 uint_t cnt; 3111 int physcontig; 3112 3113 ASSERT(dmar_object->dmao_type == DMA_OTYP_DVADDR); 3114 3115 /* shortcuts */ 3116 maxseg = sglinfo->si_max_cookie_size; 3117 size = dmar_object->dmao_size; 3118 3119 cnt = 0; 3120 sglinfo->si_bounce_on_seg = B_FALSE; 3121 3122 dvs = dmar_object->dmao_obj.dvma_obj.dv_seg; 3123 offset = dmar_object->dmao_obj.dvma_obj.dv_off; 3124 ssize = dvs->dvs_len; 3125 paddr = dvs->dvs_start; 3126 paddr += offset; 3127 psize = MIN(ssize, (maxseg - offset)); 3128 dvaddr = paddr + psize; 3129 ssize -= psize; 3130 3131 sgl[cnt].dmac_laddress = paddr; 3132 sgl[cnt].dmac_size = psize; 3133 sgl[cnt].dmac_type = 0; 3134 3135 size -= psize; 3136 while (size > 0) { 3137 if (ssize == 0) { 3138 dvs++; 3139 ssize = dvs->dvs_len; 3140 dvaddr = dvs->dvs_start; 3141 physcontig = 0; 3142 } else 3143 physcontig = 1; 3144 3145 paddr = dvaddr; 3146 psize = MIN(ssize, maxseg); 3147 dvaddr += psize; 3148 ssize -= psize; 3149 3150 if (!physcontig || !(paddr & sglinfo->si_segmask) || 3151 ((sgl[cnt].dmac_size + psize) > maxseg) || 3152 (sgl[cnt].dmac_size == 0)) { 3153 /* 3154 * if we're not already in a new cookie, go to the next 3155 * cookie. 3156 */ 3157 if (sgl[cnt].dmac_size != 0) { 3158 cnt++; 3159 } 3160 3161 /* save the cookie information */ 3162 sgl[cnt].dmac_laddress = paddr; 3163 sgl[cnt].dmac_size = psize; 3164 sgl[cnt].dmac_type = 0; 3165 } else { 3166 sgl[cnt].dmac_size += psize; 3167 3168 /* 3169 * If this cookie is used up, and more cookies 3170 * are available, then move onto the next one. 3171 */ 3172 if ((sgl[cnt].dmac_size == maxseg) && 3173 ((cnt + 1) < sglinfo->si_max_pages)) { 3174 cnt++; 3175 sgl[cnt].dmac_laddress = 0; 3176 sgl[cnt].dmac_size = 0; 3177 sgl[cnt].dmac_type = 0; 3178 } 3179 } 3180 size -= psize; 3181 } 3182 3183 /* we're done, save away how many cookies the sgl has */ 3184 if (sgl[cnt].dmac_size == 0) { 3185 sglinfo->si_sgl_size = cnt; 3186 } else { 3187 sglinfo->si_sgl_size = cnt + 1; 3188 } 3189 } 3190 3191 /* 3192 * rootnex_bind_slowpath() 3193 * Call in the bind path if the calling driver can't use the sgl without 3194 * modifying it. We either need to use the copy buffer and/or we will end up 3195 * with a partial bind. 3196 */ 3197 static int 3198 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3199 rootnex_dma_t *dma, ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3200 { 3201 rootnex_sglinfo_t *sinfo; 3202 rootnex_window_t *window; 3203 ddi_dma_cookie_t *cookie; 3204 size_t copybuf_used; 3205 size_t dmac_size; 3206 boolean_t partial; 3207 off_t cur_offset; 3208 page_t *cur_pp; 3209 major_t mnum; 3210 int e; 3211 int i; 3212 3213 3214 sinfo = &dma->dp_sglinfo; 3215 copybuf_used = 0; 3216 partial = B_FALSE; 3217 3218 /* 3219 * If we're using the copybuf, set the copybuf state in dma struct. 3220 * Needs to be first since it sets the copy buffer size. 3221 */ 3222 if (sinfo->si_copybuf_req != 0) { 3223 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 3224 if (e != DDI_SUCCESS) { 3225 return (e); 3226 } 3227 } else { 3228 dma->dp_copybuf_size = 0; 3229 } 3230 3231 /* 3232 * Figure out if we need to do a partial mapping. If so, figure out 3233 * if we need to trim the buffers when we munge the sgl. 3234 */ 3235 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 3236 (dmao->dmao_size > dma->dp_maxxfer) || 3237 ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 3238 dma->dp_partial_required = B_TRUE; 3239 if (attr->dma_attr_granular != 1) { 3240 dma->dp_trim_required = B_TRUE; 3241 } 3242 } else { 3243 dma->dp_partial_required = B_FALSE; 3244 dma->dp_trim_required = B_FALSE; 3245 } 3246 3247 /* If we need to do a partial bind, make sure the driver supports it */ 3248 if (dma->dp_partial_required && 3249 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3250 3251 mnum = ddi_driver_major(dma->dp_dip); 3252 /* 3253 * patchable which allows us to print one warning per major 3254 * number. 3255 */ 3256 if ((rootnex_bind_warn) && 3257 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3258 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3259 cmn_err(CE_WARN, "!%s: coding error detected, the " 3260 "driver is using ddi_dma_attr(9S) incorrectly. " 3261 "There is a small risk of data corruption in " 3262 "particular with large I/Os. The driver should be " 3263 "replaced with a corrected version for proper " 3264 "system operation. To disable this warning, add " 3265 "'set rootnex:rootnex_bind_warn=0' to " 3266 "/etc/system(5).", ddi_driver_name(dma->dp_dip)); 3267 } 3268 return (DDI_DMA_TOOBIG); 3269 } 3270 3271 /* 3272 * we might need multiple windows, setup state to handle them. In this 3273 * code path, we will have at least one window. 3274 */ 3275 e = rootnex_setup_windows(hp, dma, attr, dmao, kmflag); 3276 if (e != DDI_SUCCESS) { 3277 rootnex_teardown_copybuf(dma); 3278 return (e); 3279 } 3280 3281 window = &dma->dp_window[0]; 3282 cookie = &dma->dp_cookies[0]; 3283 cur_offset = 0; 3284 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3285 if (dmao->dmao_type == DMA_OTYP_PAGES) { 3286 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3287 } 3288 3289 /* loop though all the cookies we got back from get_sgl() */ 3290 for (i = 0; i < sinfo->si_sgl_size; i++) { 3291 /* 3292 * If we're using the copy buffer, check this cookie and setup 3293 * its associated copy buffer state. If this cookie uses the 3294 * copy buffer, make sure we sync this window during dma_sync. 3295 */ 3296 if (dma->dp_copybuf_size > 0) { 3297 rootnex_setup_cookie(dmao, dma, cookie, 3298 cur_offset, ©buf_used, &cur_pp); 3299 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3300 window->wd_dosync = B_TRUE; 3301 } 3302 } 3303 3304 /* 3305 * save away the cookie size, since it could be modified in 3306 * the windowing code. 3307 */ 3308 dmac_size = cookie->dmac_size; 3309 3310 /* if we went over max copybuf size */ 3311 if (dma->dp_copybuf_size && 3312 (copybuf_used > dma->dp_copybuf_size)) { 3313 partial = B_TRUE; 3314 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3315 cookie, cur_offset, ©buf_used); 3316 if (e != DDI_SUCCESS) { 3317 rootnex_teardown_copybuf(dma); 3318 rootnex_teardown_windows(dma); 3319 return (e); 3320 } 3321 3322 /* 3323 * if the coookie uses the copy buffer, make sure the 3324 * new window we just moved to is set to sync. 3325 */ 3326 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3327 window->wd_dosync = B_TRUE; 3328 } 3329 ROOTNEX_DPROBE1(rootnex__copybuf__window, dev_info_t *, 3330 dma->dp_dip); 3331 3332 /* if the cookie cnt == max sgllen, move to the next window */ 3333 } else if (window->wd_cookie_cnt >= 3334 (unsigned)attr->dma_attr_sgllen) { 3335 partial = B_TRUE; 3336 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3337 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3338 cookie, attr, cur_offset); 3339 if (e != DDI_SUCCESS) { 3340 rootnex_teardown_copybuf(dma); 3341 rootnex_teardown_windows(dma); 3342 return (e); 3343 } 3344 3345 /* 3346 * if the coookie uses the copy buffer, make sure the 3347 * new window we just moved to is set to sync. 3348 */ 3349 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3350 window->wd_dosync = B_TRUE; 3351 } 3352 ROOTNEX_DPROBE1(rootnex__sgllen__window, dev_info_t *, 3353 dma->dp_dip); 3354 3355 /* else if we will be over maxxfer */ 3356 } else if ((window->wd_size + dmac_size) > 3357 dma->dp_maxxfer) { 3358 partial = B_TRUE; 3359 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3360 cookie); 3361 if (e != DDI_SUCCESS) { 3362 rootnex_teardown_copybuf(dma); 3363 rootnex_teardown_windows(dma); 3364 return (e); 3365 } 3366 3367 /* 3368 * if the coookie uses the copy buffer, make sure the 3369 * new window we just moved to is set to sync. 3370 */ 3371 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3372 window->wd_dosync = B_TRUE; 3373 } 3374 ROOTNEX_DPROBE1(rootnex__maxxfer__window, dev_info_t *, 3375 dma->dp_dip); 3376 3377 /* else this cookie fits in the current window */ 3378 } else { 3379 window->wd_cookie_cnt++; 3380 window->wd_size += dmac_size; 3381 } 3382 3383 /* track our offset into the buffer, go to the next cookie */ 3384 ASSERT(dmac_size <= dmao->dmao_size); 3385 ASSERT(cookie->dmac_size <= dmac_size); 3386 cur_offset += dmac_size; 3387 cookie++; 3388 } 3389 3390 /* if we ended up with a zero sized window in the end, clean it up */ 3391 if (window->wd_size == 0) { 3392 hp->dmai_nwin--; 3393 window--; 3394 } 3395 3396 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3397 3398 if (!partial) { 3399 return (DDI_DMA_MAPPED); 3400 } 3401 3402 ASSERT(dma->dp_partial_required); 3403 return (DDI_DMA_PARTIAL_MAP); 3404 } 3405 3406 /* 3407 * rootnex_setup_copybuf() 3408 * Called in bind slowpath. Figures out if we're going to use the copy 3409 * buffer, and if we do, sets up the basic state to handle it. 3410 */ 3411 static int 3412 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3413 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3414 { 3415 rootnex_sglinfo_t *sinfo; 3416 ddi_dma_attr_t lattr; 3417 size_t max_copybuf; 3418 int cansleep; 3419 int e; 3420 3421 ASSERT(!dma->dp_dvma_used); 3422 3423 sinfo = &dma->dp_sglinfo; 3424 3425 /* read this first so it's consistent through the routine */ 3426 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3427 3428 /* We need to call into the rootnex on ddi_dma_sync() */ 3429 hp->dmai_rflags &= ~DMP_NOSYNC; 3430 3431 /* make sure the copybuf size <= the max size */ 3432 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3433 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3434 3435 3436 /* convert the sleep flags */ 3437 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3438 cansleep = 1; 3439 } else { 3440 cansleep = 0; 3441 } 3442 3443 /* 3444 * Allocate the actual copy buffer. This needs to fit within the DMA 3445 * engine limits, so we can't use kmem_alloc... We don't need 3446 * contiguous memory (sgllen) since we will be forcing windows on 3447 * sgllen anyway. 3448 */ 3449 lattr = *attr; 3450 lattr.dma_attr_align = MMU_PAGESIZE; 3451 lattr.dma_attr_sgllen = -1; /* no limit */ 3452 /* 3453 * if we're using the copy buffer because of seg, use that for our 3454 * upper address limit. 3455 */ 3456 if (sinfo->si_bounce_on_seg) { 3457 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3458 } 3459 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3460 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3461 if (e != DDI_SUCCESS) { 3462 return (DDI_DMA_NORESOURCES); 3463 } 3464 3465 ROOTNEX_DPROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3466 size_t, dma->dp_copybuf_size); 3467 3468 return (DDI_SUCCESS); 3469 } 3470 3471 3472 /* 3473 * rootnex_setup_windows() 3474 * Called in bind slowpath to setup the window state. We always have windows 3475 * in the slowpath. Even if the window count = 1. 3476 */ 3477 static int 3478 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3479 ddi_dma_attr_t *attr, ddi_dma_obj_t *dmao, int kmflag) 3480 { 3481 rootnex_window_t *windowp; 3482 rootnex_sglinfo_t *sinfo; 3483 size_t copy_state_size; 3484 size_t win_state_size; 3485 size_t state_available; 3486 size_t space_needed; 3487 uint_t copybuf_win; 3488 uint_t maxxfer_win; 3489 size_t space_used; 3490 uint_t sglwin; 3491 3492 3493 sinfo = &dma->dp_sglinfo; 3494 3495 dma->dp_current_win = 0; 3496 hp->dmai_nwin = 0; 3497 3498 /* If we don't need to do a partial, we only have one window */ 3499 if (!dma->dp_partial_required) { 3500 dma->dp_max_win = 1; 3501 3502 /* 3503 * we need multiple windows, need to figure out the worse case number 3504 * of windows. 3505 */ 3506 } else { 3507 /* 3508 * if we need windows because we need more copy buffer that 3509 * we allow, the worse case number of windows we could need 3510 * here would be (copybuf space required / copybuf space that 3511 * we have) plus one for remainder, and plus 2 to handle the 3512 * extra pages on the trim for the first and last pages of the 3513 * buffer (a page is the minimum window size so under the right 3514 * attr settings, you could have a window for each page). 3515 * The last page will only be hit here if the size is not a 3516 * multiple of the granularity (which theoretically shouldn't 3517 * be the case but never has been enforced, so we could have 3518 * broken things without it). 3519 */ 3520 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3521 ASSERT(dma->dp_copybuf_size > 0); 3522 copybuf_win = (sinfo->si_copybuf_req / 3523 dma->dp_copybuf_size) + 1 + 2; 3524 } else { 3525 copybuf_win = 0; 3526 } 3527 3528 /* 3529 * if we need windows because we have more cookies than the H/W 3530 * can handle, the number of windows we would need here would 3531 * be (cookie count / cookies count H/W supports minus 1[for 3532 * trim]) plus one for remainder. 3533 */ 3534 if ((unsigned)attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3535 sglwin = (sinfo->si_sgl_size / 3536 (attr->dma_attr_sgllen - 1)) + 1; 3537 } else { 3538 sglwin = 0; 3539 } 3540 3541 /* 3542 * if we need windows because we're binding more memory than the 3543 * H/W can transfer at once, the number of windows we would need 3544 * here would be (xfer count / max xfer H/W supports) plus one 3545 * for remainder, and plus 2 to handle the extra pages on the 3546 * trim (see above comment about trim) 3547 */ 3548 if (dmao->dmao_size > dma->dp_maxxfer) { 3549 maxxfer_win = (dmao->dmao_size / 3550 dma->dp_maxxfer) + 1 + 2; 3551 } else { 3552 maxxfer_win = 0; 3553 } 3554 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3555 ASSERT(dma->dp_max_win > 0); 3556 } 3557 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3558 3559 /* 3560 * Get space for window and potential copy buffer state. Before we 3561 * go and allocate memory, see if we can get away with using what's 3562 * left in the pre-allocted state or the dynamically allocated sgl. 3563 */ 3564 space_used = (uintptr_t)(sinfo->si_sgl_size * 3565 sizeof (ddi_dma_cookie_t)); 3566 3567 /* if we dynamically allocated space for the cookies */ 3568 if (dma->dp_need_to_free_cookie) { 3569 /* if we have more space in the pre-allocted buffer, use it */ 3570 ASSERT(space_used <= dma->dp_cookie_size); 3571 if ((dma->dp_cookie_size - space_used) <= 3572 rootnex_state->r_prealloc_size) { 3573 state_available = rootnex_state->r_prealloc_size; 3574 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3575 3576 /* 3577 * else, we have more free space in the dynamically allocated 3578 * buffer, i.e. the buffer wasn't worse case fragmented so we 3579 * didn't need a lot of cookies. 3580 */ 3581 } else { 3582 state_available = dma->dp_cookie_size - space_used; 3583 windowp = (rootnex_window_t *) 3584 &dma->dp_cookies[sinfo->si_sgl_size]; 3585 } 3586 3587 /* we used the pre-alloced buffer */ 3588 } else { 3589 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3590 state_available = rootnex_state->r_prealloc_size - space_used; 3591 windowp = (rootnex_window_t *) 3592 &dma->dp_cookies[sinfo->si_sgl_size]; 3593 } 3594 3595 /* 3596 * figure out how much state we need to track the copy buffer. Add an 3597 * addition 8 bytes for pointer alignemnt later. 3598 */ 3599 if (dma->dp_copybuf_size > 0) { 3600 copy_state_size = sinfo->si_max_pages * 3601 sizeof (rootnex_pgmap_t); 3602 } else { 3603 copy_state_size = 0; 3604 } 3605 /* add an additional 8 bytes for pointer alignment */ 3606 space_needed = win_state_size + copy_state_size + 0x8; 3607 3608 /* if we have enough space already, use it */ 3609 if (state_available >= space_needed) { 3610 dma->dp_window = windowp; 3611 dma->dp_need_to_free_window = B_FALSE; 3612 3613 /* not enough space, need to allocate more. */ 3614 } else { 3615 dma->dp_window = kmem_alloc(space_needed, kmflag); 3616 if (dma->dp_window == NULL) { 3617 return (DDI_DMA_NORESOURCES); 3618 } 3619 dma->dp_need_to_free_window = B_TRUE; 3620 dma->dp_window_size = space_needed; 3621 ROOTNEX_DPROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3622 dma->dp_dip, size_t, space_needed); 3623 } 3624 3625 /* 3626 * we allocate copy buffer state and window state at the same time. 3627 * setup our copy buffer state pointers. Make sure it's aligned. 3628 */ 3629 if (dma->dp_copybuf_size > 0) { 3630 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3631 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3632 3633 } else { 3634 dma->dp_pgmap = NULL; 3635 } 3636 3637 return (DDI_SUCCESS); 3638 } 3639 3640 3641 /* 3642 * rootnex_teardown_copybuf() 3643 * cleans up after rootnex_setup_copybuf() 3644 */ 3645 static void 3646 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3647 { 3648 /* if we allocated a copy buffer, free it */ 3649 if (dma->dp_cbaddr != NULL) { 3650 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3651 } 3652 } 3653 3654 3655 /* 3656 * rootnex_teardown_windows() 3657 * cleans up after rootnex_setup_windows() 3658 */ 3659 static void 3660 rootnex_teardown_windows(rootnex_dma_t *dma) 3661 { 3662 /* 3663 * if we had to allocate window state on the last bind (because we 3664 * didn't have enough pre-allocated space in the handle), free it. 3665 */ 3666 if (dma->dp_need_to_free_window) { 3667 kmem_free(dma->dp_window, dma->dp_window_size); 3668 } 3669 } 3670 3671 3672 /* 3673 * rootnex_init_win() 3674 * Called in bind slow path during creation of a new window. Initializes 3675 * window state to default values. 3676 */ 3677 /*ARGSUSED*/ 3678 static void 3679 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3680 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3681 { 3682 hp->dmai_nwin++; 3683 window->wd_dosync = B_FALSE; 3684 window->wd_offset = cur_offset; 3685 window->wd_size = 0; 3686 window->wd_first_cookie = cookie; 3687 window->wd_cookie_cnt = 0; 3688 window->wd_trim.tr_trim_first = B_FALSE; 3689 window->wd_trim.tr_trim_last = B_FALSE; 3690 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3691 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3692 } 3693 3694 3695 /* 3696 * rootnex_setup_cookie() 3697 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3698 * the sgl uses the copy buffer, we need to go through each cookie, figure 3699 * out if it uses the copy buffer, and if it does, save away everything we'll 3700 * need during sync. 3701 */ 3702 static void 3703 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3704 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3705 page_t **cur_pp) 3706 { 3707 boolean_t copybuf_sz_power_2; 3708 rootnex_sglinfo_t *sinfo; 3709 paddr_t paddr; 3710 uint_t pidx; 3711 uint_t pcnt; 3712 off_t poff; 3713 pfn_t pfn; 3714 3715 ASSERT(dmar_object->dmao_type != DMA_OTYP_DVADDR); 3716 3717 sinfo = &dma->dp_sglinfo; 3718 3719 /* 3720 * Calculate the page index relative to the start of the buffer. The 3721 * index to the current page for our buffer is the offset into the 3722 * first page of the buffer plus our current offset into the buffer 3723 * itself, shifted of course... 3724 */ 3725 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3726 ASSERT(pidx < sinfo->si_max_pages); 3727 3728 /* if this cookie uses the copy buffer */ 3729 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3730 /* 3731 * NOTE: we know that since this cookie uses the copy buffer, it 3732 * is <= MMU_PAGESIZE. 3733 */ 3734 3735 /* 3736 * get the offset into the page. For the 64-bit kernel, get the 3737 * pfn which we'll use with seg kpm. 3738 */ 3739 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3740 /* mfn_to_pfn() is a NOP on i86pc */ 3741 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3742 3743 /* figure out if the copybuf size is a power of 2 */ 3744 if (!ISP2(dma->dp_copybuf_size)) { 3745 copybuf_sz_power_2 = B_FALSE; 3746 } else { 3747 copybuf_sz_power_2 = B_TRUE; 3748 } 3749 3750 /* This page uses the copy buffer */ 3751 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3752 3753 /* 3754 * save the copy buffer KVA that we'll use with this page. 3755 * if we still fit within the copybuf, it's a simple add. 3756 * otherwise, we need to wrap over using & or % accordingly. 3757 */ 3758 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3759 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3760 *copybuf_used; 3761 } else { 3762 if (copybuf_sz_power_2) { 3763 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3764 (uintptr_t)dma->dp_cbaddr + 3765 (*copybuf_used & 3766 (dma->dp_copybuf_size - 1))); 3767 } else { 3768 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3769 (uintptr_t)dma->dp_cbaddr + 3770 (*copybuf_used % dma->dp_copybuf_size)); 3771 } 3772 } 3773 3774 /* 3775 * over write the cookie physical address with the address of 3776 * the physical address of the copy buffer page that we will 3777 * use. 3778 */ 3779 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3780 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3781 3782 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 3783 3784 /* if we have a kernel VA, it's easy, just save that address */ 3785 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3786 (sinfo->si_asp == &kas)) { 3787 /* 3788 * save away the page aligned virtual address of the 3789 * driver buffer. Offsets are handled in the sync code. 3790 */ 3791 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3792 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3793 & MMU_PAGEMASK); 3794 3795 /* we don't have a kernel VA. We need one for the bcopy. */ 3796 } else { 3797 /* 3798 * for the 64-bit kernel, it's easy. We use seg kpm to 3799 * get a Kernel VA for the corresponding pfn. 3800 */ 3801 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3802 /* go to the next page_t */ 3803 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3804 *cur_pp = (*cur_pp)->p_next; 3805 } 3806 } 3807 3808 /* add to the copy buffer count */ 3809 *copybuf_used += MMU_PAGESIZE; 3810 3811 /* 3812 * This cookie doesn't use the copy buffer. Walk through the pages this 3813 * cookie occupies to reflect this. 3814 */ 3815 } else { 3816 /* 3817 * figure out how many pages the cookie occupies. We need to 3818 * use the original page offset of the buffer and the cookies 3819 * offset in the buffer to do this. 3820 */ 3821 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3822 pcnt = mmu_btopr(cookie->dmac_size + poff); 3823 3824 while (pcnt > 0) { 3825 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3826 3827 /* 3828 * we need to update pidx and cur_pp or we'll loose 3829 * track of where we are. 3830 */ 3831 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3832 *cur_pp = (*cur_pp)->p_next; 3833 } 3834 pidx++; 3835 pcnt--; 3836 } 3837 } 3838 } 3839 3840 3841 /* 3842 * rootnex_sgllen_window_boundary() 3843 * Called in the bind slow path when the next cookie causes us to exceed (in 3844 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3845 * length supported by the DMA H/W. 3846 */ 3847 static int 3848 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3849 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3850 off_t cur_offset) 3851 { 3852 off_t new_offset; 3853 size_t trim_sz; 3854 off_t coffset; 3855 3856 3857 /* 3858 * if we know we'll never have to trim, it's pretty easy. Just move to 3859 * the next window and init it. We're done. 3860 */ 3861 if (!dma->dp_trim_required) { 3862 (*windowp)++; 3863 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3864 (*windowp)->wd_cookie_cnt++; 3865 (*windowp)->wd_size = cookie->dmac_size; 3866 return (DDI_SUCCESS); 3867 } 3868 3869 /* figure out how much we need to trim from the window */ 3870 ASSERT(attr->dma_attr_granular != 0); 3871 if (dma->dp_granularity_power_2) { 3872 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3873 } else { 3874 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3875 } 3876 3877 /* The window's a whole multiple of granularity. We're done */ 3878 if (trim_sz == 0) { 3879 (*windowp)++; 3880 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3881 (*windowp)->wd_cookie_cnt++; 3882 (*windowp)->wd_size = cookie->dmac_size; 3883 return (DDI_SUCCESS); 3884 } 3885 3886 /* 3887 * The window's not a whole multiple of granularity, since we know this 3888 * is due to the sgllen, we need to go back to the last cookie and trim 3889 * that one, add the left over part of the old cookie into the new 3890 * window, and then add in the new cookie into the new window. 3891 */ 3892 3893 /* 3894 * make sure the driver isn't making us do something bad... Trimming and 3895 * sgllen == 1 don't go together. 3896 */ 3897 if (attr->dma_attr_sgllen == 1) { 3898 return (DDI_DMA_NOMAPPING); 3899 } 3900 3901 /* 3902 * first, setup the current window to account for the trim. Need to go 3903 * back to the last cookie for this. 3904 */ 3905 cookie--; 3906 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3907 (*windowp)->wd_trim.tr_last_cookie = cookie; 3908 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3909 ASSERT(cookie->dmac_size > trim_sz); 3910 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3911 (*windowp)->wd_size -= trim_sz; 3912 3913 /* save the buffer offsets for the next window */ 3914 coffset = cookie->dmac_size - trim_sz; 3915 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3916 3917 /* 3918 * set this now in case this is the first window. all other cases are 3919 * set in dma_win() 3920 */ 3921 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3922 3923 /* 3924 * initialize the next window using what's left over in the previous 3925 * cookie. 3926 */ 3927 (*windowp)++; 3928 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3929 (*windowp)->wd_cookie_cnt++; 3930 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3931 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3932 (*windowp)->wd_trim.tr_first_size = trim_sz; 3933 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3934 (*windowp)->wd_dosync = B_TRUE; 3935 } 3936 3937 /* 3938 * now go back to the current cookie and add it to the new window. set 3939 * the new window size to the what was left over from the previous 3940 * cookie and what's in the current cookie. 3941 */ 3942 cookie++; 3943 (*windowp)->wd_cookie_cnt++; 3944 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3945 3946 /* 3947 * trim plus the next cookie could put us over maxxfer (a cookie can be 3948 * a max size of maxxfer). Handle that case. 3949 */ 3950 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3951 /* 3952 * maxxfer is already a whole multiple of granularity, and this 3953 * trim will be <= the previous trim (since a cookie can't be 3954 * larger than maxxfer). Make things simple here. 3955 */ 3956 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3957 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3958 (*windowp)->wd_trim.tr_last_cookie = cookie; 3959 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3960 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3961 (*windowp)->wd_size -= trim_sz; 3962 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3963 3964 /* save the buffer offsets for the next window */ 3965 coffset = cookie->dmac_size - trim_sz; 3966 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3967 3968 /* setup the next window */ 3969 (*windowp)++; 3970 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3971 (*windowp)->wd_cookie_cnt++; 3972 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3973 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3974 coffset; 3975 (*windowp)->wd_trim.tr_first_size = trim_sz; 3976 } 3977 3978 return (DDI_SUCCESS); 3979 } 3980 3981 3982 /* 3983 * rootnex_copybuf_window_boundary() 3984 * Called in bind slowpath when we get to a window boundary because we used 3985 * up all the copy buffer that we have. 3986 */ 3987 static int 3988 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3989 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3990 size_t *copybuf_used) 3991 { 3992 rootnex_sglinfo_t *sinfo; 3993 off_t new_offset; 3994 size_t trim_sz; 3995 paddr_t paddr; 3996 off_t coffset; 3997 uint_t pidx; 3998 off_t poff; 3999 4000 pidx = 0; 4001 sinfo = &dma->dp_sglinfo; 4002 4003 /* 4004 * the copy buffer should be a whole multiple of page size. We know that 4005 * this cookie is <= MMU_PAGESIZE. 4006 */ 4007 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 4008 4009 /* reset copybuf used */ 4010 *copybuf_used = 0; 4011 4012 /* 4013 * if we don't have to trim (since granularity is set to 1), go to the 4014 * next window and add the current cookie to it. We know the current 4015 * cookie uses the copy buffer since we're in this code path. 4016 */ 4017 if (!dma->dp_trim_required) { 4018 (*windowp)++; 4019 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4020 4021 /* Add this cookie to the new window */ 4022 (*windowp)->wd_cookie_cnt++; 4023 (*windowp)->wd_size += cookie->dmac_size; 4024 *copybuf_used += MMU_PAGESIZE; 4025 return (DDI_SUCCESS); 4026 } 4027 4028 /* 4029 * *** may need to trim, figure it out. 4030 */ 4031 4032 /* figure out how much we need to trim from the window */ 4033 if (dma->dp_granularity_power_2) { 4034 trim_sz = (*windowp)->wd_size & 4035 (hp->dmai_attr.dma_attr_granular - 1); 4036 } else { 4037 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 4038 } 4039 4040 /* 4041 * if the window's a whole multiple of granularity, go to the next 4042 * window, init it, then add in the current cookie. We know the current 4043 * cookie uses the copy buffer since we're in this code path. 4044 */ 4045 if (trim_sz == 0) { 4046 (*windowp)++; 4047 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 4048 4049 /* Add this cookie to the new window */ 4050 (*windowp)->wd_cookie_cnt++; 4051 (*windowp)->wd_size += cookie->dmac_size; 4052 *copybuf_used += MMU_PAGESIZE; 4053 return (DDI_SUCCESS); 4054 } 4055 4056 /* 4057 * *** We figured it out, we definitly need to trim 4058 */ 4059 4060 /* 4061 * make sure the driver isn't making us do something bad... 4062 * Trimming and sgllen == 1 don't go together. 4063 */ 4064 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4065 return (DDI_DMA_NOMAPPING); 4066 } 4067 4068 /* 4069 * first, setup the current window to account for the trim. Need to go 4070 * back to the last cookie for this. Some of the last cookie will be in 4071 * the current window, and some of the last cookie will be in the new 4072 * window. All of the current cookie will be in the new window. 4073 */ 4074 cookie--; 4075 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4076 (*windowp)->wd_trim.tr_last_cookie = cookie; 4077 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4078 ASSERT(cookie->dmac_size > trim_sz); 4079 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4080 (*windowp)->wd_size -= trim_sz; 4081 4082 /* 4083 * we're trimming the last cookie (not the current cookie). So that 4084 * last cookie may have or may not have been using the copy buffer ( 4085 * we know the cookie passed in uses the copy buffer since we're in 4086 * this code path). 4087 * 4088 * If the last cookie doesn't use the copy buffer, nothing special to 4089 * do. However, if it does uses the copy buffer, it will be both the 4090 * last page in the current window and the first page in the next 4091 * window. Since we are reusing the copy buffer (and KVA space on the 4092 * 32-bit kernel), this page will use the end of the copy buffer in the 4093 * current window, and the start of the copy buffer in the next window. 4094 * Track that info... The cookie physical address was already set to 4095 * the copy buffer physical address in setup_cookie.. 4096 */ 4097 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4098 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4099 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4100 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4101 (*windowp)->wd_trim.tr_last_pidx = pidx; 4102 (*windowp)->wd_trim.tr_last_cbaddr = 4103 dma->dp_pgmap[pidx].pm_cbaddr; 4104 } 4105 4106 /* save the buffer offsets for the next window */ 4107 coffset = cookie->dmac_size - trim_sz; 4108 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4109 4110 /* 4111 * set this now in case this is the first window. all other cases are 4112 * set in dma_win() 4113 */ 4114 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4115 4116 /* 4117 * initialize the next window using what's left over in the previous 4118 * cookie. 4119 */ 4120 (*windowp)++; 4121 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4122 (*windowp)->wd_cookie_cnt++; 4123 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4124 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4125 (*windowp)->wd_trim.tr_first_size = trim_sz; 4126 4127 /* 4128 * again, we're tracking if the last cookie uses the copy buffer. 4129 * read the comment above for more info on why we need to track 4130 * additional state. 4131 * 4132 * For the first cookie in the new window, we need reset the physical 4133 * address to DMA into to the start of the copy buffer plus any 4134 * initial page offset which may be present. 4135 */ 4136 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4137 (*windowp)->wd_dosync = B_TRUE; 4138 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4139 (*windowp)->wd_trim.tr_first_pidx = pidx; 4140 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4141 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4142 4143 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4144 poff; 4145 (*windowp)->wd_trim.tr_first_paddr = 4146 ROOTNEX_PADDR_TO_RBASE(paddr); 4147 4148 /* account for the cookie copybuf usage in the new window */ 4149 *copybuf_used += MMU_PAGESIZE; 4150 4151 /* 4152 * every piece of code has to have a hack, and here is this 4153 * ones :-) 4154 * 4155 * There is a complex interaction between setup_cookie and the 4156 * copybuf window boundary. The complexity had to be in either 4157 * the maxxfer window, or the copybuf window, and I chose the 4158 * copybuf code. 4159 * 4160 * So in this code path, we have taken the last cookie, 4161 * virtually broken it in half due to the trim, and it happens 4162 * to use the copybuf which further complicates life. At the 4163 * same time, we have already setup the current cookie, which 4164 * is now wrong. More background info: the current cookie uses 4165 * the copybuf, so it is only a page long max. So we need to 4166 * fix the current cookies copy buffer address, physical 4167 * address, and kva for the 32-bit kernel. We due this by 4168 * bumping them by page size (of course, we can't due this on 4169 * the physical address since the copy buffer may not be 4170 * physically contiguous). 4171 */ 4172 cookie++; 4173 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4174 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4175 4176 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4177 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4178 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(paddr); 4179 4180 } else { 4181 /* go back to the current cookie */ 4182 cookie++; 4183 } 4184 4185 /* 4186 * add the current cookie to the new window. set the new window size to 4187 * the what was left over from the previous cookie and what's in the 4188 * current cookie. 4189 */ 4190 (*windowp)->wd_cookie_cnt++; 4191 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4192 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4193 4194 /* 4195 * we know that the cookie passed in always uses the copy buffer. We 4196 * wouldn't be here if it didn't. 4197 */ 4198 *copybuf_used += MMU_PAGESIZE; 4199 4200 return (DDI_SUCCESS); 4201 } 4202 4203 4204 /* 4205 * rootnex_maxxfer_window_boundary() 4206 * Called in bind slowpath when we get to a window boundary because we will 4207 * go over maxxfer. 4208 */ 4209 static int 4210 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4211 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4212 { 4213 size_t dmac_size; 4214 off_t new_offset; 4215 size_t trim_sz; 4216 off_t coffset; 4217 4218 4219 /* 4220 * calculate how much we have to trim off of the current cookie to equal 4221 * maxxfer. We don't have to account for granularity here since our 4222 * maxxfer already takes that into account. 4223 */ 4224 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4225 ASSERT(trim_sz <= cookie->dmac_size); 4226 ASSERT(trim_sz <= dma->dp_maxxfer); 4227 4228 /* save cookie size since we need it later and we might change it */ 4229 dmac_size = cookie->dmac_size; 4230 4231 /* 4232 * if we're not trimming the entire cookie, setup the current window to 4233 * account for the trim. 4234 */ 4235 if (trim_sz < cookie->dmac_size) { 4236 (*windowp)->wd_cookie_cnt++; 4237 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4238 (*windowp)->wd_trim.tr_last_cookie = cookie; 4239 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4240 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4241 (*windowp)->wd_size = dma->dp_maxxfer; 4242 4243 /* 4244 * set the adjusted cookie size now in case this is the first 4245 * window. All other windows are taken care of in get win 4246 */ 4247 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4248 } 4249 4250 /* 4251 * coffset is the current offset within the cookie, new_offset is the 4252 * current offset with the entire buffer. 4253 */ 4254 coffset = dmac_size - trim_sz; 4255 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4256 4257 /* initialize the next window */ 4258 (*windowp)++; 4259 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4260 (*windowp)->wd_cookie_cnt++; 4261 (*windowp)->wd_size = trim_sz; 4262 if (trim_sz < dmac_size) { 4263 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4264 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4265 coffset; 4266 (*windowp)->wd_trim.tr_first_size = trim_sz; 4267 } 4268 4269 return (DDI_SUCCESS); 4270 } 4271 4272 4273 /*ARGSUSED*/ 4274 static int 4275 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4276 off_t off, size_t len, uint_t cache_flags) 4277 { 4278 rootnex_sglinfo_t *sinfo; 4279 rootnex_pgmap_t *cbpage; 4280 rootnex_window_t *win; 4281 ddi_dma_impl_t *hp; 4282 rootnex_dma_t *dma; 4283 caddr_t fromaddr; 4284 caddr_t toaddr; 4285 uint_t psize; 4286 off_t offset; 4287 uint_t pidx; 4288 size_t size; 4289 off_t poff; 4290 int e; 4291 4292 4293 hp = (ddi_dma_impl_t *)handle; 4294 dma = (rootnex_dma_t *)hp->dmai_private; 4295 sinfo = &dma->dp_sglinfo; 4296 4297 /* 4298 * if we don't have any windows, we don't need to sync. A copybuf 4299 * will cause us to have at least one window. 4300 */ 4301 if (dma->dp_window == NULL) { 4302 return (DDI_SUCCESS); 4303 } 4304 4305 /* This window may not need to be sync'd */ 4306 win = &dma->dp_window[dma->dp_current_win]; 4307 if (!win->wd_dosync) { 4308 return (DDI_SUCCESS); 4309 } 4310 4311 /* handle off and len special cases */ 4312 if ((off == 0) || (rootnex_sync_ignore_params)) { 4313 offset = win->wd_offset; 4314 } else { 4315 offset = off; 4316 } 4317 if ((len == 0) || (rootnex_sync_ignore_params)) { 4318 size = win->wd_size; 4319 } else { 4320 size = len; 4321 } 4322 4323 /* check the sync args to make sure they make a little sense */ 4324 if (rootnex_sync_check_parms) { 4325 e = rootnex_valid_sync_parms(hp, win, offset, size, 4326 cache_flags); 4327 if (e != DDI_SUCCESS) { 4328 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4329 return (DDI_FAILURE); 4330 } 4331 } 4332 4333 /* 4334 * special case the first page to handle the offset into the page. The 4335 * offset to the current page for our buffer is the offset into the 4336 * first page of the buffer plus our current offset into the buffer 4337 * itself, masked of course. 4338 */ 4339 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4340 psize = MIN((MMU_PAGESIZE - poff), size); 4341 4342 /* go through all the pages that we want to sync */ 4343 while (size > 0) { 4344 /* 4345 * Calculate the page index relative to the start of the buffer. 4346 * The index to the current page for our buffer is the offset 4347 * into the first page of the buffer plus our current offset 4348 * into the buffer itself, shifted of course... 4349 */ 4350 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4351 ASSERT(pidx < sinfo->si_max_pages); 4352 4353 /* 4354 * if this page uses the copy buffer, we need to sync it, 4355 * otherwise, go on to the next page. 4356 */ 4357 cbpage = &dma->dp_pgmap[pidx]; 4358 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4359 (cbpage->pm_uses_copybuf == B_FALSE)); 4360 if (cbpage->pm_uses_copybuf) { 4361 /* cbaddr and kaddr should be page aligned */ 4362 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4363 MMU_PAGEOFFSET) == 0); 4364 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4365 MMU_PAGEOFFSET) == 0); 4366 4367 /* 4368 * if we're copying for the device, we are going to 4369 * copy from the drivers buffer and to the rootnex 4370 * allocated copy buffer. 4371 */ 4372 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4373 fromaddr = cbpage->pm_kaddr + poff; 4374 toaddr = cbpage->pm_cbaddr + poff; 4375 ROOTNEX_DPROBE2(rootnex__sync__dev, 4376 dev_info_t *, dma->dp_dip, size_t, psize); 4377 4378 /* 4379 * if we're copying for the cpu/kernel, we are going to 4380 * copy from the rootnex allocated copy buffer to the 4381 * drivers buffer. 4382 */ 4383 } else { 4384 fromaddr = cbpage->pm_cbaddr + poff; 4385 toaddr = cbpage->pm_kaddr + poff; 4386 ROOTNEX_DPROBE2(rootnex__sync__cpu, 4387 dev_info_t *, dma->dp_dip, size_t, psize); 4388 } 4389 4390 bcopy(fromaddr, toaddr, psize); 4391 } 4392 4393 /* 4394 * decrement size until we're done, update our offset into the 4395 * buffer, and get the next page size. 4396 */ 4397 size -= psize; 4398 offset += psize; 4399 psize = MIN(MMU_PAGESIZE, size); 4400 4401 /* page offset is zero for the rest of this loop */ 4402 poff = 0; 4403 } 4404 4405 return (DDI_SUCCESS); 4406 } 4407 4408 /* 4409 * rootnex_dma_sync() 4410 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4411 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4412 * is set, ddi_dma_sync() returns immediately passing back success. 4413 */ 4414 /*ARGSUSED*/ 4415 static int 4416 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4417 off_t off, size_t len, uint_t cache_flags) 4418 { 4419 #if !defined(__xpv) 4420 if (IOMMU_USED(rdip)) { 4421 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4422 cache_flags)); 4423 } 4424 #endif 4425 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4426 cache_flags)); 4427 } 4428 4429 /* 4430 * rootnex_valid_sync_parms() 4431 * checks the parameters passed to sync to verify they are correct. 4432 */ 4433 static int 4434 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4435 off_t offset, size_t size, uint_t cache_flags) 4436 { 4437 off_t woffset; 4438 4439 4440 /* 4441 * the first part of the test to make sure the offset passed in is 4442 * within the window. 4443 */ 4444 if (offset < win->wd_offset) { 4445 return (DDI_FAILURE); 4446 } 4447 4448 /* 4449 * second and last part of the test to make sure the offset and length 4450 * passed in is within the window. 4451 */ 4452 woffset = offset - win->wd_offset; 4453 if ((woffset + size) > win->wd_size) { 4454 return (DDI_FAILURE); 4455 } 4456 4457 /* 4458 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4459 * be set too. 4460 */ 4461 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4462 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4463 return (DDI_SUCCESS); 4464 } 4465 4466 /* 4467 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4468 * should be set. Also DDI_DMA_READ should be set in the flags. 4469 */ 4470 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4471 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4472 (hp->dmai_rflags & DDI_DMA_READ)) { 4473 return (DDI_SUCCESS); 4474 } 4475 4476 return (DDI_FAILURE); 4477 } 4478 4479 4480 /*ARGSUSED*/ 4481 static int 4482 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4483 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4484 uint_t *ccountp) 4485 { 4486 rootnex_window_t *window; 4487 rootnex_trim_t *trim; 4488 ddi_dma_impl_t *hp; 4489 rootnex_dma_t *dma; 4490 ddi_dma_obj_t *dmao; 4491 4492 4493 hp = (ddi_dma_impl_t *)handle; 4494 dma = (rootnex_dma_t *)hp->dmai_private; 4495 4496 /* If we try and get a window which doesn't exist, return failure */ 4497 if (win >= hp->dmai_nwin) { 4498 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4499 return (DDI_FAILURE); 4500 } 4501 4502 dmao = dma->dp_dvma_used ? &dma->dp_dvma : &dma->dp_dma; 4503 4504 /* 4505 * if we don't have any windows, and they're asking for the first 4506 * window, setup the cookie pointer to the first cookie in the bind. 4507 * setup our return values, then increment the cookie since we return 4508 * the first cookie on the stack. 4509 */ 4510 if (dma->dp_window == NULL) { 4511 if (win != 0) { 4512 ROOTNEX_DPROF_INC( 4513 &rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4514 return (DDI_FAILURE); 4515 } 4516 hp->dmai_cookie = dma->dp_cookies; 4517 *offp = 0; 4518 *lenp = dmao->dmao_size; 4519 *ccountp = dma->dp_sglinfo.si_sgl_size; 4520 *cookiep = hp->dmai_cookie[0]; 4521 hp->dmai_cookie++; 4522 hp->dmai_ncookies = *ccountp; 4523 hp->dmai_curcookie = 1; 4524 return (DDI_SUCCESS); 4525 } 4526 4527 /* sync the old window before moving on to the new one */ 4528 window = &dma->dp_window[dma->dp_current_win]; 4529 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4530 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4531 DDI_DMA_SYNC_FORCPU); 4532 } 4533 4534 4535 /* 4536 * Move to the new window. 4537 * NOTE: current_win must be set for sync to work right 4538 */ 4539 dma->dp_current_win = win; 4540 window = &dma->dp_window[win]; 4541 4542 /* if needed, adjust the first and/or last cookies for trim */ 4543 trim = &window->wd_trim; 4544 if (trim->tr_trim_first) { 4545 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4546 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4547 if (trim->tr_first_copybuf_win) { 4548 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4549 trim->tr_first_cbaddr; 4550 } 4551 } 4552 if (trim->tr_trim_last) { 4553 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4554 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4555 if (trim->tr_last_copybuf_win) { 4556 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4557 trim->tr_last_cbaddr; 4558 } 4559 } 4560 4561 /* 4562 * setup the cookie pointer to the first cookie in the window. setup 4563 * our return values, then increment the cookie since we return the 4564 * first cookie on the stack. 4565 */ 4566 hp->dmai_cookie = window->wd_first_cookie; 4567 *offp = window->wd_offset; 4568 *lenp = window->wd_size; 4569 *ccountp = window->wd_cookie_cnt; 4570 *cookiep = hp->dmai_cookie[0]; 4571 hp->dmai_ncookies = *ccountp; 4572 hp->dmai_curcookie = 1; 4573 hp->dmai_cookie++; 4574 4575 4576 /* if the new window uses the copy buffer, sync it for the device */ 4577 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4578 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4579 DDI_DMA_SYNC_FORDEV); 4580 } 4581 4582 return (DDI_SUCCESS); 4583 } 4584 4585 /* 4586 * rootnex_dma_win() 4587 * called from ddi_dma_getwin() 4588 */ 4589 /*ARGSUSED*/ 4590 static int 4591 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4592 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4593 uint_t *ccountp) 4594 { 4595 #if !defined(__xpv) 4596 if (IOMMU_USED(rdip)) { 4597 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4598 cookiep, ccountp)); 4599 } 4600 #endif 4601 4602 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4603 cookiep, ccountp)); 4604 } 4605 4606 #if !defined(__xpv) 4607 /*ARGSUSED*/ 4608 static int 4609 rootnex_coredma_hdl_setprivate(dev_info_t *dip, dev_info_t *rdip, 4610 ddi_dma_handle_t handle, void *v) 4611 { 4612 ddi_dma_impl_t *hp; 4613 rootnex_dma_t *dma; 4614 4615 hp = (ddi_dma_impl_t *)handle; 4616 dma = (rootnex_dma_t *)hp->dmai_private; 4617 dma->dp_iommu_private = v; 4618 4619 return (DDI_SUCCESS); 4620 } 4621 4622 /*ARGSUSED*/ 4623 static void * 4624 rootnex_coredma_hdl_getprivate(dev_info_t *dip, dev_info_t *rdip, 4625 ddi_dma_handle_t handle) 4626 { 4627 ddi_dma_impl_t *hp; 4628 rootnex_dma_t *dma; 4629 4630 hp = (ddi_dma_impl_t *)handle; 4631 dma = (rootnex_dma_t *)hp->dmai_private; 4632 4633 return (dma->dp_iommu_private); 4634 } 4635 #endif 4636 4637 /* 4638 * ************************ 4639 * obsoleted dma routines 4640 * ************************ 4641 */ 4642 4643 /* 4644 * rootnex_dma_mctl() 4645 * 4646 * We don't support this legacy interface any more on x86. 4647 */ 4648 /* ARGSUSED */ 4649 static int 4650 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4651 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4652 uint_t cache_flags) 4653 { 4654 /* 4655 * The only thing dma_mctl is usef for anymore is legacy SPARC 4656 * dvma and sbus-specific routines. 4657 */ 4658 return (DDI_FAILURE); 4659 } 4660 4661 /* 4662 * ********* 4663 * FMA Code 4664 * ********* 4665 */ 4666 4667 /* 4668 * rootnex_fm_init() 4669 * FMA init busop 4670 */ 4671 /* ARGSUSED */ 4672 static int 4673 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4674 ddi_iblock_cookie_t *ibc) 4675 { 4676 *ibc = rootnex_state->r_err_ibc; 4677 4678 return (ddi_system_fmcap); 4679 } 4680 4681 /* 4682 * rootnex_dma_check() 4683 * Function called after a dma fault occurred to find out whether the 4684 * fault address is associated with a driver that is able to handle faults 4685 * and recover from faults. 4686 */ 4687 /* ARGSUSED */ 4688 static int 4689 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4690 const void *not_used) 4691 { 4692 rootnex_window_t *window; 4693 uint64_t start_addr; 4694 uint64_t fault_addr; 4695 ddi_dma_impl_t *hp; 4696 rootnex_dma_t *dma; 4697 uint64_t end_addr; 4698 size_t csize; 4699 int i; 4700 int j; 4701 4702 4703 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4704 hp = (ddi_dma_impl_t *)handle; 4705 ASSERT(hp); 4706 4707 dma = (rootnex_dma_t *)hp->dmai_private; 4708 4709 /* Get the address that we need to search for */ 4710 fault_addr = *(uint64_t *)addr; 4711 4712 /* 4713 * if we don't have any windows, we can just walk through all the 4714 * cookies. 4715 */ 4716 if (dma->dp_window == NULL) { 4717 /* for each cookie */ 4718 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4719 /* 4720 * if the faulted address is within the physical address 4721 * range of the cookie, return DDI_FM_NONFATAL. 4722 */ 4723 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4724 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4725 dma->dp_cookies[i].dmac_size))) { 4726 return (DDI_FM_NONFATAL); 4727 } 4728 } 4729 4730 /* fault_addr not within this DMA handle */ 4731 return (DDI_FM_UNKNOWN); 4732 } 4733 4734 /* we have mutiple windows, walk through each window */ 4735 for (i = 0; i < hp->dmai_nwin; i++) { 4736 window = &dma->dp_window[i]; 4737 4738 /* Go through all the cookies in the window */ 4739 for (j = 0; j < window->wd_cookie_cnt; j++) { 4740 4741 start_addr = window->wd_first_cookie[j].dmac_laddress; 4742 csize = window->wd_first_cookie[j].dmac_size; 4743 4744 /* 4745 * if we are trimming the first cookie in the window, 4746 * and this is the first cookie, adjust the start 4747 * address and size of the cookie to account for the 4748 * trim. 4749 */ 4750 if (window->wd_trim.tr_trim_first && (j == 0)) { 4751 start_addr = window->wd_trim.tr_first_paddr; 4752 csize = window->wd_trim.tr_first_size; 4753 } 4754 4755 /* 4756 * if we are trimming the last cookie in the window, 4757 * and this is the last cookie, adjust the start 4758 * address and size of the cookie to account for the 4759 * trim. 4760 */ 4761 if (window->wd_trim.tr_trim_last && 4762 (j == (window->wd_cookie_cnt - 1))) { 4763 start_addr = window->wd_trim.tr_last_paddr; 4764 csize = window->wd_trim.tr_last_size; 4765 } 4766 4767 end_addr = start_addr + csize; 4768 4769 /* 4770 * if the faulted address is within the physical 4771 * address of the cookie, return DDI_FM_NONFATAL. 4772 */ 4773 if ((fault_addr >= start_addr) && 4774 (fault_addr <= end_addr)) { 4775 return (DDI_FM_NONFATAL); 4776 } 4777 } 4778 } 4779 4780 /* fault_addr not within this DMA handle */ 4781 return (DDI_FM_UNKNOWN); 4782 } 4783 4784 /*ARGSUSED*/ 4785 static int 4786 rootnex_quiesce(dev_info_t *dip) 4787 { 4788 #if !defined(__xpv) 4789 return (immu_quiesce()); 4790 #else 4791 return (DDI_SUCCESS); 4792 #endif 4793 } 4794 4795 #if defined(__xpv) 4796 void 4797 immu_init(void) 4798 { 4799 ; 4800 } 4801 4802 void 4803 immu_startup(void) 4804 { 4805 ; 4806 } 4807 /*ARGSUSED*/ 4808 void 4809 immu_physmem_update(uint64_t addr, uint64_t size) 4810 { 4811 ; 4812 } 4813 #endif 4814