1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* 26 * x86 root nexus driver 27 */ 28 29 #include <sys/sysmacros.h> 30 #include <sys/conf.h> 31 #include <sys/autoconf.h> 32 #include <sys/sysmacros.h> 33 #include <sys/debug.h> 34 #include <sys/psw.h> 35 #include <sys/ddidmareq.h> 36 #include <sys/promif.h> 37 #include <sys/devops.h> 38 #include <sys/kmem.h> 39 #include <sys/cmn_err.h> 40 #include <vm/seg.h> 41 #include <vm/seg_kmem.h> 42 #include <vm/seg_dev.h> 43 #include <sys/vmem.h> 44 #include <sys/mman.h> 45 #include <vm/hat.h> 46 #include <vm/as.h> 47 #include <vm/page.h> 48 #include <sys/avintr.h> 49 #include <sys/errno.h> 50 #include <sys/modctl.h> 51 #include <sys/ddi_impldefs.h> 52 #include <sys/sunddi.h> 53 #include <sys/sunndi.h> 54 #include <sys/mach_intr.h> 55 #include <sys/psm.h> 56 #include <sys/ontrap.h> 57 #include <sys/atomic.h> 58 #include <sys/sdt.h> 59 #include <sys/rootnex.h> 60 #include <vm/hat_i86.h> 61 #include <sys/ddifm.h> 62 #include <sys/ddi_isa.h> 63 64 #ifdef __xpv 65 #include <sys/bootinfo.h> 66 #include <sys/hypervisor.h> 67 #include <sys/bootconf.h> 68 #include <vm/kboot_mmu.h> 69 #endif 70 71 #if defined(__amd64) && !defined(__xpv) 72 #include <sys/immu.h> 73 #endif 74 75 76 /* 77 * enable/disable extra checking of function parameters. Useful for debugging 78 * drivers. 79 */ 80 #ifdef DEBUG 81 int rootnex_alloc_check_parms = 1; 82 int rootnex_bind_check_parms = 1; 83 int rootnex_bind_check_inuse = 1; 84 int rootnex_unbind_verify_buffer = 0; 85 int rootnex_sync_check_parms = 1; 86 #else 87 int rootnex_alloc_check_parms = 0; 88 int rootnex_bind_check_parms = 0; 89 int rootnex_bind_check_inuse = 0; 90 int rootnex_unbind_verify_buffer = 0; 91 int rootnex_sync_check_parms = 0; 92 #endif 93 94 boolean_t rootnex_dmar_not_setup; 95 96 /* Master Abort and Target Abort panic flag */ 97 int rootnex_fm_ma_ta_panic_flag = 0; 98 99 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 100 int rootnex_bind_fail = 1; 101 int rootnex_bind_warn = 1; 102 uint8_t *rootnex_warn_list; 103 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 104 #define ROOTNEX_BIND_WARNING (0x1 << 0) 105 106 /* 107 * revert back to old broken behavior of always sync'ing entire copy buffer. 108 * This is useful if be have a buggy driver which doesn't correctly pass in 109 * the offset and size into ddi_dma_sync(). 110 */ 111 int rootnex_sync_ignore_params = 0; 112 113 /* 114 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 115 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 116 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 117 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 118 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 119 * (< 8K). We will still need to allocate the copy buffer during bind though 120 * (if we need one). These can only be modified in /etc/system before rootnex 121 * attach. 122 */ 123 #if defined(__amd64) 124 int rootnex_prealloc_cookies = 65; 125 int rootnex_prealloc_windows = 4; 126 int rootnex_prealloc_copybuf = 2; 127 #else 128 int rootnex_prealloc_cookies = 33; 129 int rootnex_prealloc_windows = 4; 130 int rootnex_prealloc_copybuf = 2; 131 #endif 132 133 /* driver global state */ 134 static rootnex_state_t *rootnex_state; 135 136 /* shortcut to rootnex counters */ 137 static uint64_t *rootnex_cnt; 138 139 /* 140 * XXX - does x86 even need these or are they left over from the SPARC days? 141 */ 142 /* statically defined integer/boolean properties for the root node */ 143 static rootnex_intprop_t rootnex_intprp[] = { 144 { "PAGESIZE", PAGESIZE }, 145 { "MMU_PAGESIZE", MMU_PAGESIZE }, 146 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 147 { DDI_RELATIVE_ADDRESSING, 1 }, 148 }; 149 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 150 151 #ifdef __xpv 152 typedef maddr_t rootnex_addr_t; 153 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 154 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 155 #else 156 typedef paddr_t rootnex_addr_t; 157 #endif 158 159 #if !defined(__xpv) 160 char _depends_on[] = "mach/pcplusmp misc/iommulib misc/acpica"; 161 #endif 162 163 static struct cb_ops rootnex_cb_ops = { 164 nodev, /* open */ 165 nodev, /* close */ 166 nodev, /* strategy */ 167 nodev, /* print */ 168 nodev, /* dump */ 169 nodev, /* read */ 170 nodev, /* write */ 171 nodev, /* ioctl */ 172 nodev, /* devmap */ 173 nodev, /* mmap */ 174 nodev, /* segmap */ 175 nochpoll, /* chpoll */ 176 ddi_prop_op, /* cb_prop_op */ 177 NULL, /* struct streamtab */ 178 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 179 CB_REV, /* Rev */ 180 nodev, /* cb_aread */ 181 nodev /* cb_awrite */ 182 }; 183 184 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 185 off_t offset, off_t len, caddr_t *vaddrp); 186 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 187 struct hat *hat, struct seg *seg, caddr_t addr, 188 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 189 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 190 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 191 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 192 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 193 ddi_dma_handle_t *handlep); 194 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 195 ddi_dma_handle_t handle); 196 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 197 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 198 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 199 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 200 ddi_dma_handle_t handle); 201 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 202 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 203 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 205 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 206 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 207 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 208 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 209 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 210 ddi_ctl_enum_t ctlop, void *arg, void *result); 211 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 212 ddi_iblock_cookie_t *ibc); 213 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 214 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 215 216 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 217 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 218 ddi_dma_handle_t *handlep); 219 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 220 ddi_dma_handle_t handle); 221 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 223 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 224 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 225 ddi_dma_handle_t handle); 226 #if defined(__amd64) && !defined(__xpv) 227 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 228 ddi_dma_handle_t handle); 229 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 230 ddi_dma_cookie_t **cookiepp, uint_t *ccountp); 231 static int rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 232 ddi_dma_cookie_t *cookiep, uint_t ccount); 233 static int rootnex_coredma_clear_cookies(dev_info_t *dip, 234 ddi_dma_handle_t handle); 235 static int rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle); 236 #endif 237 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 238 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 239 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 240 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 241 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 242 243 static struct bus_ops rootnex_bus_ops = { 244 BUSO_REV, 245 rootnex_map, 246 NULL, 247 NULL, 248 NULL, 249 rootnex_map_fault, 250 rootnex_dma_map, 251 rootnex_dma_allochdl, 252 rootnex_dma_freehdl, 253 rootnex_dma_bindhdl, 254 rootnex_dma_unbindhdl, 255 rootnex_dma_sync, 256 rootnex_dma_win, 257 rootnex_dma_mctl, 258 rootnex_ctlops, 259 ddi_bus_prop_op, 260 i_ddi_rootnex_get_eventcookie, 261 i_ddi_rootnex_add_eventcall, 262 i_ddi_rootnex_remove_eventcall, 263 i_ddi_rootnex_post_event, 264 0, /* bus_intr_ctl */ 265 0, /* bus_config */ 266 0, /* bus_unconfig */ 267 rootnex_fm_init, /* bus_fm_init */ 268 NULL, /* bus_fm_fini */ 269 NULL, /* bus_fm_access_enter */ 270 NULL, /* bus_fm_access_exit */ 271 NULL, /* bus_powr */ 272 rootnex_intr_ops /* bus_intr_op */ 273 }; 274 275 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 276 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 277 static int rootnex_quiesce(dev_info_t *dip); 278 279 static struct dev_ops rootnex_ops = { 280 DEVO_REV, 281 0, 282 ddi_no_info, 283 nulldev, 284 nulldev, 285 rootnex_attach, 286 rootnex_detach, 287 nulldev, 288 &rootnex_cb_ops, 289 &rootnex_bus_ops, 290 NULL, 291 rootnex_quiesce, /* quiesce */ 292 }; 293 294 static struct modldrv rootnex_modldrv = { 295 &mod_driverops, 296 "i86pc root nexus", 297 &rootnex_ops 298 }; 299 300 static struct modlinkage rootnex_modlinkage = { 301 MODREV_1, 302 (void *)&rootnex_modldrv, 303 NULL 304 }; 305 306 #if defined(__amd64) && !defined(__xpv) 307 static iommulib_nexops_t iommulib_nexops = { 308 IOMMU_NEXOPS_VERSION, 309 "Rootnex IOMMU ops Vers 1.1", 310 NULL, 311 rootnex_coredma_allochdl, 312 rootnex_coredma_freehdl, 313 rootnex_coredma_bindhdl, 314 rootnex_coredma_unbindhdl, 315 rootnex_coredma_reset_cookies, 316 rootnex_coredma_get_cookies, 317 rootnex_coredma_set_cookies, 318 rootnex_coredma_clear_cookies, 319 rootnex_coredma_get_sleep_flags, 320 rootnex_coredma_sync, 321 rootnex_coredma_win, 322 rootnex_dma_map, 323 rootnex_dma_mctl 324 }; 325 #endif 326 327 /* 328 * extern hacks 329 */ 330 extern struct seg_ops segdev_ops; 331 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 332 #ifdef DDI_MAP_DEBUG 333 extern int ddi_map_debug_flag; 334 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 335 #endif 336 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 337 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 338 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 339 psm_intr_op_t, int *); 340 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 341 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 342 343 /* 344 * Use device arena to use for device control register mappings. 345 * Various kernel memory walkers (debugger, dtrace) need to know 346 * to avoid this address range to prevent undesired device activity. 347 */ 348 extern void *device_arena_alloc(size_t size, int vm_flag); 349 extern void device_arena_free(void * vaddr, size_t size); 350 351 352 /* 353 * Internal functions 354 */ 355 static int rootnex_dma_init(); 356 static void rootnex_add_props(dev_info_t *); 357 static int rootnex_ctl_reportdev(dev_info_t *dip); 358 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 359 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 360 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 361 static int rootnex_map_handle(ddi_map_req_t *mp); 362 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 363 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 364 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 365 ddi_dma_attr_t *attr); 366 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 367 rootnex_sglinfo_t *sglinfo); 368 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 369 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 370 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 371 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 372 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 373 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 374 ddi_dma_attr_t *attr, int kmflag); 375 static void rootnex_teardown_windows(rootnex_dma_t *dma); 376 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 377 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 378 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 379 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 380 size_t *copybuf_used, page_t **cur_pp); 381 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 382 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 383 ddi_dma_attr_t *attr, off_t cur_offset); 384 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 385 rootnex_dma_t *dma, rootnex_window_t **windowp, 386 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 387 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 388 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 389 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 390 off_t offset, size_t size, uint_t cache_flags); 391 static int rootnex_verify_buffer(rootnex_dma_t *dma); 392 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 393 const void *comp_addr, const void *not_used); 394 static boolean_t rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, 395 rootnex_sglinfo_t *sglinfo); 396 397 /* 398 * _init() 399 * 400 */ 401 int 402 _init(void) 403 { 404 405 rootnex_state = NULL; 406 return (mod_install(&rootnex_modlinkage)); 407 } 408 409 410 /* 411 * _info() 412 * 413 */ 414 int 415 _info(struct modinfo *modinfop) 416 { 417 return (mod_info(&rootnex_modlinkage, modinfop)); 418 } 419 420 421 /* 422 * _fini() 423 * 424 */ 425 int 426 _fini(void) 427 { 428 return (EBUSY); 429 } 430 431 432 /* 433 * rootnex_attach() 434 * 435 */ 436 static int 437 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 438 { 439 int fmcap; 440 int e; 441 442 switch (cmd) { 443 case DDI_ATTACH: 444 break; 445 case DDI_RESUME: 446 #if defined(__amd64) && !defined(__xpv) 447 return (immu_unquiesce()); 448 #else 449 return (DDI_SUCCESS); 450 #endif 451 default: 452 return (DDI_FAILURE); 453 } 454 455 /* 456 * We should only have one instance of rootnex. Save it away since we 457 * don't have an easy way to get it back later. 458 */ 459 ASSERT(rootnex_state == NULL); 460 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 461 462 rootnex_state->r_dip = dip; 463 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 464 rootnex_state->r_reserved_msg_printed = B_FALSE; 465 rootnex_cnt = &rootnex_state->r_counters[0]; 466 467 /* 468 * Set minimum fm capability level for i86pc platforms and then 469 * initialize error handling. Since we're the rootnex, we don't 470 * care what's returned in the fmcap field. 471 */ 472 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 473 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 474 fmcap = ddi_system_fmcap; 475 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 476 477 /* initialize DMA related state */ 478 e = rootnex_dma_init(); 479 if (e != DDI_SUCCESS) { 480 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 481 return (DDI_FAILURE); 482 } 483 484 /* Add static root node properties */ 485 rootnex_add_props(dip); 486 487 /* since we can't call ddi_report_dev() */ 488 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 489 490 /* Initialize rootnex event handle */ 491 i_ddi_rootnex_init_events(dip); 492 493 #if defined(__amd64) && !defined(__xpv) 494 e = iommulib_nexus_register(dip, &iommulib_nexops, 495 &rootnex_state->r_iommulib_handle); 496 497 ASSERT(e == DDI_SUCCESS); 498 #endif 499 500 return (DDI_SUCCESS); 501 } 502 503 504 /* 505 * rootnex_detach() 506 * 507 */ 508 /*ARGSUSED*/ 509 static int 510 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 511 { 512 switch (cmd) { 513 case DDI_SUSPEND: 514 #if defined(__amd64) && !defined(__xpv) 515 return (immu_quiesce()); 516 #else 517 return (DDI_SUCCESS); 518 #endif 519 default: 520 return (DDI_FAILURE); 521 } 522 /*NOTREACHED*/ 523 524 } 525 526 527 /* 528 * rootnex_dma_init() 529 * 530 */ 531 /*ARGSUSED*/ 532 static int 533 rootnex_dma_init() 534 { 535 size_t bufsize; 536 537 538 /* 539 * size of our cookie/window/copybuf state needed in dma bind that we 540 * pre-alloc in dma_alloc_handle 541 */ 542 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 543 rootnex_state->r_prealloc_size = 544 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 545 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 546 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 547 548 /* 549 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 550 * allocate 16 extra bytes for struct pointer alignment 551 * (p->dmai_private & dma->dp_prealloc_buffer) 552 */ 553 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 554 rootnex_state->r_prealloc_size + 0x10; 555 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 556 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 557 if (rootnex_state->r_dmahdl_cache == NULL) { 558 return (DDI_FAILURE); 559 } 560 561 /* 562 * allocate array to track which major numbers we have printed warnings 563 * for. 564 */ 565 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 566 KM_SLEEP); 567 568 return (DDI_SUCCESS); 569 } 570 571 572 /* 573 * rootnex_add_props() 574 * 575 */ 576 static void 577 rootnex_add_props(dev_info_t *dip) 578 { 579 rootnex_intprop_t *rpp; 580 int i; 581 582 /* Add static integer/boolean properties to the root node */ 583 rpp = rootnex_intprp; 584 for (i = 0; i < NROOT_INTPROPS; i++) { 585 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 586 rpp[i].prop_name, rpp[i].prop_value); 587 } 588 } 589 590 591 592 /* 593 * ************************* 594 * ctlops related routines 595 * ************************* 596 */ 597 598 /* 599 * rootnex_ctlops() 600 * 601 */ 602 /*ARGSUSED*/ 603 static int 604 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 605 void *arg, void *result) 606 { 607 int n, *ptr; 608 struct ddi_parent_private_data *pdp; 609 610 switch (ctlop) { 611 case DDI_CTLOPS_DMAPMAPC: 612 /* 613 * Return 'partial' to indicate that dma mapping 614 * has to be done in the main MMU. 615 */ 616 return (DDI_DMA_PARTIAL); 617 618 case DDI_CTLOPS_BTOP: 619 /* 620 * Convert byte count input to physical page units. 621 * (byte counts that are not a page-size multiple 622 * are rounded down) 623 */ 624 *(ulong_t *)result = btop(*(ulong_t *)arg); 625 return (DDI_SUCCESS); 626 627 case DDI_CTLOPS_PTOB: 628 /* 629 * Convert size in physical pages to bytes 630 */ 631 *(ulong_t *)result = ptob(*(ulong_t *)arg); 632 return (DDI_SUCCESS); 633 634 case DDI_CTLOPS_BTOPR: 635 /* 636 * Convert byte count input to physical page units 637 * (byte counts that are not a page-size multiple 638 * are rounded up) 639 */ 640 *(ulong_t *)result = btopr(*(ulong_t *)arg); 641 return (DDI_SUCCESS); 642 643 case DDI_CTLOPS_INITCHILD: 644 return (impl_ddi_sunbus_initchild(arg)); 645 646 case DDI_CTLOPS_UNINITCHILD: 647 impl_ddi_sunbus_removechild(arg); 648 return (DDI_SUCCESS); 649 650 case DDI_CTLOPS_REPORTDEV: 651 return (rootnex_ctl_reportdev(rdip)); 652 653 case DDI_CTLOPS_IOMIN: 654 /* 655 * Nothing to do here but reflect back.. 656 */ 657 return (DDI_SUCCESS); 658 659 case DDI_CTLOPS_REGSIZE: 660 case DDI_CTLOPS_NREGS: 661 break; 662 663 case DDI_CTLOPS_SIDDEV: 664 if (ndi_dev_is_prom_node(rdip)) 665 return (DDI_SUCCESS); 666 if (ndi_dev_is_persistent_node(rdip)) 667 return (DDI_SUCCESS); 668 return (DDI_FAILURE); 669 670 case DDI_CTLOPS_POWER: 671 return ((*pm_platform_power)((power_req_t *)arg)); 672 673 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 674 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 675 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 676 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 677 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 678 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 679 if (!rootnex_state->r_reserved_msg_printed) { 680 rootnex_state->r_reserved_msg_printed = B_TRUE; 681 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 682 "1 or more reserved/obsolete operations."); 683 } 684 return (DDI_FAILURE); 685 686 default: 687 return (DDI_FAILURE); 688 } 689 /* 690 * The rest are for "hardware" properties 691 */ 692 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 693 return (DDI_FAILURE); 694 695 if (ctlop == DDI_CTLOPS_NREGS) { 696 ptr = (int *)result; 697 *ptr = pdp->par_nreg; 698 } else { 699 off_t *size = (off_t *)result; 700 701 ptr = (int *)arg; 702 n = *ptr; 703 if (n >= pdp->par_nreg) { 704 return (DDI_FAILURE); 705 } 706 *size = (off_t)pdp->par_reg[n].regspec_size; 707 } 708 return (DDI_SUCCESS); 709 } 710 711 712 /* 713 * rootnex_ctl_reportdev() 714 * 715 */ 716 static int 717 rootnex_ctl_reportdev(dev_info_t *dev) 718 { 719 int i, n, len, f_len = 0; 720 char *buf; 721 722 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 723 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 724 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 725 len = strlen(buf); 726 727 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 728 729 struct regspec *rp = sparc_pd_getreg(dev, i); 730 731 if (i == 0) 732 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 733 ": "); 734 else 735 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 736 " and "); 737 len = strlen(buf); 738 739 switch (rp->regspec_bustype) { 740 741 case BTEISA: 742 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 743 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 744 break; 745 746 case BTISA: 747 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 748 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 749 break; 750 751 default: 752 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 753 "space %x offset %x", 754 rp->regspec_bustype, rp->regspec_addr); 755 break; 756 } 757 len = strlen(buf); 758 } 759 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 760 int pri; 761 762 if (i != 0) { 763 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 764 ","); 765 len = strlen(buf); 766 } 767 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 768 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 769 " sparc ipl %d", pri); 770 len = strlen(buf); 771 } 772 #ifdef DEBUG 773 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 774 cmn_err(CE_NOTE, "next message is truncated: " 775 "printed length 1024, real length %d", f_len); 776 } 777 #endif /* DEBUG */ 778 cmn_err(CE_CONT, "?%s\n", buf); 779 kmem_free(buf, REPORTDEV_BUFSIZE); 780 return (DDI_SUCCESS); 781 } 782 783 784 /* 785 * ****************** 786 * map related code 787 * ****************** 788 */ 789 790 /* 791 * rootnex_map() 792 * 793 */ 794 static int 795 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 796 off_t len, caddr_t *vaddrp) 797 { 798 struct regspec *rp, tmp_reg; 799 ddi_map_req_t mr = *mp; /* Get private copy of request */ 800 int error; 801 802 mp = &mr; 803 804 switch (mp->map_op) { 805 case DDI_MO_MAP_LOCKED: 806 case DDI_MO_UNMAP: 807 case DDI_MO_MAP_HANDLE: 808 break; 809 default: 810 #ifdef DDI_MAP_DEBUG 811 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 812 mp->map_op); 813 #endif /* DDI_MAP_DEBUG */ 814 return (DDI_ME_UNIMPLEMENTED); 815 } 816 817 if (mp->map_flags & DDI_MF_USER_MAPPING) { 818 #ifdef DDI_MAP_DEBUG 819 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 820 #endif /* DDI_MAP_DEBUG */ 821 return (DDI_ME_UNIMPLEMENTED); 822 } 823 824 /* 825 * First, if given an rnumber, convert it to a regspec... 826 * (Presumably, this is on behalf of a child of the root node?) 827 */ 828 829 if (mp->map_type == DDI_MT_RNUMBER) { 830 831 int rnumber = mp->map_obj.rnumber; 832 #ifdef DDI_MAP_DEBUG 833 static char *out_of_range = 834 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 835 #endif /* DDI_MAP_DEBUG */ 836 837 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 838 if (rp == NULL) { 839 #ifdef DDI_MAP_DEBUG 840 cmn_err(CE_WARN, out_of_range, rnumber, 841 ddi_get_name(rdip)); 842 #endif /* DDI_MAP_DEBUG */ 843 return (DDI_ME_RNUMBER_RANGE); 844 } 845 846 /* 847 * Convert the given ddi_map_req_t from rnumber to regspec... 848 */ 849 850 mp->map_type = DDI_MT_REGSPEC; 851 mp->map_obj.rp = rp; 852 } 853 854 /* 855 * Adjust offset and length correspnding to called values... 856 * XXX: A non-zero length means override the one in the regspec 857 * XXX: (regardless of what's in the parent's range?) 858 */ 859 860 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 861 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 862 863 #ifdef DDI_MAP_DEBUG 864 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 865 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 866 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 867 len, mp->map_handlep); 868 #endif /* DDI_MAP_DEBUG */ 869 870 /* 871 * I/O or memory mapping: 872 * 873 * <bustype=0, addr=x, len=x>: memory 874 * <bustype=1, addr=x, len=x>: i/o 875 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 876 */ 877 878 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 879 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 880 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 881 ddi_get_name(rdip), rp->regspec_bustype, 882 rp->regspec_addr, rp->regspec_size); 883 return (DDI_ME_INVAL); 884 } 885 886 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 887 /* 888 * compatibility i/o mapping 889 */ 890 rp->regspec_bustype += (uint_t)offset; 891 } else { 892 /* 893 * Normal memory or i/o mapping 894 */ 895 rp->regspec_addr += (uint_t)offset; 896 } 897 898 if (len != 0) 899 rp->regspec_size = (uint_t)len; 900 901 #ifdef DDI_MAP_DEBUG 902 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 903 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 904 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 905 offset, len, mp->map_handlep); 906 #endif /* DDI_MAP_DEBUG */ 907 908 /* 909 * Apply any parent ranges at this level, if applicable. 910 * (This is where nexus specific regspec translation takes place. 911 * Use of this function is implicit agreement that translation is 912 * provided via ddi_apply_range.) 913 */ 914 915 #ifdef DDI_MAP_DEBUG 916 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 917 ddi_get_name(dip), ddi_get_name(rdip)); 918 #endif /* DDI_MAP_DEBUG */ 919 920 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 921 return (error); 922 923 switch (mp->map_op) { 924 case DDI_MO_MAP_LOCKED: 925 926 /* 927 * Set up the locked down kernel mapping to the regspec... 928 */ 929 930 return (rootnex_map_regspec(mp, vaddrp)); 931 932 case DDI_MO_UNMAP: 933 934 /* 935 * Release mapping... 936 */ 937 938 return (rootnex_unmap_regspec(mp, vaddrp)); 939 940 case DDI_MO_MAP_HANDLE: 941 942 return (rootnex_map_handle(mp)); 943 944 default: 945 return (DDI_ME_UNIMPLEMENTED); 946 } 947 } 948 949 950 /* 951 * rootnex_map_fault() 952 * 953 * fault in mappings for requestors 954 */ 955 /*ARGSUSED*/ 956 static int 957 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 958 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 959 uint_t lock) 960 { 961 962 #ifdef DDI_MAP_DEBUG 963 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 964 ddi_map_debug(" Seg <%s>\n", 965 seg->s_ops == &segdev_ops ? "segdev" : 966 seg == &kvseg ? "segkmem" : "NONE!"); 967 #endif /* DDI_MAP_DEBUG */ 968 969 /* 970 * This is all terribly broken, but it is a start 971 * 972 * XXX Note that this test means that segdev_ops 973 * must be exported from seg_dev.c. 974 * XXX What about devices with their own segment drivers? 975 */ 976 if (seg->s_ops == &segdev_ops) { 977 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 978 979 if (hat == NULL) { 980 /* 981 * This is one plausible interpretation of 982 * a null hat i.e. use the first hat on the 983 * address space hat list which by convention is 984 * the hat of the system MMU. At alternative 985 * would be to panic .. this might well be better .. 986 */ 987 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 988 hat = seg->s_as->a_hat; 989 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 990 } 991 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 992 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 993 } else if (seg == &kvseg && dp == NULL) { 994 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 995 HAT_LOAD_LOCK); 996 } else 997 return (DDI_FAILURE); 998 return (DDI_SUCCESS); 999 } 1000 1001 1002 /* 1003 * rootnex_map_regspec() 1004 * we don't support mapping of I/O cards above 4Gb 1005 */ 1006 static int 1007 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1008 { 1009 rootnex_addr_t rbase; 1010 void *cvaddr; 1011 uint_t npages, pgoffset; 1012 struct regspec *rp; 1013 ddi_acc_hdl_t *hp; 1014 ddi_acc_impl_t *ap; 1015 uint_t hat_acc_flags; 1016 paddr_t pbase; 1017 1018 rp = mp->map_obj.rp; 1019 hp = mp->map_handlep; 1020 1021 #ifdef DDI_MAP_DEBUG 1022 ddi_map_debug( 1023 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1024 rp->regspec_bustype, rp->regspec_addr, 1025 rp->regspec_size, mp->map_handlep); 1026 #endif /* DDI_MAP_DEBUG */ 1027 1028 /* 1029 * I/O or memory mapping 1030 * 1031 * <bustype=0, addr=x, len=x>: memory 1032 * <bustype=1, addr=x, len=x>: i/o 1033 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1034 */ 1035 1036 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1037 cmn_err(CE_WARN, "rootnex: invalid register spec" 1038 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1039 rp->regspec_addr, rp->regspec_size); 1040 return (DDI_FAILURE); 1041 } 1042 1043 if (rp->regspec_bustype != 0) { 1044 /* 1045 * I/O space - needs a handle. 1046 */ 1047 if (hp == NULL) { 1048 return (DDI_FAILURE); 1049 } 1050 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1051 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1052 impl_acc_hdl_init(hp); 1053 1054 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1055 #ifdef DDI_MAP_DEBUG 1056 ddi_map_debug("rootnex_map_regspec: mmap() " 1057 "to I/O space is not supported.\n"); 1058 #endif /* DDI_MAP_DEBUG */ 1059 return (DDI_ME_INVAL); 1060 } else { 1061 /* 1062 * 1275-compliant vs. compatibility i/o mapping 1063 */ 1064 *vaddrp = 1065 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1066 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1067 ((caddr_t)(uintptr_t)rp->regspec_addr); 1068 #ifdef __xpv 1069 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1070 hp->ah_pfn = xen_assign_pfn( 1071 mmu_btop((ulong_t)rp->regspec_addr & 1072 MMU_PAGEMASK)); 1073 } else { 1074 hp->ah_pfn = mmu_btop( 1075 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1076 } 1077 #else 1078 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1079 MMU_PAGEMASK); 1080 #endif 1081 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1082 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1083 } 1084 1085 #ifdef DDI_MAP_DEBUG 1086 ddi_map_debug( 1087 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1088 rp->regspec_size, *vaddrp); 1089 #endif /* DDI_MAP_DEBUG */ 1090 return (DDI_SUCCESS); 1091 } 1092 1093 /* 1094 * Memory space 1095 */ 1096 1097 if (hp != NULL) { 1098 /* 1099 * hat layer ignores 1100 * hp->ah_acc.devacc_attr_endian_flags. 1101 */ 1102 switch (hp->ah_acc.devacc_attr_dataorder) { 1103 case DDI_STRICTORDER_ACC: 1104 hat_acc_flags = HAT_STRICTORDER; 1105 break; 1106 case DDI_UNORDERED_OK_ACC: 1107 hat_acc_flags = HAT_UNORDERED_OK; 1108 break; 1109 case DDI_MERGING_OK_ACC: 1110 hat_acc_flags = HAT_MERGING_OK; 1111 break; 1112 case DDI_LOADCACHING_OK_ACC: 1113 hat_acc_flags = HAT_LOADCACHING_OK; 1114 break; 1115 case DDI_STORECACHING_OK_ACC: 1116 hat_acc_flags = HAT_STORECACHING_OK; 1117 break; 1118 } 1119 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1120 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1121 impl_acc_hdl_init(hp); 1122 hp->ah_hat_flags = hat_acc_flags; 1123 } else { 1124 hat_acc_flags = HAT_STRICTORDER; 1125 } 1126 1127 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1128 #ifdef __xpv 1129 /* 1130 * If we're dom0, we're using a real device so we need to translate 1131 * the MA to a PA. 1132 */ 1133 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1134 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1135 } else { 1136 pbase = rbase; 1137 } 1138 #else 1139 pbase = rbase; 1140 #endif 1141 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1142 1143 if (rp->regspec_size == 0) { 1144 #ifdef DDI_MAP_DEBUG 1145 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1146 #endif /* DDI_MAP_DEBUG */ 1147 return (DDI_ME_INVAL); 1148 } 1149 1150 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1151 /* extra cast to make gcc happy */ 1152 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1153 } else { 1154 npages = mmu_btopr(rp->regspec_size + pgoffset); 1155 1156 #ifdef DDI_MAP_DEBUG 1157 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1158 "physical %llx", npages, pbase); 1159 #endif /* DDI_MAP_DEBUG */ 1160 1161 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1162 if (cvaddr == NULL) 1163 return (DDI_ME_NORESOURCES); 1164 1165 /* 1166 * Now map in the pages we've allocated... 1167 */ 1168 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1169 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1170 HAT_LOAD_LOCK); 1171 *vaddrp = (caddr_t)cvaddr + pgoffset; 1172 1173 /* save away pfn and npages for FMA */ 1174 hp = mp->map_handlep; 1175 if (hp) { 1176 hp->ah_pfn = mmu_btop(pbase); 1177 hp->ah_pnum = npages; 1178 } 1179 } 1180 1181 #ifdef DDI_MAP_DEBUG 1182 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1183 #endif /* DDI_MAP_DEBUG */ 1184 return (DDI_SUCCESS); 1185 } 1186 1187 1188 /* 1189 * rootnex_unmap_regspec() 1190 * 1191 */ 1192 static int 1193 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1194 { 1195 caddr_t addr = (caddr_t)*vaddrp; 1196 uint_t npages, pgoffset; 1197 struct regspec *rp; 1198 1199 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1200 return (0); 1201 1202 rp = mp->map_obj.rp; 1203 1204 if (rp->regspec_size == 0) { 1205 #ifdef DDI_MAP_DEBUG 1206 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1207 #endif /* DDI_MAP_DEBUG */ 1208 return (DDI_ME_INVAL); 1209 } 1210 1211 /* 1212 * I/O or memory mapping: 1213 * 1214 * <bustype=0, addr=x, len=x>: memory 1215 * <bustype=1, addr=x, len=x>: i/o 1216 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1217 */ 1218 if (rp->regspec_bustype != 0) { 1219 /* 1220 * This is I/O space, which requires no particular 1221 * processing on unmap since it isn't mapped in the 1222 * first place. 1223 */ 1224 return (DDI_SUCCESS); 1225 } 1226 1227 /* 1228 * Memory space 1229 */ 1230 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1231 npages = mmu_btopr(rp->regspec_size + pgoffset); 1232 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1233 device_arena_free(addr - pgoffset, ptob(npages)); 1234 1235 /* 1236 * Destroy the pointer - the mapping has logically gone 1237 */ 1238 *vaddrp = NULL; 1239 1240 return (DDI_SUCCESS); 1241 } 1242 1243 1244 /* 1245 * rootnex_map_handle() 1246 * 1247 */ 1248 static int 1249 rootnex_map_handle(ddi_map_req_t *mp) 1250 { 1251 rootnex_addr_t rbase; 1252 ddi_acc_hdl_t *hp; 1253 uint_t pgoffset; 1254 struct regspec *rp; 1255 paddr_t pbase; 1256 1257 rp = mp->map_obj.rp; 1258 1259 #ifdef DDI_MAP_DEBUG 1260 ddi_map_debug( 1261 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1262 rp->regspec_bustype, rp->regspec_addr, 1263 rp->regspec_size, mp->map_handlep); 1264 #endif /* DDI_MAP_DEBUG */ 1265 1266 /* 1267 * I/O or memory mapping: 1268 * 1269 * <bustype=0, addr=x, len=x>: memory 1270 * <bustype=1, addr=x, len=x>: i/o 1271 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1272 */ 1273 if (rp->regspec_bustype != 0) { 1274 /* 1275 * This refers to I/O space, and we don't support "mapping" 1276 * I/O space to a user. 1277 */ 1278 return (DDI_FAILURE); 1279 } 1280 1281 /* 1282 * Set up the hat_flags for the mapping. 1283 */ 1284 hp = mp->map_handlep; 1285 1286 switch (hp->ah_acc.devacc_attr_endian_flags) { 1287 case DDI_NEVERSWAP_ACC: 1288 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1289 break; 1290 case DDI_STRUCTURE_LE_ACC: 1291 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1292 break; 1293 case DDI_STRUCTURE_BE_ACC: 1294 return (DDI_FAILURE); 1295 default: 1296 return (DDI_REGS_ACC_CONFLICT); 1297 } 1298 1299 switch (hp->ah_acc.devacc_attr_dataorder) { 1300 case DDI_STRICTORDER_ACC: 1301 break; 1302 case DDI_UNORDERED_OK_ACC: 1303 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1304 break; 1305 case DDI_MERGING_OK_ACC: 1306 hp->ah_hat_flags |= HAT_MERGING_OK; 1307 break; 1308 case DDI_LOADCACHING_OK_ACC: 1309 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1310 break; 1311 case DDI_STORECACHING_OK_ACC: 1312 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1313 break; 1314 default: 1315 return (DDI_FAILURE); 1316 } 1317 1318 rbase = (rootnex_addr_t)rp->regspec_addr & 1319 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1320 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1321 1322 if (rp->regspec_size == 0) 1323 return (DDI_ME_INVAL); 1324 1325 #ifdef __xpv 1326 /* 1327 * If we're dom0, we're using a real device so we need to translate 1328 * the MA to a PA. 1329 */ 1330 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1331 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1332 (rbase & MMU_PAGEOFFSET); 1333 } else { 1334 pbase = rbase; 1335 } 1336 #else 1337 pbase = rbase; 1338 #endif 1339 1340 hp->ah_pfn = mmu_btop(pbase); 1341 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1342 1343 return (DDI_SUCCESS); 1344 } 1345 1346 1347 1348 /* 1349 * ************************ 1350 * interrupt related code 1351 * ************************ 1352 */ 1353 1354 /* 1355 * rootnex_intr_ops() 1356 * bus_intr_op() function for interrupt support 1357 */ 1358 /* ARGSUSED */ 1359 static int 1360 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1361 ddi_intr_handle_impl_t *hdlp, void *result) 1362 { 1363 struct intrspec *ispec; 1364 struct ddi_parent_private_data *pdp; 1365 1366 DDI_INTR_NEXDBG((CE_CONT, 1367 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1368 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1369 1370 /* Process the interrupt operation */ 1371 switch (intr_op) { 1372 case DDI_INTROP_GETCAP: 1373 /* First check with pcplusmp */ 1374 if (psm_intr_ops == NULL) 1375 return (DDI_FAILURE); 1376 1377 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1378 *(int *)result = 0; 1379 return (DDI_FAILURE); 1380 } 1381 break; 1382 case DDI_INTROP_SETCAP: 1383 if (psm_intr_ops == NULL) 1384 return (DDI_FAILURE); 1385 1386 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1387 return (DDI_FAILURE); 1388 break; 1389 case DDI_INTROP_ALLOC: 1390 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1391 return (DDI_FAILURE); 1392 hdlp->ih_pri = ispec->intrspec_pri; 1393 *(int *)result = hdlp->ih_scratch1; 1394 break; 1395 case DDI_INTROP_FREE: 1396 pdp = ddi_get_parent_data(rdip); 1397 /* 1398 * Special case for 'pcic' driver' only. 1399 * If an intrspec was created for it, clean it up here 1400 * See detailed comments on this in the function 1401 * rootnex_get_ispec(). 1402 */ 1403 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1404 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1405 pdp->par_nintr); 1406 /* 1407 * Set it to zero; so that 1408 * DDI framework doesn't free it again 1409 */ 1410 pdp->par_intr = NULL; 1411 pdp->par_nintr = 0; 1412 } 1413 break; 1414 case DDI_INTROP_GETPRI: 1415 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1416 return (DDI_FAILURE); 1417 *(int *)result = ispec->intrspec_pri; 1418 break; 1419 case DDI_INTROP_SETPRI: 1420 /* Validate the interrupt priority passed to us */ 1421 if (*(int *)result > LOCK_LEVEL) 1422 return (DDI_FAILURE); 1423 1424 /* Ensure that PSM is all initialized and ispec is ok */ 1425 if ((psm_intr_ops == NULL) || 1426 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1427 return (DDI_FAILURE); 1428 1429 /* Change the priority */ 1430 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1431 PSM_FAILURE) 1432 return (DDI_FAILURE); 1433 1434 /* update the ispec with the new priority */ 1435 ispec->intrspec_pri = *(int *)result; 1436 break; 1437 case DDI_INTROP_ADDISR: 1438 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1439 return (DDI_FAILURE); 1440 ispec->intrspec_func = hdlp->ih_cb_func; 1441 break; 1442 case DDI_INTROP_REMISR: 1443 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1444 return (DDI_FAILURE); 1445 ispec->intrspec_func = (uint_t (*)()) 0; 1446 break; 1447 case DDI_INTROP_ENABLE: 1448 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1449 return (DDI_FAILURE); 1450 1451 /* Call psmi to translate irq with the dip */ 1452 if (psm_intr_ops == NULL) 1453 return (DDI_FAILURE); 1454 1455 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1456 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1457 (int *)&hdlp->ih_vector) == PSM_FAILURE) 1458 return (DDI_FAILURE); 1459 1460 /* Add the interrupt handler */ 1461 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1462 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1463 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1464 return (DDI_FAILURE); 1465 break; 1466 case DDI_INTROP_DISABLE: 1467 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1468 return (DDI_FAILURE); 1469 1470 /* Call psm_ops() to translate irq with the dip */ 1471 if (psm_intr_ops == NULL) 1472 return (DDI_FAILURE); 1473 1474 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1475 (void) (*psm_intr_ops)(rdip, hdlp, 1476 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1477 1478 /* Remove the interrupt handler */ 1479 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1480 hdlp->ih_cb_func, hdlp->ih_vector); 1481 break; 1482 case DDI_INTROP_SETMASK: 1483 if (psm_intr_ops == NULL) 1484 return (DDI_FAILURE); 1485 1486 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1487 return (DDI_FAILURE); 1488 break; 1489 case DDI_INTROP_CLRMASK: 1490 if (psm_intr_ops == NULL) 1491 return (DDI_FAILURE); 1492 1493 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1494 return (DDI_FAILURE); 1495 break; 1496 case DDI_INTROP_GETPENDING: 1497 if (psm_intr_ops == NULL) 1498 return (DDI_FAILURE); 1499 1500 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1501 result)) { 1502 *(int *)result = 0; 1503 return (DDI_FAILURE); 1504 } 1505 break; 1506 case DDI_INTROP_NAVAIL: 1507 case DDI_INTROP_NINTRS: 1508 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1509 if (*(int *)result == 0) { 1510 /* 1511 * Special case for 'pcic' driver' only. This driver 1512 * driver is a child of 'isa' and 'rootnex' drivers. 1513 * 1514 * See detailed comments on this in the function 1515 * rootnex_get_ispec(). 1516 * 1517 * Children of 'pcic' send 'NINITR' request all the 1518 * way to rootnex driver. But, the 'pdp->par_nintr' 1519 * field may not initialized. So, we fake it here 1520 * to return 1 (a la what PCMCIA nexus does). 1521 */ 1522 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1523 *(int *)result = 1; 1524 else 1525 return (DDI_FAILURE); 1526 } 1527 break; 1528 case DDI_INTROP_SUPPORTED_TYPES: 1529 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1530 break; 1531 default: 1532 return (DDI_FAILURE); 1533 } 1534 1535 return (DDI_SUCCESS); 1536 } 1537 1538 1539 /* 1540 * rootnex_get_ispec() 1541 * convert an interrupt number to an interrupt specification. 1542 * The interrupt number determines which interrupt spec will be 1543 * returned if more than one exists. 1544 * 1545 * Look into the parent private data area of the 'rdip' to find out 1546 * the interrupt specification. First check to make sure there is 1547 * one that matchs "inumber" and then return a pointer to it. 1548 * 1549 * Return NULL if one could not be found. 1550 * 1551 * NOTE: This is needed for rootnex_intr_ops() 1552 */ 1553 static struct intrspec * 1554 rootnex_get_ispec(dev_info_t *rdip, int inum) 1555 { 1556 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1557 1558 /* 1559 * Special case handling for drivers that provide their own 1560 * intrspec structures instead of relying on the DDI framework. 1561 * 1562 * A broken hardware driver in ON could potentially provide its 1563 * own intrspec structure, instead of relying on the hardware. 1564 * If these drivers are children of 'rootnex' then we need to 1565 * continue to provide backward compatibility to them here. 1566 * 1567 * Following check is a special case for 'pcic' driver which 1568 * was found to have broken hardwre andby provides its own intrspec. 1569 * 1570 * Verbatim comments from this driver are shown here: 1571 * "Don't use the ddi_add_intr since we don't have a 1572 * default intrspec in all cases." 1573 * 1574 * Since an 'ispec' may not be always created for it, 1575 * check for that and create one if so. 1576 * 1577 * NOTE: Currently 'pcic' is the only driver found to do this. 1578 */ 1579 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1580 pdp->par_nintr = 1; 1581 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1582 pdp->par_nintr, KM_SLEEP); 1583 } 1584 1585 /* Validate the interrupt number */ 1586 if (inum >= pdp->par_nintr) 1587 return (NULL); 1588 1589 /* Get the interrupt structure pointer and return that */ 1590 return ((struct intrspec *)&pdp->par_intr[inum]); 1591 } 1592 1593 1594 /* 1595 * ****************** 1596 * dma related code 1597 * ****************** 1598 */ 1599 1600 /*ARGSUSED*/ 1601 static int 1602 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1603 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1604 ddi_dma_handle_t *handlep) 1605 { 1606 uint64_t maxsegmentsize_ll; 1607 uint_t maxsegmentsize; 1608 ddi_dma_impl_t *hp; 1609 rootnex_dma_t *dma; 1610 uint64_t count_max; 1611 uint64_t seg; 1612 int kmflag; 1613 int e; 1614 1615 1616 /* convert our sleep flags */ 1617 if (waitfp == DDI_DMA_SLEEP) { 1618 kmflag = KM_SLEEP; 1619 } else { 1620 kmflag = KM_NOSLEEP; 1621 } 1622 1623 /* 1624 * We try to do only one memory allocation here. We'll do a little 1625 * pointer manipulation later. If the bind ends up taking more than 1626 * our prealloc's space, we'll have to allocate more memory in the 1627 * bind operation. Not great, but much better than before and the 1628 * best we can do with the current bind interfaces. 1629 */ 1630 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1631 if (hp == NULL) { 1632 if (waitfp != DDI_DMA_DONTWAIT) { 1633 ddi_set_callback(waitfp, arg, 1634 &rootnex_state->r_dvma_call_list_id); 1635 } 1636 return (DDI_DMA_NORESOURCES); 1637 } 1638 1639 /* Do our pointer manipulation now, align the structures */ 1640 hp->dmai_private = (void *)(((uintptr_t)hp + 1641 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1642 dma = (rootnex_dma_t *)hp->dmai_private; 1643 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1644 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1645 1646 /* setup the handle */ 1647 rootnex_clean_dmahdl(hp); 1648 hp->dmai_error.err_fep = NULL; 1649 hp->dmai_error.err_cf = NULL; 1650 dma->dp_dip = rdip; 1651 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1652 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1653 hp->dmai_minxfer = attr->dma_attr_minxfer; 1654 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1655 hp->dmai_rdip = rdip; 1656 hp->dmai_attr = *attr; 1657 1658 /* we don't need to worry about the SPL since we do a tryenter */ 1659 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1660 1661 /* 1662 * Figure out our maximum segment size. If the segment size is greater 1663 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1664 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1665 * dma_attr_count_max are size-1 type values. 1666 * 1667 * Maximum segment size is the largest physically contiguous chunk of 1668 * memory that we can return from a bind (i.e. the maximum size of a 1669 * single cookie). 1670 */ 1671 1672 /* handle the rollover cases */ 1673 seg = attr->dma_attr_seg + 1; 1674 if (seg < attr->dma_attr_seg) { 1675 seg = attr->dma_attr_seg; 1676 } 1677 count_max = attr->dma_attr_count_max + 1; 1678 if (count_max < attr->dma_attr_count_max) { 1679 count_max = attr->dma_attr_count_max; 1680 } 1681 1682 /* 1683 * granularity may or may not be a power of two. If it isn't, we can't 1684 * use a simple mask. 1685 */ 1686 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1687 dma->dp_granularity_power_2 = B_FALSE; 1688 } else { 1689 dma->dp_granularity_power_2 = B_TRUE; 1690 } 1691 1692 /* 1693 * maxxfer should be a whole multiple of granularity. If we're going to 1694 * break up a window because we're greater than maxxfer, we might as 1695 * well make sure it's maxxfer is a whole multiple so we don't have to 1696 * worry about triming the window later on for this case. 1697 */ 1698 if (attr->dma_attr_granular > 1) { 1699 if (dma->dp_granularity_power_2) { 1700 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1701 (attr->dma_attr_maxxfer & 1702 (attr->dma_attr_granular - 1)); 1703 } else { 1704 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1705 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1706 } 1707 } else { 1708 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1709 } 1710 1711 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1712 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1713 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1714 maxsegmentsize = 0xFFFFFFFF; 1715 } else { 1716 maxsegmentsize = maxsegmentsize_ll; 1717 } 1718 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1719 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1720 dma->dp_sglinfo.si_flags = attr->dma_attr_flags; 1721 1722 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1723 if (rootnex_alloc_check_parms) { 1724 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1725 if (e != DDI_SUCCESS) { 1726 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1727 (void) rootnex_dma_freehdl(dip, rdip, 1728 (ddi_dma_handle_t)hp); 1729 return (e); 1730 } 1731 } 1732 1733 *handlep = (ddi_dma_handle_t)hp; 1734 1735 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1736 ROOTNEX_DPROBE1(rootnex__alloc__handle, uint64_t, 1737 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1738 1739 return (DDI_SUCCESS); 1740 } 1741 1742 1743 /* 1744 * rootnex_dma_allochdl() 1745 * called from ddi_dma_alloc_handle(). 1746 */ 1747 static int 1748 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1749 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1750 { 1751 int retval; 1752 #if defined(__amd64) && !defined(__xpv) 1753 uint_t error = ENOTSUP; 1754 1755 retval = iommulib_nex_open(rdip, &error); 1756 1757 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1758 /* No IOMMU */ 1759 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1760 handlep)); 1761 } else if (retval != DDI_SUCCESS) { 1762 return (DDI_FAILURE); 1763 } 1764 1765 ASSERT(IOMMU_USED(rdip)); 1766 1767 /* has an IOMMU */ 1768 retval = iommulib_nexdma_allochdl(dip, rdip, attr, 1769 waitfp, arg, handlep); 1770 #else 1771 retval = rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1772 handlep); 1773 #endif 1774 if (retval == DDI_SUCCESS) 1775 ndi_fmc_insert(rdip, DMA_HANDLE, *handlep, NULL); 1776 return (retval); 1777 } 1778 1779 /*ARGSUSED*/ 1780 static int 1781 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1782 ddi_dma_handle_t handle) 1783 { 1784 ddi_dma_impl_t *hp; 1785 rootnex_dma_t *dma; 1786 1787 1788 hp = (ddi_dma_impl_t *)handle; 1789 dma = (rootnex_dma_t *)hp->dmai_private; 1790 1791 /* unbind should have been called first */ 1792 ASSERT(!dma->dp_inuse); 1793 1794 mutex_destroy(&dma->dp_mutex); 1795 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1796 1797 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1798 ROOTNEX_DPROBE1(rootnex__free__handle, uint64_t, 1799 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1800 1801 if (rootnex_state->r_dvma_call_list_id) 1802 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1803 1804 return (DDI_SUCCESS); 1805 } 1806 1807 /* 1808 * rootnex_dma_freehdl() 1809 * called from ddi_dma_free_handle(). 1810 */ 1811 static int 1812 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1813 { 1814 ndi_fmc_remove(rdip, DMA_HANDLE, handle); 1815 #if defined(__amd64) && !defined(__xpv) 1816 if (IOMMU_USED(rdip)) { 1817 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1818 } 1819 #endif 1820 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1821 } 1822 1823 /*ARGSUSED*/ 1824 static int 1825 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1826 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1827 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1828 { 1829 rootnex_sglinfo_t *sinfo; 1830 ddi_dma_attr_t *attr; 1831 ddi_dma_impl_t *hp; 1832 rootnex_dma_t *dma; 1833 int kmflag; 1834 int e; 1835 1836 hp = (ddi_dma_impl_t *)handle; 1837 dma = (rootnex_dma_t *)hp->dmai_private; 1838 sinfo = &dma->dp_sglinfo; 1839 attr = &hp->dmai_attr; 1840 1841 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1842 dma->dp_sleep_flags = KM_SLEEP; 1843 } else { 1844 dma->dp_sleep_flags = KM_NOSLEEP; 1845 } 1846 1847 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1848 1849 /* 1850 * This is useful for debugging a driver. Not as useful in a production 1851 * system. The only time this will fail is if you have a driver bug. 1852 */ 1853 if (rootnex_bind_check_inuse) { 1854 /* 1855 * No one else should ever have this lock unless someone else 1856 * is trying to use this handle. So contention on the lock 1857 * is the same as inuse being set. 1858 */ 1859 e = mutex_tryenter(&dma->dp_mutex); 1860 if (e == 0) { 1861 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1862 return (DDI_DMA_INUSE); 1863 } 1864 if (dma->dp_inuse) { 1865 mutex_exit(&dma->dp_mutex); 1866 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1867 return (DDI_DMA_INUSE); 1868 } 1869 dma->dp_inuse = B_TRUE; 1870 mutex_exit(&dma->dp_mutex); 1871 } 1872 1873 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1874 if (rootnex_bind_check_parms) { 1875 e = rootnex_valid_bind_parms(dmareq, attr); 1876 if (e != DDI_SUCCESS) { 1877 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1878 rootnex_clean_dmahdl(hp); 1879 return (e); 1880 } 1881 } 1882 1883 /* save away the original bind info */ 1884 dma->dp_dma = dmareq->dmar_object; 1885 1886 #if defined(__amd64) && !defined(__xpv) 1887 e = immu_map_sgl(hp, dmareq, rootnex_prealloc_cookies, rdip); 1888 switch (e) { 1889 case DDI_DMA_MAPPED: 1890 goto out; 1891 case DDI_DMA_USE_PHYSICAL: 1892 break; 1893 case DDI_DMA_PARTIAL: 1894 ddi_err(DER_PANIC, rdip, "Partial DVMA map"); 1895 e = DDI_DMA_NORESOURCES; 1896 /*FALLTHROUGH*/ 1897 default: 1898 ddi_err(DER_MODE, rdip, "DVMA map failed"); 1899 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1900 rootnex_clean_dmahdl(hp); 1901 return (e); 1902 } 1903 #endif 1904 1905 /* 1906 * Figure out a rough estimate of what maximum number of pages this 1907 * buffer could use (a high estimate of course). 1908 */ 1909 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1910 1911 /* 1912 * We'll use the pre-allocated cookies for any bind that will *always* 1913 * fit (more important to be consistent, we don't want to create 1914 * additional degenerate cases). 1915 */ 1916 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1917 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1918 dma->dp_need_to_free_cookie = B_FALSE; 1919 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1920 uint_t, sinfo->si_max_pages); 1921 1922 /* 1923 * For anything larger than that, we'll go ahead and allocate the 1924 * maximum number of pages we expect to see. Hopefuly, we won't be 1925 * seeing this path in the fast path for high performance devices very 1926 * frequently. 1927 * 1928 * a ddi bind interface that allowed the driver to provide storage to 1929 * the bind interface would speed this case up. 1930 */ 1931 } else { 1932 /* convert the sleep flags */ 1933 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1934 kmflag = KM_SLEEP; 1935 } else { 1936 kmflag = KM_NOSLEEP; 1937 } 1938 1939 /* 1940 * Save away how much memory we allocated. If we're doing a 1941 * nosleep, the alloc could fail... 1942 */ 1943 dma->dp_cookie_size = sinfo->si_max_pages * 1944 sizeof (ddi_dma_cookie_t); 1945 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1946 if (dma->dp_cookies == NULL) { 1947 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1948 rootnex_clean_dmahdl(hp); 1949 return (DDI_DMA_NORESOURCES); 1950 } 1951 dma->dp_need_to_free_cookie = B_TRUE; 1952 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1953 sinfo->si_max_pages); 1954 } 1955 hp->dmai_cookie = dma->dp_cookies; 1956 1957 /* 1958 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1959 * looking at the constraints in the dma structure. It will then put 1960 * some additional state about the sgl in the dma struct (i.e. is 1961 * the sgl clean, or do we need to do some munging; how many pages 1962 * need to be copied, etc.) 1963 */ 1964 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1965 &dma->dp_sglinfo); 1966 1967 out: 1968 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1969 /* if we don't need a copy buffer, we don't need to sync */ 1970 if (sinfo->si_copybuf_req == 0) { 1971 hp->dmai_rflags |= DMP_NOSYNC; 1972 } 1973 1974 /* 1975 * if we don't need the copybuf and we don't need to do a partial, we 1976 * hit the fast path. All the high performance devices should be trying 1977 * to hit this path. To hit this path, a device should be able to reach 1978 * all of memory, shouldn't try to bind more than it can transfer, and 1979 * the buffer shouldn't require more cookies than the driver/device can 1980 * handle [sgllen]). 1981 */ 1982 if ((sinfo->si_copybuf_req == 0) && 1983 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1984 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1985 /* 1986 * If the driver supports FMA, insert the handle in the FMA DMA 1987 * handle cache. 1988 */ 1989 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 1990 hp->dmai_error.err_cf = rootnex_dma_check; 1991 1992 /* 1993 * copy out the first cookie and ccountp, set the cookie 1994 * pointer to the second cookie. The first cookie is passed 1995 * back on the stack. Additional cookies are accessed via 1996 * ddi_dma_nextcookie() 1997 */ 1998 *cookiep = dma->dp_cookies[0]; 1999 *ccountp = sinfo->si_sgl_size; 2000 hp->dmai_cookie++; 2001 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2002 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2003 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, 2004 uint64_t, rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], 2005 uint_t, dma->dp_dma.dmao_size); 2006 2007 2008 return (DDI_DMA_MAPPED); 2009 } 2010 2011 /* 2012 * go to the slow path, we may need to alloc more memory, create 2013 * multiple windows, and munge up a sgl to make the device happy. 2014 */ 2015 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2016 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2017 if (dma->dp_need_to_free_cookie) { 2018 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2019 } 2020 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2021 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2022 return (e); 2023 } 2024 2025 /* 2026 * If the driver supports FMA, insert the handle in the FMA DMA handle 2027 * cache. 2028 */ 2029 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) 2030 hp->dmai_error.err_cf = rootnex_dma_check; 2031 2032 /* if the first window uses the copy buffer, sync it for the device */ 2033 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2034 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2035 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2036 DDI_DMA_SYNC_FORDEV); 2037 } 2038 2039 /* 2040 * copy out the first cookie and ccountp, set the cookie pointer to the 2041 * second cookie. Make sure the partial flag is set/cleared correctly. 2042 * If we have a partial map (i.e. multiple windows), the number of 2043 * cookies we return is the number of cookies in the first window. 2044 */ 2045 if (e == DDI_DMA_MAPPED) { 2046 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2047 *ccountp = sinfo->si_sgl_size; 2048 hp->dmai_nwin = 1; 2049 } else { 2050 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2051 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2052 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2053 } 2054 *cookiep = dma->dp_cookies[0]; 2055 hp->dmai_cookie++; 2056 2057 ROOTNEX_DPROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2058 ROOTNEX_DPROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2059 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2060 dma->dp_dma.dmao_size); 2061 return (e); 2062 } 2063 2064 /* 2065 * rootnex_dma_bindhdl() 2066 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2067 */ 2068 static int 2069 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2070 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2071 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2072 { 2073 #if defined(__amd64) && !defined(__xpv) 2074 if (IOMMU_USED(rdip)) { 2075 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2076 cookiep, ccountp)); 2077 } 2078 #endif 2079 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2080 cookiep, ccountp)); 2081 } 2082 2083 2084 2085 /*ARGSUSED*/ 2086 static int 2087 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2088 ddi_dma_handle_t handle) 2089 { 2090 ddi_dma_impl_t *hp; 2091 rootnex_dma_t *dma; 2092 int e; 2093 2094 hp = (ddi_dma_impl_t *)handle; 2095 dma = (rootnex_dma_t *)hp->dmai_private; 2096 2097 /* make sure the buffer wasn't free'd before calling unbind */ 2098 if (rootnex_unbind_verify_buffer) { 2099 e = rootnex_verify_buffer(dma); 2100 if (e != DDI_SUCCESS) { 2101 ASSERT(0); 2102 return (DDI_FAILURE); 2103 } 2104 } 2105 2106 /* sync the current window before unbinding the buffer */ 2107 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2108 (hp->dmai_rflags & DDI_DMA_READ)) { 2109 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 2110 DDI_DMA_SYNC_FORCPU); 2111 } 2112 2113 /* 2114 * cleanup and copy buffer or window state. if we didn't use the copy 2115 * buffer or windows, there won't be much to do :-) 2116 */ 2117 rootnex_teardown_copybuf(dma); 2118 rootnex_teardown_windows(dma); 2119 2120 #if defined(__amd64) && !defined(__xpv) 2121 /* 2122 * Clean up the page tables and free the dvma 2123 */ 2124 e = immu_unmap_sgl(hp, rdip); 2125 if (e != DDI_DMA_USE_PHYSICAL && e != DDI_SUCCESS) { 2126 return (e); 2127 } 2128 #endif 2129 2130 /* 2131 * If we had to allocate space to for the worse case sgl (it didn't 2132 * fit into our pre-allocate buffer), free that up now 2133 */ 2134 if (dma->dp_need_to_free_cookie) { 2135 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2136 } 2137 2138 /* 2139 * clean up the handle so it's ready for the next bind (i.e. if the 2140 * handle is reused). 2141 */ 2142 rootnex_clean_dmahdl(hp); 2143 hp->dmai_error.err_cf = NULL; 2144 2145 if (rootnex_state->r_dvma_call_list_id) 2146 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2147 2148 ROOTNEX_DPROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2149 ROOTNEX_DPROBE1(rootnex__unbind, uint64_t, 2150 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2151 2152 return (DDI_SUCCESS); 2153 } 2154 2155 /* 2156 * rootnex_dma_unbindhdl() 2157 * called from ddi_dma_unbind_handle() 2158 */ 2159 /*ARGSUSED*/ 2160 static int 2161 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2162 ddi_dma_handle_t handle) 2163 { 2164 #if defined(__amd64) && !defined(__xpv) 2165 if (IOMMU_USED(rdip)) { 2166 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2167 } 2168 #endif 2169 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2170 } 2171 2172 #if defined(__amd64) && !defined(__xpv) 2173 2174 static int 2175 rootnex_coredma_get_sleep_flags(ddi_dma_handle_t handle) 2176 { 2177 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2178 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2179 2180 if (dma->dp_sleep_flags != KM_SLEEP && 2181 dma->dp_sleep_flags != KM_NOSLEEP) 2182 cmn_err(CE_PANIC, "kmem sleep flags not set in DMA handle"); 2183 return (dma->dp_sleep_flags); 2184 } 2185 /*ARGSUSED*/ 2186 static void 2187 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2188 { 2189 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2190 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2191 rootnex_window_t *window; 2192 2193 if (dma->dp_window) { 2194 window = &dma->dp_window[dma->dp_current_win]; 2195 hp->dmai_cookie = window->wd_first_cookie; 2196 } else { 2197 hp->dmai_cookie = dma->dp_cookies; 2198 } 2199 hp->dmai_cookie++; 2200 } 2201 2202 /*ARGSUSED*/ 2203 static int 2204 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2205 ddi_dma_cookie_t **cookiepp, uint_t *ccountp) 2206 { 2207 int i; 2208 int km_flags; 2209 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2210 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2211 rootnex_window_t *window; 2212 ddi_dma_cookie_t *cp; 2213 ddi_dma_cookie_t *cookie; 2214 2215 ASSERT(*cookiepp == NULL); 2216 ASSERT(*ccountp == 0); 2217 2218 if (dma->dp_window) { 2219 window = &dma->dp_window[dma->dp_current_win]; 2220 cp = window->wd_first_cookie; 2221 *ccountp = window->wd_cookie_cnt; 2222 } else { 2223 cp = dma->dp_cookies; 2224 *ccountp = dma->dp_sglinfo.si_sgl_size; 2225 } 2226 2227 km_flags = rootnex_coredma_get_sleep_flags(handle); 2228 cookie = kmem_zalloc(sizeof (ddi_dma_cookie_t) * (*ccountp), km_flags); 2229 if (cookie == NULL) { 2230 return (DDI_DMA_NORESOURCES); 2231 } 2232 2233 for (i = 0; i < *ccountp; i++) { 2234 cookie[i].dmac_notused = cp[i].dmac_notused; 2235 cookie[i].dmac_type = cp[i].dmac_type; 2236 cookie[i].dmac_address = cp[i].dmac_address; 2237 cookie[i].dmac_size = cp[i].dmac_size; 2238 } 2239 2240 *cookiepp = cookie; 2241 2242 return (DDI_SUCCESS); 2243 } 2244 2245 /*ARGSUSED*/ 2246 static int 2247 rootnex_coredma_set_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2248 ddi_dma_cookie_t *cookiep, uint_t ccount) 2249 { 2250 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2251 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2252 rootnex_window_t *window; 2253 ddi_dma_cookie_t *cur_cookiep; 2254 2255 ASSERT(cookiep); 2256 ASSERT(ccount != 0); 2257 ASSERT(dma->dp_need_to_switch_cookies == B_FALSE); 2258 2259 if (dma->dp_window) { 2260 window = &dma->dp_window[dma->dp_current_win]; 2261 dma->dp_saved_cookies = window->wd_first_cookie; 2262 window->wd_first_cookie = cookiep; 2263 ASSERT(ccount == window->wd_cookie_cnt); 2264 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2265 + window->wd_first_cookie; 2266 } else { 2267 dma->dp_saved_cookies = dma->dp_cookies; 2268 dma->dp_cookies = cookiep; 2269 ASSERT(ccount == dma->dp_sglinfo.si_sgl_size); 2270 cur_cookiep = (hp->dmai_cookie - dma->dp_saved_cookies) 2271 + dma->dp_cookies; 2272 } 2273 2274 dma->dp_need_to_switch_cookies = B_TRUE; 2275 hp->dmai_cookie = cur_cookiep; 2276 2277 return (DDI_SUCCESS); 2278 } 2279 2280 /*ARGSUSED*/ 2281 static int 2282 rootnex_coredma_clear_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2283 { 2284 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2285 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2286 rootnex_window_t *window; 2287 ddi_dma_cookie_t *cur_cookiep; 2288 ddi_dma_cookie_t *cookie_array; 2289 uint_t ccount; 2290 2291 /* check if cookies have not been switched */ 2292 if (dma->dp_need_to_switch_cookies == B_FALSE) 2293 return (DDI_SUCCESS); 2294 2295 ASSERT(dma->dp_saved_cookies); 2296 2297 if (dma->dp_window) { 2298 window = &dma->dp_window[dma->dp_current_win]; 2299 cookie_array = window->wd_first_cookie; 2300 window->wd_first_cookie = dma->dp_saved_cookies; 2301 dma->dp_saved_cookies = NULL; 2302 ccount = window->wd_cookie_cnt; 2303 cur_cookiep = (hp->dmai_cookie - cookie_array) 2304 + window->wd_first_cookie; 2305 } else { 2306 cookie_array = dma->dp_cookies; 2307 dma->dp_cookies = dma->dp_saved_cookies; 2308 dma->dp_saved_cookies = NULL; 2309 ccount = dma->dp_sglinfo.si_sgl_size; 2310 cur_cookiep = (hp->dmai_cookie - cookie_array) 2311 + dma->dp_cookies; 2312 } 2313 2314 kmem_free(cookie_array, sizeof (ddi_dma_cookie_t) * ccount); 2315 2316 hp->dmai_cookie = cur_cookiep; 2317 2318 dma->dp_need_to_switch_cookies = B_FALSE; 2319 2320 return (DDI_SUCCESS); 2321 } 2322 2323 #endif 2324 2325 /* 2326 * rootnex_verify_buffer() 2327 * verify buffer wasn't free'd 2328 */ 2329 static int 2330 rootnex_verify_buffer(rootnex_dma_t *dma) 2331 { 2332 page_t **pplist; 2333 caddr_t vaddr; 2334 uint_t pcnt; 2335 uint_t poff; 2336 page_t *pp; 2337 char b; 2338 int i; 2339 2340 /* Figure out how many pages this buffer occupies */ 2341 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2342 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2343 } else { 2344 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2345 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2346 } 2347 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2348 2349 switch (dma->dp_dma.dmao_type) { 2350 case DMA_OTYP_PAGES: 2351 /* 2352 * for a linked list of pp's walk through them to make sure 2353 * they're locked and not free. 2354 */ 2355 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2356 for (i = 0; i < pcnt; i++) { 2357 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2358 return (DDI_FAILURE); 2359 } 2360 pp = pp->p_next; 2361 } 2362 break; 2363 2364 case DMA_OTYP_VADDR: 2365 case DMA_OTYP_BUFVADDR: 2366 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2367 /* 2368 * for an array of pp's walk through them to make sure they're 2369 * not free. It's possible that they may not be locked. 2370 */ 2371 if (pplist) { 2372 for (i = 0; i < pcnt; i++) { 2373 if (PP_ISFREE(pplist[i])) { 2374 return (DDI_FAILURE); 2375 } 2376 } 2377 2378 /* For a virtual address, try to peek at each page */ 2379 } else { 2380 if (dma->dp_sglinfo.si_asp == &kas) { 2381 for (i = 0; i < pcnt; i++) { 2382 if (ddi_peek8(NULL, vaddr, &b) == 2383 DDI_FAILURE) 2384 return (DDI_FAILURE); 2385 vaddr += MMU_PAGESIZE; 2386 } 2387 } 2388 } 2389 break; 2390 2391 default: 2392 ASSERT(0); 2393 break; 2394 } 2395 2396 return (DDI_SUCCESS); 2397 } 2398 2399 2400 /* 2401 * rootnex_clean_dmahdl() 2402 * Clean the dma handle. This should be called on a handle alloc and an 2403 * unbind handle. Set the handle state to the default settings. 2404 */ 2405 static void 2406 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2407 { 2408 rootnex_dma_t *dma; 2409 2410 2411 dma = (rootnex_dma_t *)hp->dmai_private; 2412 2413 hp->dmai_nwin = 0; 2414 dma->dp_current_cookie = 0; 2415 dma->dp_copybuf_size = 0; 2416 dma->dp_window = NULL; 2417 dma->dp_cbaddr = NULL; 2418 dma->dp_inuse = B_FALSE; 2419 dma->dp_need_to_free_cookie = B_FALSE; 2420 dma->dp_need_to_switch_cookies = B_FALSE; 2421 dma->dp_saved_cookies = NULL; 2422 dma->dp_sleep_flags = KM_PANIC; 2423 dma->dp_need_to_free_window = B_FALSE; 2424 dma->dp_partial_required = B_FALSE; 2425 dma->dp_trim_required = B_FALSE; 2426 dma->dp_sglinfo.si_copybuf_req = 0; 2427 #if !defined(__amd64) 2428 dma->dp_cb_remaping = B_FALSE; 2429 dma->dp_kva = NULL; 2430 #endif 2431 2432 /* FMA related initialization */ 2433 hp->dmai_fault = 0; 2434 hp->dmai_fault_check = NULL; 2435 hp->dmai_fault_notify = NULL; 2436 hp->dmai_error.err_ena = 0; 2437 hp->dmai_error.err_status = DDI_FM_OK; 2438 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2439 hp->dmai_error.err_ontrap = NULL; 2440 } 2441 2442 2443 /* 2444 * rootnex_valid_alloc_parms() 2445 * Called in ddi_dma_alloc_handle path to validate its parameters. 2446 */ 2447 static int 2448 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2449 { 2450 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2451 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2452 (attr->dma_attr_granular > MMU_PAGESIZE) || 2453 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2454 return (DDI_DMA_BADATTR); 2455 } 2456 2457 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2458 return (DDI_DMA_BADATTR); 2459 } 2460 2461 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2462 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2463 attr->dma_attr_sgllen <= 0) { 2464 return (DDI_DMA_BADATTR); 2465 } 2466 2467 /* We should be able to DMA into every byte offset in a page */ 2468 if (maxsegmentsize < MMU_PAGESIZE) { 2469 return (DDI_DMA_BADATTR); 2470 } 2471 2472 /* if we're bouncing on seg, seg must be <= addr_hi */ 2473 if ((attr->dma_attr_flags & _DDI_DMA_BOUNCE_ON_SEG) && 2474 (attr->dma_attr_seg > attr->dma_attr_addr_hi)) { 2475 return (DDI_DMA_BADATTR); 2476 } 2477 return (DDI_SUCCESS); 2478 } 2479 2480 /* 2481 * rootnex_valid_bind_parms() 2482 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2483 */ 2484 /* ARGSUSED */ 2485 static int 2486 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2487 { 2488 #if !defined(__amd64) 2489 /* 2490 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2491 * we can track the offset for the obsoleted interfaces. 2492 */ 2493 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2494 return (DDI_DMA_TOOBIG); 2495 } 2496 #endif 2497 2498 return (DDI_SUCCESS); 2499 } 2500 2501 2502 /* 2503 * rootnex_need_bounce_seg() 2504 * check to see if the buffer lives on both side of the seg. 2505 */ 2506 static boolean_t 2507 rootnex_need_bounce_seg(ddi_dma_obj_t *dmar_object, rootnex_sglinfo_t *sglinfo) 2508 { 2509 ddi_dma_atyp_t buftype; 2510 rootnex_addr_t raddr; 2511 boolean_t lower_addr; 2512 boolean_t upper_addr; 2513 uint64_t offset; 2514 page_t **pplist; 2515 uint64_t paddr; 2516 uint32_t psize; 2517 uint32_t size; 2518 caddr_t vaddr; 2519 uint_t pcnt; 2520 page_t *pp; 2521 2522 2523 /* shortcuts */ 2524 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2525 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2526 buftype = dmar_object->dmao_type; 2527 size = dmar_object->dmao_size; 2528 2529 lower_addr = B_FALSE; 2530 upper_addr = B_FALSE; 2531 pcnt = 0; 2532 2533 /* 2534 * Process the first page to handle the initial offset of the buffer. 2535 * We'll use the base address we get later when we loop through all 2536 * the pages. 2537 */ 2538 if (buftype == DMA_OTYP_PAGES) { 2539 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2540 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2541 MMU_PAGEOFFSET; 2542 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2543 psize = MIN(size, (MMU_PAGESIZE - offset)); 2544 pp = pp->p_next; 2545 sglinfo->si_asp = NULL; 2546 } else if (pplist != NULL) { 2547 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2548 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2549 if (sglinfo->si_asp == NULL) { 2550 sglinfo->si_asp = &kas; 2551 } 2552 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2553 paddr += offset; 2554 psize = MIN(size, (MMU_PAGESIZE - offset)); 2555 pcnt++; 2556 } else { 2557 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2558 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2559 if (sglinfo->si_asp == NULL) { 2560 sglinfo->si_asp = &kas; 2561 } 2562 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2563 paddr += offset; 2564 psize = MIN(size, (MMU_PAGESIZE - offset)); 2565 vaddr += psize; 2566 } 2567 2568 #ifdef __xpv 2569 /* 2570 * If we're dom0, we're using a real device so we need to load 2571 * the cookies with MFNs instead of PFNs. 2572 */ 2573 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2574 #else 2575 raddr = paddr; 2576 #endif 2577 2578 if ((raddr + psize) > sglinfo->si_segmask) { 2579 upper_addr = B_TRUE; 2580 } else { 2581 lower_addr = B_TRUE; 2582 } 2583 size -= psize; 2584 2585 /* 2586 * Walk through the rest of the pages in the buffer. Track to see 2587 * if we have pages on both sides of the segment boundary. 2588 */ 2589 while (size > 0) { 2590 /* partial or full page */ 2591 psize = MIN(size, MMU_PAGESIZE); 2592 2593 if (buftype == DMA_OTYP_PAGES) { 2594 /* get the paddr from the page_t */ 2595 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2596 paddr = pfn_to_pa(pp->p_pagenum); 2597 pp = pp->p_next; 2598 } else if (pplist != NULL) { 2599 /* index into the array of page_t's to get the paddr */ 2600 ASSERT(!PP_ISFREE(pplist[pcnt])); 2601 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2602 pcnt++; 2603 } else { 2604 /* call into the VM to get the paddr */ 2605 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2606 vaddr)); 2607 vaddr += psize; 2608 } 2609 2610 #ifdef __xpv 2611 /* 2612 * If we're dom0, we're using a real device so we need to load 2613 * the cookies with MFNs instead of PFNs. 2614 */ 2615 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2616 #else 2617 raddr = paddr; 2618 #endif 2619 2620 if ((raddr + psize) > sglinfo->si_segmask) { 2621 upper_addr = B_TRUE; 2622 } else { 2623 lower_addr = B_TRUE; 2624 } 2625 /* 2626 * if the buffer lives both above and below the segment 2627 * boundary, or the current page is the page immediately 2628 * after the segment, we will use a copy/bounce buffer for 2629 * all pages > seg. 2630 */ 2631 if ((lower_addr && upper_addr) || 2632 (raddr == (sglinfo->si_segmask + 1))) { 2633 return (B_TRUE); 2634 } 2635 2636 size -= psize; 2637 } 2638 2639 return (B_FALSE); 2640 } 2641 2642 2643 /* 2644 * rootnex_get_sgl() 2645 * Called in bind fastpath to get the sgl. Most of this will be replaced 2646 * with a call to the vm layer when vm2.0 comes around... 2647 */ 2648 static void 2649 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2650 rootnex_sglinfo_t *sglinfo) 2651 { 2652 ddi_dma_atyp_t buftype; 2653 rootnex_addr_t raddr; 2654 uint64_t last_page; 2655 uint64_t offset; 2656 uint64_t addrhi; 2657 uint64_t addrlo; 2658 uint64_t maxseg; 2659 page_t **pplist; 2660 uint64_t paddr; 2661 uint32_t psize; 2662 uint32_t size; 2663 caddr_t vaddr; 2664 uint_t pcnt; 2665 page_t *pp; 2666 uint_t cnt; 2667 2668 2669 /* shortcuts */ 2670 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2671 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2672 maxseg = sglinfo->si_max_cookie_size; 2673 buftype = dmar_object->dmao_type; 2674 addrhi = sglinfo->si_max_addr; 2675 addrlo = sglinfo->si_min_addr; 2676 size = dmar_object->dmao_size; 2677 2678 pcnt = 0; 2679 cnt = 0; 2680 2681 2682 /* 2683 * check to see if we need to use the copy buffer for pages over 2684 * the segment attr. 2685 */ 2686 sglinfo->si_bounce_on_seg = B_FALSE; 2687 if (sglinfo->si_flags & _DDI_DMA_BOUNCE_ON_SEG) { 2688 sglinfo->si_bounce_on_seg = rootnex_need_bounce_seg( 2689 dmar_object, sglinfo); 2690 } 2691 2692 /* 2693 * if we were passed down a linked list of pages, i.e. pointer to 2694 * page_t, use this to get our physical address and buf offset. 2695 */ 2696 if (buftype == DMA_OTYP_PAGES) { 2697 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2698 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2699 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2700 MMU_PAGEOFFSET; 2701 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2702 psize = MIN(size, (MMU_PAGESIZE - offset)); 2703 pp = pp->p_next; 2704 sglinfo->si_asp = NULL; 2705 2706 /* 2707 * We weren't passed down a linked list of pages, but if we were passed 2708 * down an array of pages, use this to get our physical address and buf 2709 * offset. 2710 */ 2711 } else if (pplist != NULL) { 2712 ASSERT((buftype == DMA_OTYP_VADDR) || 2713 (buftype == DMA_OTYP_BUFVADDR)); 2714 2715 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2716 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2717 if (sglinfo->si_asp == NULL) { 2718 sglinfo->si_asp = &kas; 2719 } 2720 2721 ASSERT(!PP_ISFREE(pplist[pcnt])); 2722 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2723 paddr += offset; 2724 psize = MIN(size, (MMU_PAGESIZE - offset)); 2725 pcnt++; 2726 2727 /* 2728 * All we have is a virtual address, we'll need to call into the VM 2729 * to get the physical address. 2730 */ 2731 } else { 2732 ASSERT((buftype == DMA_OTYP_VADDR) || 2733 (buftype == DMA_OTYP_BUFVADDR)); 2734 2735 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2736 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2737 if (sglinfo->si_asp == NULL) { 2738 sglinfo->si_asp = &kas; 2739 } 2740 2741 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2742 paddr += offset; 2743 psize = MIN(size, (MMU_PAGESIZE - offset)); 2744 vaddr += psize; 2745 } 2746 2747 #ifdef __xpv 2748 /* 2749 * If we're dom0, we're using a real device so we need to load 2750 * the cookies with MFNs instead of PFNs. 2751 */ 2752 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2753 #else 2754 raddr = paddr; 2755 #endif 2756 2757 /* 2758 * Setup the first cookie with the physical address of the page and the 2759 * size of the page (which takes into account the initial offset into 2760 * the page. 2761 */ 2762 sgl[cnt].dmac_laddress = raddr; 2763 sgl[cnt].dmac_size = psize; 2764 sgl[cnt].dmac_type = 0; 2765 2766 /* 2767 * Save away the buffer offset into the page. We'll need this later in 2768 * the copy buffer code to help figure out the page index within the 2769 * buffer and the offset into the current page. 2770 */ 2771 sglinfo->si_buf_offset = offset; 2772 2773 /* 2774 * If we are using the copy buffer for anything over the segment 2775 * boundary, and this page is over the segment boundary. 2776 * OR 2777 * if the DMA engine can't reach the physical address. 2778 */ 2779 if (((sglinfo->si_bounce_on_seg) && 2780 ((raddr + psize) > sglinfo->si_segmask)) || 2781 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2782 /* 2783 * Increase how much copy buffer we use. We always increase by 2784 * pagesize so we don't have to worry about converting offsets. 2785 * Set a flag in the cookies dmac_type to indicate that it uses 2786 * the copy buffer. If this isn't the last cookie, go to the 2787 * next cookie (since we separate each page which uses the copy 2788 * buffer in case the copy buffer is not physically contiguous. 2789 */ 2790 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2791 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2792 if ((cnt + 1) < sglinfo->si_max_pages) { 2793 cnt++; 2794 sgl[cnt].dmac_laddress = 0; 2795 sgl[cnt].dmac_size = 0; 2796 sgl[cnt].dmac_type = 0; 2797 } 2798 } 2799 2800 /* 2801 * save this page's physical address so we can figure out if the next 2802 * page is physically contiguous. Keep decrementing size until we are 2803 * done with the buffer. 2804 */ 2805 last_page = raddr & MMU_PAGEMASK; 2806 size -= psize; 2807 2808 while (size > 0) { 2809 /* Get the size for this page (i.e. partial or full page) */ 2810 psize = MIN(size, MMU_PAGESIZE); 2811 2812 if (buftype == DMA_OTYP_PAGES) { 2813 /* get the paddr from the page_t */ 2814 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2815 paddr = pfn_to_pa(pp->p_pagenum); 2816 pp = pp->p_next; 2817 } else if (pplist != NULL) { 2818 /* index into the array of page_t's to get the paddr */ 2819 ASSERT(!PP_ISFREE(pplist[pcnt])); 2820 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2821 pcnt++; 2822 } else { 2823 /* call into the VM to get the paddr */ 2824 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2825 vaddr)); 2826 vaddr += psize; 2827 } 2828 2829 #ifdef __xpv 2830 /* 2831 * If we're dom0, we're using a real device so we need to load 2832 * the cookies with MFNs instead of PFNs. 2833 */ 2834 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2835 #else 2836 raddr = paddr; 2837 #endif 2838 2839 /* 2840 * If we are using the copy buffer for anything over the 2841 * segment boundary, and this page is over the segment 2842 * boundary. 2843 * OR 2844 * if the DMA engine can't reach the physical address. 2845 */ 2846 if (((sglinfo->si_bounce_on_seg) && 2847 ((raddr + psize) > sglinfo->si_segmask)) || 2848 ((raddr < addrlo) || ((raddr + psize) > addrhi))) { 2849 2850 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2851 2852 /* 2853 * if there is something in the current cookie, go to 2854 * the next one. We only want one page in a cookie which 2855 * uses the copybuf since the copybuf doesn't have to 2856 * be physically contiguous. 2857 */ 2858 if (sgl[cnt].dmac_size != 0) { 2859 cnt++; 2860 } 2861 sgl[cnt].dmac_laddress = raddr; 2862 sgl[cnt].dmac_size = psize; 2863 #if defined(__amd64) 2864 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2865 #else 2866 /* 2867 * save the buf offset for 32-bit kernel. used in the 2868 * obsoleted interfaces. 2869 */ 2870 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2871 (dmar_object->dmao_size - size); 2872 #endif 2873 /* if this isn't the last cookie, go to the next one */ 2874 if ((cnt + 1) < sglinfo->si_max_pages) { 2875 cnt++; 2876 sgl[cnt].dmac_laddress = 0; 2877 sgl[cnt].dmac_size = 0; 2878 sgl[cnt].dmac_type = 0; 2879 } 2880 2881 /* 2882 * this page didn't need the copy buffer, if it's not physically 2883 * contiguous, or it would put us over a segment boundary, or it 2884 * puts us over the max cookie size, or the current sgl doesn't 2885 * have anything in it. 2886 */ 2887 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2888 !(raddr & sglinfo->si_segmask) || 2889 ((sgl[cnt].dmac_size + psize) > maxseg) || 2890 (sgl[cnt].dmac_size == 0)) { 2891 /* 2892 * if we're not already in a new cookie, go to the next 2893 * cookie. 2894 */ 2895 if (sgl[cnt].dmac_size != 0) { 2896 cnt++; 2897 } 2898 2899 /* save the cookie information */ 2900 sgl[cnt].dmac_laddress = raddr; 2901 sgl[cnt].dmac_size = psize; 2902 #if defined(__amd64) 2903 sgl[cnt].dmac_type = 0; 2904 #else 2905 /* 2906 * save the buf offset for 32-bit kernel. used in the 2907 * obsoleted interfaces. 2908 */ 2909 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2910 #endif 2911 2912 /* 2913 * this page didn't need the copy buffer, it is physically 2914 * contiguous with the last page, and it's <= the max cookie 2915 * size. 2916 */ 2917 } else { 2918 sgl[cnt].dmac_size += psize; 2919 2920 /* 2921 * if this exactly == the maximum cookie size, and 2922 * it isn't the last cookie, go to the next cookie. 2923 */ 2924 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2925 ((cnt + 1) < sglinfo->si_max_pages)) { 2926 cnt++; 2927 sgl[cnt].dmac_laddress = 0; 2928 sgl[cnt].dmac_size = 0; 2929 sgl[cnt].dmac_type = 0; 2930 } 2931 } 2932 2933 /* 2934 * save this page's physical address so we can figure out if the 2935 * next page is physically contiguous. Keep decrementing size 2936 * until we are done with the buffer. 2937 */ 2938 last_page = raddr; 2939 size -= psize; 2940 } 2941 2942 /* we're done, save away how many cookies the sgl has */ 2943 if (sgl[cnt].dmac_size == 0) { 2944 ASSERT(cnt < sglinfo->si_max_pages); 2945 sglinfo->si_sgl_size = cnt; 2946 } else { 2947 sglinfo->si_sgl_size = cnt + 1; 2948 } 2949 } 2950 2951 /* 2952 * rootnex_bind_slowpath() 2953 * Call in the bind path if the calling driver can't use the sgl without 2954 * modifying it. We either need to use the copy buffer and/or we will end up 2955 * with a partial bind. 2956 */ 2957 static int 2958 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2959 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2960 { 2961 rootnex_sglinfo_t *sinfo; 2962 rootnex_window_t *window; 2963 ddi_dma_cookie_t *cookie; 2964 size_t copybuf_used; 2965 size_t dmac_size; 2966 boolean_t partial; 2967 off_t cur_offset; 2968 page_t *cur_pp; 2969 major_t mnum; 2970 int e; 2971 int i; 2972 2973 2974 sinfo = &dma->dp_sglinfo; 2975 copybuf_used = 0; 2976 partial = B_FALSE; 2977 2978 /* 2979 * If we're using the copybuf, set the copybuf state in dma struct. 2980 * Needs to be first since it sets the copy buffer size. 2981 */ 2982 if (sinfo->si_copybuf_req != 0) { 2983 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2984 if (e != DDI_SUCCESS) { 2985 return (e); 2986 } 2987 } else { 2988 dma->dp_copybuf_size = 0; 2989 } 2990 2991 /* 2992 * Figure out if we need to do a partial mapping. If so, figure out 2993 * if we need to trim the buffers when we munge the sgl. 2994 */ 2995 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2996 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2997 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2998 dma->dp_partial_required = B_TRUE; 2999 if (attr->dma_attr_granular != 1) { 3000 dma->dp_trim_required = B_TRUE; 3001 } 3002 } else { 3003 dma->dp_partial_required = B_FALSE; 3004 dma->dp_trim_required = B_FALSE; 3005 } 3006 3007 /* If we need to do a partial bind, make sure the driver supports it */ 3008 if (dma->dp_partial_required && 3009 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 3010 3011 mnum = ddi_driver_major(dma->dp_dip); 3012 /* 3013 * patchable which allows us to print one warning per major 3014 * number. 3015 */ 3016 if ((rootnex_bind_warn) && 3017 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 3018 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 3019 cmn_err(CE_WARN, "!%s: coding error detected, the " 3020 "driver is using ddi_dma_attr(9S) incorrectly. " 3021 "There is a small risk of data corruption in " 3022 "particular with large I/Os. The driver should be " 3023 "replaced with a corrected version for proper " 3024 "system operation. To disable this warning, add " 3025 "'set rootnex:rootnex_bind_warn=0' to " 3026 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 3027 } 3028 return (DDI_DMA_TOOBIG); 3029 } 3030 3031 /* 3032 * we might need multiple windows, setup state to handle them. In this 3033 * code path, we will have at least one window. 3034 */ 3035 e = rootnex_setup_windows(hp, dma, attr, kmflag); 3036 if (e != DDI_SUCCESS) { 3037 rootnex_teardown_copybuf(dma); 3038 return (e); 3039 } 3040 3041 window = &dma->dp_window[0]; 3042 cookie = &dma->dp_cookies[0]; 3043 cur_offset = 0; 3044 rootnex_init_win(hp, dma, window, cookie, cur_offset); 3045 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 3046 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 3047 } 3048 3049 /* loop though all the cookies we got back from get_sgl() */ 3050 for (i = 0; i < sinfo->si_sgl_size; i++) { 3051 /* 3052 * If we're using the copy buffer, check this cookie and setup 3053 * its associated copy buffer state. If this cookie uses the 3054 * copy buffer, make sure we sync this window during dma_sync. 3055 */ 3056 if (dma->dp_copybuf_size > 0) { 3057 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 3058 cur_offset, ©buf_used, &cur_pp); 3059 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3060 window->wd_dosync = B_TRUE; 3061 } 3062 } 3063 3064 /* 3065 * save away the cookie size, since it could be modified in 3066 * the windowing code. 3067 */ 3068 dmac_size = cookie->dmac_size; 3069 3070 /* if we went over max copybuf size */ 3071 if (dma->dp_copybuf_size && 3072 (copybuf_used > dma->dp_copybuf_size)) { 3073 partial = B_TRUE; 3074 e = rootnex_copybuf_window_boundary(hp, dma, &window, 3075 cookie, cur_offset, ©buf_used); 3076 if (e != DDI_SUCCESS) { 3077 rootnex_teardown_copybuf(dma); 3078 rootnex_teardown_windows(dma); 3079 return (e); 3080 } 3081 3082 /* 3083 * if the coookie uses the copy buffer, make sure the 3084 * new window we just moved to is set to sync. 3085 */ 3086 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3087 window->wd_dosync = B_TRUE; 3088 } 3089 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 3090 dma->dp_dip); 3091 3092 /* if the cookie cnt == max sgllen, move to the next window */ 3093 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 3094 partial = B_TRUE; 3095 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 3096 e = rootnex_sgllen_window_boundary(hp, dma, &window, 3097 cookie, attr, cur_offset); 3098 if (e != DDI_SUCCESS) { 3099 rootnex_teardown_copybuf(dma); 3100 rootnex_teardown_windows(dma); 3101 return (e); 3102 } 3103 3104 /* 3105 * if the coookie uses the copy buffer, make sure the 3106 * new window we just moved to is set to sync. 3107 */ 3108 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3109 window->wd_dosync = B_TRUE; 3110 } 3111 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 3112 dma->dp_dip); 3113 3114 /* else if we will be over maxxfer */ 3115 } else if ((window->wd_size + dmac_size) > 3116 dma->dp_maxxfer) { 3117 partial = B_TRUE; 3118 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 3119 cookie); 3120 if (e != DDI_SUCCESS) { 3121 rootnex_teardown_copybuf(dma); 3122 rootnex_teardown_windows(dma); 3123 return (e); 3124 } 3125 3126 /* 3127 * if the coookie uses the copy buffer, make sure the 3128 * new window we just moved to is set to sync. 3129 */ 3130 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3131 window->wd_dosync = B_TRUE; 3132 } 3133 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 3134 dma->dp_dip); 3135 3136 /* else this cookie fits in the current window */ 3137 } else { 3138 window->wd_cookie_cnt++; 3139 window->wd_size += dmac_size; 3140 } 3141 3142 /* track our offset into the buffer, go to the next cookie */ 3143 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 3144 ASSERT(cookie->dmac_size <= dmac_size); 3145 cur_offset += dmac_size; 3146 cookie++; 3147 } 3148 3149 /* if we ended up with a zero sized window in the end, clean it up */ 3150 if (window->wd_size == 0) { 3151 hp->dmai_nwin--; 3152 window--; 3153 } 3154 3155 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 3156 3157 if (!partial) { 3158 return (DDI_DMA_MAPPED); 3159 } 3160 3161 ASSERT(dma->dp_partial_required); 3162 return (DDI_DMA_PARTIAL_MAP); 3163 } 3164 3165 3166 /* 3167 * rootnex_setup_copybuf() 3168 * Called in bind slowpath. Figures out if we're going to use the copy 3169 * buffer, and if we do, sets up the basic state to handle it. 3170 */ 3171 static int 3172 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 3173 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 3174 { 3175 rootnex_sglinfo_t *sinfo; 3176 ddi_dma_attr_t lattr; 3177 size_t max_copybuf; 3178 int cansleep; 3179 int e; 3180 #if !defined(__amd64) 3181 int vmflag; 3182 #endif 3183 3184 3185 sinfo = &dma->dp_sglinfo; 3186 3187 /* read this first so it's consistent through the routine */ 3188 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 3189 3190 /* We need to call into the rootnex on ddi_dma_sync() */ 3191 hp->dmai_rflags &= ~DMP_NOSYNC; 3192 3193 /* make sure the copybuf size <= the max size */ 3194 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 3195 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 3196 3197 #if !defined(__amd64) 3198 /* 3199 * if we don't have kva space to copy to/from, allocate the KVA space 3200 * now. We only do this for the 32-bit kernel. We use seg kpm space for 3201 * the 64-bit kernel. 3202 */ 3203 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 3204 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 3205 3206 /* convert the sleep flags */ 3207 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3208 vmflag = VM_SLEEP; 3209 } else { 3210 vmflag = VM_NOSLEEP; 3211 } 3212 3213 /* allocate Kernel VA space that we can bcopy to/from */ 3214 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 3215 vmflag); 3216 if (dma->dp_kva == NULL) { 3217 return (DDI_DMA_NORESOURCES); 3218 } 3219 } 3220 #endif 3221 3222 /* convert the sleep flags */ 3223 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 3224 cansleep = 1; 3225 } else { 3226 cansleep = 0; 3227 } 3228 3229 /* 3230 * Allocate the actual copy buffer. This needs to fit within the DMA 3231 * engine limits, so we can't use kmem_alloc... We don't need 3232 * contiguous memory (sgllen) since we will be forcing windows on 3233 * sgllen anyway. 3234 */ 3235 lattr = *attr; 3236 lattr.dma_attr_align = MMU_PAGESIZE; 3237 /* 3238 * this should be < 0 to indicate no limit, but due to a bug in 3239 * the rootnex, we'll set it to the maximum positive int. 3240 */ 3241 lattr.dma_attr_sgllen = 0x7fffffff; 3242 /* 3243 * if we're using the copy buffer because of seg, use that for our 3244 * upper address limit. 3245 */ 3246 if (sinfo->si_bounce_on_seg) { 3247 lattr.dma_attr_addr_hi = lattr.dma_attr_seg; 3248 } 3249 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 3250 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 3251 if (e != DDI_SUCCESS) { 3252 #if !defined(__amd64) 3253 if (dma->dp_kva != NULL) { 3254 vmem_free(heap_arena, dma->dp_kva, 3255 dma->dp_copybuf_size); 3256 } 3257 #endif 3258 return (DDI_DMA_NORESOURCES); 3259 } 3260 3261 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 3262 size_t, dma->dp_copybuf_size); 3263 3264 return (DDI_SUCCESS); 3265 } 3266 3267 3268 /* 3269 * rootnex_setup_windows() 3270 * Called in bind slowpath to setup the window state. We always have windows 3271 * in the slowpath. Even if the window count = 1. 3272 */ 3273 static int 3274 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3275 ddi_dma_attr_t *attr, int kmflag) 3276 { 3277 rootnex_window_t *windowp; 3278 rootnex_sglinfo_t *sinfo; 3279 size_t copy_state_size; 3280 size_t win_state_size; 3281 size_t state_available; 3282 size_t space_needed; 3283 uint_t copybuf_win; 3284 uint_t maxxfer_win; 3285 size_t space_used; 3286 uint_t sglwin; 3287 3288 3289 sinfo = &dma->dp_sglinfo; 3290 3291 dma->dp_current_win = 0; 3292 hp->dmai_nwin = 0; 3293 3294 /* If we don't need to do a partial, we only have one window */ 3295 if (!dma->dp_partial_required) { 3296 dma->dp_max_win = 1; 3297 3298 /* 3299 * we need multiple windows, need to figure out the worse case number 3300 * of windows. 3301 */ 3302 } else { 3303 /* 3304 * if we need windows because we need more copy buffer that 3305 * we allow, the worse case number of windows we could need 3306 * here would be (copybuf space required / copybuf space that 3307 * we have) plus one for remainder, and plus 2 to handle the 3308 * extra pages on the trim for the first and last pages of the 3309 * buffer (a page is the minimum window size so under the right 3310 * attr settings, you could have a window for each page). 3311 * The last page will only be hit here if the size is not a 3312 * multiple of the granularity (which theoretically shouldn't 3313 * be the case but never has been enforced, so we could have 3314 * broken things without it). 3315 */ 3316 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3317 ASSERT(dma->dp_copybuf_size > 0); 3318 copybuf_win = (sinfo->si_copybuf_req / 3319 dma->dp_copybuf_size) + 1 + 2; 3320 } else { 3321 copybuf_win = 0; 3322 } 3323 3324 /* 3325 * if we need windows because we have more cookies than the H/W 3326 * can handle, the number of windows we would need here would 3327 * be (cookie count / cookies count H/W supports minus 1[for 3328 * trim]) plus one for remainder. 3329 */ 3330 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3331 sglwin = (sinfo->si_sgl_size / 3332 (attr->dma_attr_sgllen - 1)) + 1; 3333 } else { 3334 sglwin = 0; 3335 } 3336 3337 /* 3338 * if we need windows because we're binding more memory than the 3339 * H/W can transfer at once, the number of windows we would need 3340 * here would be (xfer count / max xfer H/W supports) plus one 3341 * for remainder, and plus 2 to handle the extra pages on the 3342 * trim (see above comment about trim) 3343 */ 3344 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3345 maxxfer_win = (dma->dp_dma.dmao_size / 3346 dma->dp_maxxfer) + 1 + 2; 3347 } else { 3348 maxxfer_win = 0; 3349 } 3350 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3351 ASSERT(dma->dp_max_win > 0); 3352 } 3353 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3354 3355 /* 3356 * Get space for window and potential copy buffer state. Before we 3357 * go and allocate memory, see if we can get away with using what's 3358 * left in the pre-allocted state or the dynamically allocated sgl. 3359 */ 3360 space_used = (uintptr_t)(sinfo->si_sgl_size * 3361 sizeof (ddi_dma_cookie_t)); 3362 3363 /* if we dynamically allocated space for the cookies */ 3364 if (dma->dp_need_to_free_cookie) { 3365 /* if we have more space in the pre-allocted buffer, use it */ 3366 ASSERT(space_used <= dma->dp_cookie_size); 3367 if ((dma->dp_cookie_size - space_used) <= 3368 rootnex_state->r_prealloc_size) { 3369 state_available = rootnex_state->r_prealloc_size; 3370 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3371 3372 /* 3373 * else, we have more free space in the dynamically allocated 3374 * buffer, i.e. the buffer wasn't worse case fragmented so we 3375 * didn't need a lot of cookies. 3376 */ 3377 } else { 3378 state_available = dma->dp_cookie_size - space_used; 3379 windowp = (rootnex_window_t *) 3380 &dma->dp_cookies[sinfo->si_sgl_size]; 3381 } 3382 3383 /* we used the pre-alloced buffer */ 3384 } else { 3385 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3386 state_available = rootnex_state->r_prealloc_size - space_used; 3387 windowp = (rootnex_window_t *) 3388 &dma->dp_cookies[sinfo->si_sgl_size]; 3389 } 3390 3391 /* 3392 * figure out how much state we need to track the copy buffer. Add an 3393 * addition 8 bytes for pointer alignemnt later. 3394 */ 3395 if (dma->dp_copybuf_size > 0) { 3396 copy_state_size = sinfo->si_max_pages * 3397 sizeof (rootnex_pgmap_t); 3398 } else { 3399 copy_state_size = 0; 3400 } 3401 /* add an additional 8 bytes for pointer alignment */ 3402 space_needed = win_state_size + copy_state_size + 0x8; 3403 3404 /* if we have enough space already, use it */ 3405 if (state_available >= space_needed) { 3406 dma->dp_window = windowp; 3407 dma->dp_need_to_free_window = B_FALSE; 3408 3409 /* not enough space, need to allocate more. */ 3410 } else { 3411 dma->dp_window = kmem_alloc(space_needed, kmflag); 3412 if (dma->dp_window == NULL) { 3413 return (DDI_DMA_NORESOURCES); 3414 } 3415 dma->dp_need_to_free_window = B_TRUE; 3416 dma->dp_window_size = space_needed; 3417 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3418 dma->dp_dip, size_t, space_needed); 3419 } 3420 3421 /* 3422 * we allocate copy buffer state and window state at the same time. 3423 * setup our copy buffer state pointers. Make sure it's aligned. 3424 */ 3425 if (dma->dp_copybuf_size > 0) { 3426 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3427 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3428 3429 #if !defined(__amd64) 3430 /* 3431 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3432 * false/NULL. Should be quicker to bzero vs loop and set. 3433 */ 3434 bzero(dma->dp_pgmap, copy_state_size); 3435 #endif 3436 } else { 3437 dma->dp_pgmap = NULL; 3438 } 3439 3440 return (DDI_SUCCESS); 3441 } 3442 3443 3444 /* 3445 * rootnex_teardown_copybuf() 3446 * cleans up after rootnex_setup_copybuf() 3447 */ 3448 static void 3449 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3450 { 3451 #if !defined(__amd64) 3452 int i; 3453 3454 /* 3455 * if we allocated kernel heap VMEM space, go through all the pages and 3456 * map out any of the ones that we're mapped into the kernel heap VMEM 3457 * arena. Then free the VMEM space. 3458 */ 3459 if (dma->dp_kva != NULL) { 3460 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3461 if (dma->dp_pgmap[i].pm_mapped) { 3462 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3463 MMU_PAGESIZE, HAT_UNLOAD); 3464 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3465 } 3466 } 3467 3468 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3469 } 3470 3471 #endif 3472 3473 /* if we allocated a copy buffer, free it */ 3474 if (dma->dp_cbaddr != NULL) { 3475 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3476 } 3477 } 3478 3479 3480 /* 3481 * rootnex_teardown_windows() 3482 * cleans up after rootnex_setup_windows() 3483 */ 3484 static void 3485 rootnex_teardown_windows(rootnex_dma_t *dma) 3486 { 3487 /* 3488 * if we had to allocate window state on the last bind (because we 3489 * didn't have enough pre-allocated space in the handle), free it. 3490 */ 3491 if (dma->dp_need_to_free_window) { 3492 kmem_free(dma->dp_window, dma->dp_window_size); 3493 } 3494 } 3495 3496 3497 /* 3498 * rootnex_init_win() 3499 * Called in bind slow path during creation of a new window. Initializes 3500 * window state to default values. 3501 */ 3502 /*ARGSUSED*/ 3503 static void 3504 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3505 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3506 { 3507 hp->dmai_nwin++; 3508 window->wd_dosync = B_FALSE; 3509 window->wd_offset = cur_offset; 3510 window->wd_size = 0; 3511 window->wd_first_cookie = cookie; 3512 window->wd_cookie_cnt = 0; 3513 window->wd_trim.tr_trim_first = B_FALSE; 3514 window->wd_trim.tr_trim_last = B_FALSE; 3515 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3516 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3517 #if !defined(__amd64) 3518 window->wd_remap_copybuf = dma->dp_cb_remaping; 3519 #endif 3520 } 3521 3522 3523 /* 3524 * rootnex_setup_cookie() 3525 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3526 * the sgl uses the copy buffer, we need to go through each cookie, figure 3527 * out if it uses the copy buffer, and if it does, save away everything we'll 3528 * need during sync. 3529 */ 3530 static void 3531 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3532 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3533 page_t **cur_pp) 3534 { 3535 boolean_t copybuf_sz_power_2; 3536 rootnex_sglinfo_t *sinfo; 3537 paddr_t paddr; 3538 uint_t pidx; 3539 uint_t pcnt; 3540 off_t poff; 3541 #if defined(__amd64) 3542 pfn_t pfn; 3543 #else 3544 page_t **pplist; 3545 #endif 3546 3547 sinfo = &dma->dp_sglinfo; 3548 3549 /* 3550 * Calculate the page index relative to the start of the buffer. The 3551 * index to the current page for our buffer is the offset into the 3552 * first page of the buffer plus our current offset into the buffer 3553 * itself, shifted of course... 3554 */ 3555 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3556 ASSERT(pidx < sinfo->si_max_pages); 3557 3558 /* if this cookie uses the copy buffer */ 3559 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3560 /* 3561 * NOTE: we know that since this cookie uses the copy buffer, it 3562 * is <= MMU_PAGESIZE. 3563 */ 3564 3565 /* 3566 * get the offset into the page. For the 64-bit kernel, get the 3567 * pfn which we'll use with seg kpm. 3568 */ 3569 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3570 #if defined(__amd64) 3571 /* mfn_to_pfn() is a NOP on i86pc */ 3572 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3573 #endif /* __amd64 */ 3574 3575 /* figure out if the copybuf size is a power of 2 */ 3576 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3577 copybuf_sz_power_2 = B_FALSE; 3578 } else { 3579 copybuf_sz_power_2 = B_TRUE; 3580 } 3581 3582 /* This page uses the copy buffer */ 3583 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3584 3585 /* 3586 * save the copy buffer KVA that we'll use with this page. 3587 * if we still fit within the copybuf, it's a simple add. 3588 * otherwise, we need to wrap over using & or % accordingly. 3589 */ 3590 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3591 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3592 *copybuf_used; 3593 } else { 3594 if (copybuf_sz_power_2) { 3595 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3596 (uintptr_t)dma->dp_cbaddr + 3597 (*copybuf_used & 3598 (dma->dp_copybuf_size - 1))); 3599 } else { 3600 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3601 (uintptr_t)dma->dp_cbaddr + 3602 (*copybuf_used % dma->dp_copybuf_size)); 3603 } 3604 } 3605 3606 /* 3607 * over write the cookie physical address with the address of 3608 * the physical address of the copy buffer page that we will 3609 * use. 3610 */ 3611 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3612 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3613 3614 #ifdef __xpv 3615 /* 3616 * If we're dom0, we're using a real device so we need to load 3617 * the cookies with MAs instead of PAs. 3618 */ 3619 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3620 #else 3621 cookie->dmac_laddress = paddr; 3622 #endif 3623 3624 /* if we have a kernel VA, it's easy, just save that address */ 3625 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3626 (sinfo->si_asp == &kas)) { 3627 /* 3628 * save away the page aligned virtual address of the 3629 * driver buffer. Offsets are handled in the sync code. 3630 */ 3631 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3632 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3633 & MMU_PAGEMASK); 3634 #if !defined(__amd64) 3635 /* 3636 * we didn't need to, and will never need to map this 3637 * page. 3638 */ 3639 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3640 #endif 3641 3642 /* we don't have a kernel VA. We need one for the bcopy. */ 3643 } else { 3644 #if defined(__amd64) 3645 /* 3646 * for the 64-bit kernel, it's easy. We use seg kpm to 3647 * get a Kernel VA for the corresponding pfn. 3648 */ 3649 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3650 #else 3651 /* 3652 * for the 32-bit kernel, this is a pain. First we'll 3653 * save away the page_t or user VA for this page. This 3654 * is needed in rootnex_dma_win() when we switch to a 3655 * new window which requires us to re-map the copy 3656 * buffer. 3657 */ 3658 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3659 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3660 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3661 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3662 } else if (pplist != NULL) { 3663 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3664 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3665 } else { 3666 dma->dp_pgmap[pidx].pm_pp = NULL; 3667 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3668 (((uintptr_t) 3669 dmar_object->dmao_obj.virt_obj.v_addr + 3670 cur_offset) & MMU_PAGEMASK); 3671 } 3672 3673 /* 3674 * save away the page aligned virtual address which was 3675 * allocated from the kernel heap arena (taking into 3676 * account if we need more copy buffer than we alloced 3677 * and use multiple windows to handle this, i.e. &,%). 3678 * NOTE: there isn't and physical memory backing up this 3679 * virtual address space currently. 3680 */ 3681 if ((*copybuf_used + MMU_PAGESIZE) <= 3682 dma->dp_copybuf_size) { 3683 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3684 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3685 MMU_PAGEMASK); 3686 } else { 3687 if (copybuf_sz_power_2) { 3688 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3689 (((uintptr_t)dma->dp_kva + 3690 (*copybuf_used & 3691 (dma->dp_copybuf_size - 1))) & 3692 MMU_PAGEMASK); 3693 } else { 3694 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3695 (((uintptr_t)dma->dp_kva + 3696 (*copybuf_used % 3697 dma->dp_copybuf_size)) & 3698 MMU_PAGEMASK); 3699 } 3700 } 3701 3702 /* 3703 * if we haven't used up the available copy buffer yet, 3704 * map the kva to the physical page. 3705 */ 3706 if (!dma->dp_cb_remaping && ((*copybuf_used + 3707 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3708 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3709 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3710 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3711 dma->dp_pgmap[pidx].pm_kaddr); 3712 } else { 3713 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3714 sinfo->si_asp, 3715 dma->dp_pgmap[pidx].pm_kaddr); 3716 } 3717 3718 /* 3719 * we've used up the available copy buffer, this page 3720 * will have to be mapped during rootnex_dma_win() when 3721 * we switch to a new window which requires a re-map 3722 * the copy buffer. (32-bit kernel only) 3723 */ 3724 } else { 3725 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3726 } 3727 #endif 3728 /* go to the next page_t */ 3729 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3730 *cur_pp = (*cur_pp)->p_next; 3731 } 3732 } 3733 3734 /* add to the copy buffer count */ 3735 *copybuf_used += MMU_PAGESIZE; 3736 3737 /* 3738 * This cookie doesn't use the copy buffer. Walk through the pages this 3739 * cookie occupies to reflect this. 3740 */ 3741 } else { 3742 /* 3743 * figure out how many pages the cookie occupies. We need to 3744 * use the original page offset of the buffer and the cookies 3745 * offset in the buffer to do this. 3746 */ 3747 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3748 pcnt = mmu_btopr(cookie->dmac_size + poff); 3749 3750 while (pcnt > 0) { 3751 #if !defined(__amd64) 3752 /* 3753 * the 32-bit kernel doesn't have seg kpm, so we need 3754 * to map in the driver buffer (if it didn't come down 3755 * with a kernel VA) on the fly. Since this page doesn't 3756 * use the copy buffer, it's not, or will it ever, have 3757 * to be mapped in. 3758 */ 3759 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3760 #endif 3761 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3762 3763 /* 3764 * we need to update pidx and cur_pp or we'll loose 3765 * track of where we are. 3766 */ 3767 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3768 *cur_pp = (*cur_pp)->p_next; 3769 } 3770 pidx++; 3771 pcnt--; 3772 } 3773 } 3774 } 3775 3776 3777 /* 3778 * rootnex_sgllen_window_boundary() 3779 * Called in the bind slow path when the next cookie causes us to exceed (in 3780 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3781 * length supported by the DMA H/W. 3782 */ 3783 static int 3784 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3785 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3786 off_t cur_offset) 3787 { 3788 off_t new_offset; 3789 size_t trim_sz; 3790 off_t coffset; 3791 3792 3793 /* 3794 * if we know we'll never have to trim, it's pretty easy. Just move to 3795 * the next window and init it. We're done. 3796 */ 3797 if (!dma->dp_trim_required) { 3798 (*windowp)++; 3799 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3800 (*windowp)->wd_cookie_cnt++; 3801 (*windowp)->wd_size = cookie->dmac_size; 3802 return (DDI_SUCCESS); 3803 } 3804 3805 /* figure out how much we need to trim from the window */ 3806 ASSERT(attr->dma_attr_granular != 0); 3807 if (dma->dp_granularity_power_2) { 3808 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3809 } else { 3810 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3811 } 3812 3813 /* The window's a whole multiple of granularity. We're done */ 3814 if (trim_sz == 0) { 3815 (*windowp)++; 3816 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3817 (*windowp)->wd_cookie_cnt++; 3818 (*windowp)->wd_size = cookie->dmac_size; 3819 return (DDI_SUCCESS); 3820 } 3821 3822 /* 3823 * The window's not a whole multiple of granularity, since we know this 3824 * is due to the sgllen, we need to go back to the last cookie and trim 3825 * that one, add the left over part of the old cookie into the new 3826 * window, and then add in the new cookie into the new window. 3827 */ 3828 3829 /* 3830 * make sure the driver isn't making us do something bad... Trimming and 3831 * sgllen == 1 don't go together. 3832 */ 3833 if (attr->dma_attr_sgllen == 1) { 3834 return (DDI_DMA_NOMAPPING); 3835 } 3836 3837 /* 3838 * first, setup the current window to account for the trim. Need to go 3839 * back to the last cookie for this. 3840 */ 3841 cookie--; 3842 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3843 (*windowp)->wd_trim.tr_last_cookie = cookie; 3844 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3845 ASSERT(cookie->dmac_size > trim_sz); 3846 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3847 (*windowp)->wd_size -= trim_sz; 3848 3849 /* save the buffer offsets for the next window */ 3850 coffset = cookie->dmac_size - trim_sz; 3851 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3852 3853 /* 3854 * set this now in case this is the first window. all other cases are 3855 * set in dma_win() 3856 */ 3857 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3858 3859 /* 3860 * initialize the next window using what's left over in the previous 3861 * cookie. 3862 */ 3863 (*windowp)++; 3864 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3865 (*windowp)->wd_cookie_cnt++; 3866 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3867 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3868 (*windowp)->wd_trim.tr_first_size = trim_sz; 3869 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3870 (*windowp)->wd_dosync = B_TRUE; 3871 } 3872 3873 /* 3874 * now go back to the current cookie and add it to the new window. set 3875 * the new window size to the what was left over from the previous 3876 * cookie and what's in the current cookie. 3877 */ 3878 cookie++; 3879 (*windowp)->wd_cookie_cnt++; 3880 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3881 3882 /* 3883 * trim plus the next cookie could put us over maxxfer (a cookie can be 3884 * a max size of maxxfer). Handle that case. 3885 */ 3886 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3887 /* 3888 * maxxfer is already a whole multiple of granularity, and this 3889 * trim will be <= the previous trim (since a cookie can't be 3890 * larger than maxxfer). Make things simple here. 3891 */ 3892 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3893 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3894 (*windowp)->wd_trim.tr_last_cookie = cookie; 3895 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3896 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3897 (*windowp)->wd_size -= trim_sz; 3898 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3899 3900 /* save the buffer offsets for the next window */ 3901 coffset = cookie->dmac_size - trim_sz; 3902 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3903 3904 /* setup the next window */ 3905 (*windowp)++; 3906 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3907 (*windowp)->wd_cookie_cnt++; 3908 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3909 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3910 coffset; 3911 (*windowp)->wd_trim.tr_first_size = trim_sz; 3912 } 3913 3914 return (DDI_SUCCESS); 3915 } 3916 3917 3918 /* 3919 * rootnex_copybuf_window_boundary() 3920 * Called in bind slowpath when we get to a window boundary because we used 3921 * up all the copy buffer that we have. 3922 */ 3923 static int 3924 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3925 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3926 size_t *copybuf_used) 3927 { 3928 rootnex_sglinfo_t *sinfo; 3929 off_t new_offset; 3930 size_t trim_sz; 3931 paddr_t paddr; 3932 off_t coffset; 3933 uint_t pidx; 3934 off_t poff; 3935 3936 3937 sinfo = &dma->dp_sglinfo; 3938 3939 /* 3940 * the copy buffer should be a whole multiple of page size. We know that 3941 * this cookie is <= MMU_PAGESIZE. 3942 */ 3943 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3944 3945 /* 3946 * from now on, all new windows in this bind need to be re-mapped during 3947 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3948 * space... 3949 */ 3950 #if !defined(__amd64) 3951 dma->dp_cb_remaping = B_TRUE; 3952 #endif 3953 3954 /* reset copybuf used */ 3955 *copybuf_used = 0; 3956 3957 /* 3958 * if we don't have to trim (since granularity is set to 1), go to the 3959 * next window and add the current cookie to it. We know the current 3960 * cookie uses the copy buffer since we're in this code path. 3961 */ 3962 if (!dma->dp_trim_required) { 3963 (*windowp)++; 3964 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3965 3966 /* Add this cookie to the new window */ 3967 (*windowp)->wd_cookie_cnt++; 3968 (*windowp)->wd_size += cookie->dmac_size; 3969 *copybuf_used += MMU_PAGESIZE; 3970 return (DDI_SUCCESS); 3971 } 3972 3973 /* 3974 * *** may need to trim, figure it out. 3975 */ 3976 3977 /* figure out how much we need to trim from the window */ 3978 if (dma->dp_granularity_power_2) { 3979 trim_sz = (*windowp)->wd_size & 3980 (hp->dmai_attr.dma_attr_granular - 1); 3981 } else { 3982 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3983 } 3984 3985 /* 3986 * if the window's a whole multiple of granularity, go to the next 3987 * window, init it, then add in the current cookie. We know the current 3988 * cookie uses the copy buffer since we're in this code path. 3989 */ 3990 if (trim_sz == 0) { 3991 (*windowp)++; 3992 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3993 3994 /* Add this cookie to the new window */ 3995 (*windowp)->wd_cookie_cnt++; 3996 (*windowp)->wd_size += cookie->dmac_size; 3997 *copybuf_used += MMU_PAGESIZE; 3998 return (DDI_SUCCESS); 3999 } 4000 4001 /* 4002 * *** We figured it out, we definitly need to trim 4003 */ 4004 4005 /* 4006 * make sure the driver isn't making us do something bad... 4007 * Trimming and sgllen == 1 don't go together. 4008 */ 4009 if (hp->dmai_attr.dma_attr_sgllen == 1) { 4010 return (DDI_DMA_NOMAPPING); 4011 } 4012 4013 /* 4014 * first, setup the current window to account for the trim. Need to go 4015 * back to the last cookie for this. Some of the last cookie will be in 4016 * the current window, and some of the last cookie will be in the new 4017 * window. All of the current cookie will be in the new window. 4018 */ 4019 cookie--; 4020 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4021 (*windowp)->wd_trim.tr_last_cookie = cookie; 4022 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4023 ASSERT(cookie->dmac_size > trim_sz); 4024 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4025 (*windowp)->wd_size -= trim_sz; 4026 4027 /* 4028 * we're trimming the last cookie (not the current cookie). So that 4029 * last cookie may have or may not have been using the copy buffer ( 4030 * we know the cookie passed in uses the copy buffer since we're in 4031 * this code path). 4032 * 4033 * If the last cookie doesn't use the copy buffer, nothing special to 4034 * do. However, if it does uses the copy buffer, it will be both the 4035 * last page in the current window and the first page in the next 4036 * window. Since we are reusing the copy buffer (and KVA space on the 4037 * 32-bit kernel), this page will use the end of the copy buffer in the 4038 * current window, and the start of the copy buffer in the next window. 4039 * Track that info... The cookie physical address was already set to 4040 * the copy buffer physical address in setup_cookie.. 4041 */ 4042 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4043 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 4044 (*windowp)->wd_size) >> MMU_PAGESHIFT; 4045 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 4046 (*windowp)->wd_trim.tr_last_pidx = pidx; 4047 (*windowp)->wd_trim.tr_last_cbaddr = 4048 dma->dp_pgmap[pidx].pm_cbaddr; 4049 #if !defined(__amd64) 4050 (*windowp)->wd_trim.tr_last_kaddr = 4051 dma->dp_pgmap[pidx].pm_kaddr; 4052 #endif 4053 } 4054 4055 /* save the buffer offsets for the next window */ 4056 coffset = cookie->dmac_size - trim_sz; 4057 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4058 4059 /* 4060 * set this now in case this is the first window. all other cases are 4061 * set in dma_win() 4062 */ 4063 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4064 4065 /* 4066 * initialize the next window using what's left over in the previous 4067 * cookie. 4068 */ 4069 (*windowp)++; 4070 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4071 (*windowp)->wd_cookie_cnt++; 4072 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4073 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 4074 (*windowp)->wd_trim.tr_first_size = trim_sz; 4075 4076 /* 4077 * again, we're tracking if the last cookie uses the copy buffer. 4078 * read the comment above for more info on why we need to track 4079 * additional state. 4080 * 4081 * For the first cookie in the new window, we need reset the physical 4082 * address to DMA into to the start of the copy buffer plus any 4083 * initial page offset which may be present. 4084 */ 4085 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 4086 (*windowp)->wd_dosync = B_TRUE; 4087 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 4088 (*windowp)->wd_trim.tr_first_pidx = pidx; 4089 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 4090 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 4091 4092 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 4093 poff; 4094 #ifdef __xpv 4095 /* 4096 * If we're dom0, we're using a real device so we need to load 4097 * the cookies with MAs instead of PAs. 4098 */ 4099 (*windowp)->wd_trim.tr_first_paddr = 4100 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 4101 #else 4102 (*windowp)->wd_trim.tr_first_paddr = paddr; 4103 #endif 4104 4105 #if !defined(__amd64) 4106 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 4107 #endif 4108 /* account for the cookie copybuf usage in the new window */ 4109 *copybuf_used += MMU_PAGESIZE; 4110 4111 /* 4112 * every piece of code has to have a hack, and here is this 4113 * ones :-) 4114 * 4115 * There is a complex interaction between setup_cookie and the 4116 * copybuf window boundary. The complexity had to be in either 4117 * the maxxfer window, or the copybuf window, and I chose the 4118 * copybuf code. 4119 * 4120 * So in this code path, we have taken the last cookie, 4121 * virtually broken it in half due to the trim, and it happens 4122 * to use the copybuf which further complicates life. At the 4123 * same time, we have already setup the current cookie, which 4124 * is now wrong. More background info: the current cookie uses 4125 * the copybuf, so it is only a page long max. So we need to 4126 * fix the current cookies copy buffer address, physical 4127 * address, and kva for the 32-bit kernel. We due this by 4128 * bumping them by page size (of course, we can't due this on 4129 * the physical address since the copy buffer may not be 4130 * physically contiguous). 4131 */ 4132 cookie++; 4133 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 4134 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 4135 4136 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 4137 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 4138 #ifdef __xpv 4139 /* 4140 * If we're dom0, we're using a real device so we need to load 4141 * the cookies with MAs instead of PAs. 4142 */ 4143 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 4144 #else 4145 cookie->dmac_laddress = paddr; 4146 #endif 4147 4148 #if !defined(__amd64) 4149 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 4150 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 4151 #endif 4152 } else { 4153 /* go back to the current cookie */ 4154 cookie++; 4155 } 4156 4157 /* 4158 * add the current cookie to the new window. set the new window size to 4159 * the what was left over from the previous cookie and what's in the 4160 * current cookie. 4161 */ 4162 (*windowp)->wd_cookie_cnt++; 4163 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 4164 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 4165 4166 /* 4167 * we know that the cookie passed in always uses the copy buffer. We 4168 * wouldn't be here if it didn't. 4169 */ 4170 *copybuf_used += MMU_PAGESIZE; 4171 4172 return (DDI_SUCCESS); 4173 } 4174 4175 4176 /* 4177 * rootnex_maxxfer_window_boundary() 4178 * Called in bind slowpath when we get to a window boundary because we will 4179 * go over maxxfer. 4180 */ 4181 static int 4182 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 4183 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 4184 { 4185 size_t dmac_size; 4186 off_t new_offset; 4187 size_t trim_sz; 4188 off_t coffset; 4189 4190 4191 /* 4192 * calculate how much we have to trim off of the current cookie to equal 4193 * maxxfer. We don't have to account for granularity here since our 4194 * maxxfer already takes that into account. 4195 */ 4196 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 4197 ASSERT(trim_sz <= cookie->dmac_size); 4198 ASSERT(trim_sz <= dma->dp_maxxfer); 4199 4200 /* save cookie size since we need it later and we might change it */ 4201 dmac_size = cookie->dmac_size; 4202 4203 /* 4204 * if we're not trimming the entire cookie, setup the current window to 4205 * account for the trim. 4206 */ 4207 if (trim_sz < cookie->dmac_size) { 4208 (*windowp)->wd_cookie_cnt++; 4209 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 4210 (*windowp)->wd_trim.tr_last_cookie = cookie; 4211 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 4212 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 4213 (*windowp)->wd_size = dma->dp_maxxfer; 4214 4215 /* 4216 * set the adjusted cookie size now in case this is the first 4217 * window. All other windows are taken care of in get win 4218 */ 4219 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 4220 } 4221 4222 /* 4223 * coffset is the current offset within the cookie, new_offset is the 4224 * current offset with the entire buffer. 4225 */ 4226 coffset = dmac_size - trim_sz; 4227 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 4228 4229 /* initialize the next window */ 4230 (*windowp)++; 4231 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 4232 (*windowp)->wd_cookie_cnt++; 4233 (*windowp)->wd_size = trim_sz; 4234 if (trim_sz < dmac_size) { 4235 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 4236 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 4237 coffset; 4238 (*windowp)->wd_trim.tr_first_size = trim_sz; 4239 } 4240 4241 return (DDI_SUCCESS); 4242 } 4243 4244 4245 /*ARGSUSED*/ 4246 static int 4247 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4248 off_t off, size_t len, uint_t cache_flags) 4249 { 4250 rootnex_sglinfo_t *sinfo; 4251 rootnex_pgmap_t *cbpage; 4252 rootnex_window_t *win; 4253 ddi_dma_impl_t *hp; 4254 rootnex_dma_t *dma; 4255 caddr_t fromaddr; 4256 caddr_t toaddr; 4257 uint_t psize; 4258 off_t offset; 4259 uint_t pidx; 4260 size_t size; 4261 off_t poff; 4262 int e; 4263 4264 4265 hp = (ddi_dma_impl_t *)handle; 4266 dma = (rootnex_dma_t *)hp->dmai_private; 4267 sinfo = &dma->dp_sglinfo; 4268 4269 /* 4270 * if we don't have any windows, we don't need to sync. A copybuf 4271 * will cause us to have at least one window. 4272 */ 4273 if (dma->dp_window == NULL) { 4274 return (DDI_SUCCESS); 4275 } 4276 4277 /* This window may not need to be sync'd */ 4278 win = &dma->dp_window[dma->dp_current_win]; 4279 if (!win->wd_dosync) { 4280 return (DDI_SUCCESS); 4281 } 4282 4283 /* handle off and len special cases */ 4284 if ((off == 0) || (rootnex_sync_ignore_params)) { 4285 offset = win->wd_offset; 4286 } else { 4287 offset = off; 4288 } 4289 if ((len == 0) || (rootnex_sync_ignore_params)) { 4290 size = win->wd_size; 4291 } else { 4292 size = len; 4293 } 4294 4295 /* check the sync args to make sure they make a little sense */ 4296 if (rootnex_sync_check_parms) { 4297 e = rootnex_valid_sync_parms(hp, win, offset, size, 4298 cache_flags); 4299 if (e != DDI_SUCCESS) { 4300 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4301 return (DDI_FAILURE); 4302 } 4303 } 4304 4305 /* 4306 * special case the first page to handle the offset into the page. The 4307 * offset to the current page for our buffer is the offset into the 4308 * first page of the buffer plus our current offset into the buffer 4309 * itself, masked of course. 4310 */ 4311 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4312 psize = MIN((MMU_PAGESIZE - poff), size); 4313 4314 /* go through all the pages that we want to sync */ 4315 while (size > 0) { 4316 /* 4317 * Calculate the page index relative to the start of the buffer. 4318 * The index to the current page for our buffer is the offset 4319 * into the first page of the buffer plus our current offset 4320 * into the buffer itself, shifted of course... 4321 */ 4322 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4323 ASSERT(pidx < sinfo->si_max_pages); 4324 4325 /* 4326 * if this page uses the copy buffer, we need to sync it, 4327 * otherwise, go on to the next page. 4328 */ 4329 cbpage = &dma->dp_pgmap[pidx]; 4330 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4331 (cbpage->pm_uses_copybuf == B_FALSE)); 4332 if (cbpage->pm_uses_copybuf) { 4333 /* cbaddr and kaddr should be page aligned */ 4334 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4335 MMU_PAGEOFFSET) == 0); 4336 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4337 MMU_PAGEOFFSET) == 0); 4338 4339 /* 4340 * if we're copying for the device, we are going to 4341 * copy from the drivers buffer and to the rootnex 4342 * allocated copy buffer. 4343 */ 4344 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4345 fromaddr = cbpage->pm_kaddr + poff; 4346 toaddr = cbpage->pm_cbaddr + poff; 4347 DTRACE_PROBE2(rootnex__sync__dev, 4348 dev_info_t *, dma->dp_dip, size_t, psize); 4349 4350 /* 4351 * if we're copying for the cpu/kernel, we are going to 4352 * copy from the rootnex allocated copy buffer to the 4353 * drivers buffer. 4354 */ 4355 } else { 4356 fromaddr = cbpage->pm_cbaddr + poff; 4357 toaddr = cbpage->pm_kaddr + poff; 4358 DTRACE_PROBE2(rootnex__sync__cpu, 4359 dev_info_t *, dma->dp_dip, size_t, psize); 4360 } 4361 4362 bcopy(fromaddr, toaddr, psize); 4363 } 4364 4365 /* 4366 * decrement size until we're done, update our offset into the 4367 * buffer, and get the next page size. 4368 */ 4369 size -= psize; 4370 offset += psize; 4371 psize = MIN(MMU_PAGESIZE, size); 4372 4373 /* page offset is zero for the rest of this loop */ 4374 poff = 0; 4375 } 4376 4377 return (DDI_SUCCESS); 4378 } 4379 4380 /* 4381 * rootnex_dma_sync() 4382 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4383 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4384 * is set, ddi_dma_sync() returns immediately passing back success. 4385 */ 4386 /*ARGSUSED*/ 4387 static int 4388 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4389 off_t off, size_t len, uint_t cache_flags) 4390 { 4391 #if defined(__amd64) && !defined(__xpv) 4392 if (IOMMU_USED(rdip)) { 4393 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4394 cache_flags)); 4395 } 4396 #endif 4397 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4398 cache_flags)); 4399 } 4400 4401 /* 4402 * rootnex_valid_sync_parms() 4403 * checks the parameters passed to sync to verify they are correct. 4404 */ 4405 static int 4406 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4407 off_t offset, size_t size, uint_t cache_flags) 4408 { 4409 off_t woffset; 4410 4411 4412 /* 4413 * the first part of the test to make sure the offset passed in is 4414 * within the window. 4415 */ 4416 if (offset < win->wd_offset) { 4417 return (DDI_FAILURE); 4418 } 4419 4420 /* 4421 * second and last part of the test to make sure the offset and length 4422 * passed in is within the window. 4423 */ 4424 woffset = offset - win->wd_offset; 4425 if ((woffset + size) > win->wd_size) { 4426 return (DDI_FAILURE); 4427 } 4428 4429 /* 4430 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4431 * be set too. 4432 */ 4433 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4434 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4435 return (DDI_SUCCESS); 4436 } 4437 4438 /* 4439 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4440 * should be set. Also DDI_DMA_READ should be set in the flags. 4441 */ 4442 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4443 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4444 (hp->dmai_rflags & DDI_DMA_READ)) { 4445 return (DDI_SUCCESS); 4446 } 4447 4448 return (DDI_FAILURE); 4449 } 4450 4451 4452 /*ARGSUSED*/ 4453 static int 4454 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4455 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4456 uint_t *ccountp) 4457 { 4458 rootnex_window_t *window; 4459 rootnex_trim_t *trim; 4460 ddi_dma_impl_t *hp; 4461 rootnex_dma_t *dma; 4462 #if !defined(__amd64) 4463 rootnex_sglinfo_t *sinfo; 4464 rootnex_pgmap_t *pmap; 4465 uint_t pidx; 4466 uint_t pcnt; 4467 off_t poff; 4468 int i; 4469 #endif 4470 4471 4472 hp = (ddi_dma_impl_t *)handle; 4473 dma = (rootnex_dma_t *)hp->dmai_private; 4474 #if !defined(__amd64) 4475 sinfo = &dma->dp_sglinfo; 4476 #endif 4477 4478 /* If we try and get a window which doesn't exist, return failure */ 4479 if (win >= hp->dmai_nwin) { 4480 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4481 return (DDI_FAILURE); 4482 } 4483 4484 /* 4485 * if we don't have any windows, and they're asking for the first 4486 * window, setup the cookie pointer to the first cookie in the bind. 4487 * setup our return values, then increment the cookie since we return 4488 * the first cookie on the stack. 4489 */ 4490 if (dma->dp_window == NULL) { 4491 if (win != 0) { 4492 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4493 return (DDI_FAILURE); 4494 } 4495 hp->dmai_cookie = dma->dp_cookies; 4496 *offp = 0; 4497 *lenp = dma->dp_dma.dmao_size; 4498 *ccountp = dma->dp_sglinfo.si_sgl_size; 4499 *cookiep = hp->dmai_cookie[0]; 4500 hp->dmai_cookie++; 4501 return (DDI_SUCCESS); 4502 } 4503 4504 /* sync the old window before moving on to the new one */ 4505 window = &dma->dp_window[dma->dp_current_win]; 4506 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4507 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4508 DDI_DMA_SYNC_FORCPU); 4509 } 4510 4511 #if !defined(__amd64) 4512 /* 4513 * before we move to the next window, if we need to re-map, unmap all 4514 * the pages in this window. 4515 */ 4516 if (dma->dp_cb_remaping) { 4517 /* 4518 * If we switch to this window again, we'll need to map in 4519 * on the fly next time. 4520 */ 4521 window->wd_remap_copybuf = B_TRUE; 4522 4523 /* 4524 * calculate the page index into the buffer where this window 4525 * starts, and the number of pages this window takes up. 4526 */ 4527 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4528 MMU_PAGESHIFT; 4529 poff = (sinfo->si_buf_offset + window->wd_offset) & 4530 MMU_PAGEOFFSET; 4531 pcnt = mmu_btopr(window->wd_size + poff); 4532 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4533 4534 /* unmap pages which are currently mapped in this window */ 4535 for (i = 0; i < pcnt; i++) { 4536 if (dma->dp_pgmap[pidx].pm_mapped) { 4537 hat_unload(kas.a_hat, 4538 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4539 HAT_UNLOAD); 4540 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4541 } 4542 pidx++; 4543 } 4544 } 4545 #endif 4546 4547 /* 4548 * Move to the new window. 4549 * NOTE: current_win must be set for sync to work right 4550 */ 4551 dma->dp_current_win = win; 4552 window = &dma->dp_window[win]; 4553 4554 /* if needed, adjust the first and/or last cookies for trim */ 4555 trim = &window->wd_trim; 4556 if (trim->tr_trim_first) { 4557 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4558 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4559 #if !defined(__amd64) 4560 window->wd_first_cookie->dmac_type = 4561 (window->wd_first_cookie->dmac_type & 4562 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4563 #endif 4564 if (trim->tr_first_copybuf_win) { 4565 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4566 trim->tr_first_cbaddr; 4567 #if !defined(__amd64) 4568 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4569 trim->tr_first_kaddr; 4570 #endif 4571 } 4572 } 4573 if (trim->tr_trim_last) { 4574 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4575 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4576 if (trim->tr_last_copybuf_win) { 4577 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4578 trim->tr_last_cbaddr; 4579 #if !defined(__amd64) 4580 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4581 trim->tr_last_kaddr; 4582 #endif 4583 } 4584 } 4585 4586 /* 4587 * setup the cookie pointer to the first cookie in the window. setup 4588 * our return values, then increment the cookie since we return the 4589 * first cookie on the stack. 4590 */ 4591 hp->dmai_cookie = window->wd_first_cookie; 4592 *offp = window->wd_offset; 4593 *lenp = window->wd_size; 4594 *ccountp = window->wd_cookie_cnt; 4595 *cookiep = hp->dmai_cookie[0]; 4596 hp->dmai_cookie++; 4597 4598 #if !defined(__amd64) 4599 /* re-map copybuf if required for this window */ 4600 if (dma->dp_cb_remaping) { 4601 /* 4602 * calculate the page index into the buffer where this 4603 * window starts. 4604 */ 4605 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4606 MMU_PAGESHIFT; 4607 ASSERT(pidx < sinfo->si_max_pages); 4608 4609 /* 4610 * the first page can get unmapped if it's shared with the 4611 * previous window. Even if the rest of this window is already 4612 * mapped in, we need to still check this one. 4613 */ 4614 pmap = &dma->dp_pgmap[pidx]; 4615 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4616 if (pmap->pm_pp != NULL) { 4617 pmap->pm_mapped = B_TRUE; 4618 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4619 } else if (pmap->pm_vaddr != NULL) { 4620 pmap->pm_mapped = B_TRUE; 4621 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4622 pmap->pm_kaddr); 4623 } 4624 } 4625 pidx++; 4626 4627 /* map in the rest of the pages if required */ 4628 if (window->wd_remap_copybuf) { 4629 window->wd_remap_copybuf = B_FALSE; 4630 4631 /* figure out many pages this window takes up */ 4632 poff = (sinfo->si_buf_offset + window->wd_offset) & 4633 MMU_PAGEOFFSET; 4634 pcnt = mmu_btopr(window->wd_size + poff); 4635 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4636 4637 /* map pages which require it */ 4638 for (i = 1; i < pcnt; i++) { 4639 pmap = &dma->dp_pgmap[pidx]; 4640 if (pmap->pm_uses_copybuf) { 4641 ASSERT(pmap->pm_mapped == B_FALSE); 4642 if (pmap->pm_pp != NULL) { 4643 pmap->pm_mapped = B_TRUE; 4644 i86_pp_map(pmap->pm_pp, 4645 pmap->pm_kaddr); 4646 } else if (pmap->pm_vaddr != NULL) { 4647 pmap->pm_mapped = B_TRUE; 4648 i86_va_map(pmap->pm_vaddr, 4649 sinfo->si_asp, 4650 pmap->pm_kaddr); 4651 } 4652 } 4653 pidx++; 4654 } 4655 } 4656 } 4657 #endif 4658 4659 /* if the new window uses the copy buffer, sync it for the device */ 4660 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4661 (void) rootnex_coredma_sync(dip, rdip, handle, 0, 0, 4662 DDI_DMA_SYNC_FORDEV); 4663 } 4664 4665 return (DDI_SUCCESS); 4666 } 4667 4668 /* 4669 * rootnex_dma_win() 4670 * called from ddi_dma_getwin() 4671 */ 4672 /*ARGSUSED*/ 4673 static int 4674 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4675 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4676 uint_t *ccountp) 4677 { 4678 #if defined(__amd64) && !defined(__xpv) 4679 if (IOMMU_USED(rdip)) { 4680 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4681 cookiep, ccountp)); 4682 } 4683 #endif 4684 4685 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4686 cookiep, ccountp)); 4687 } 4688 4689 /* 4690 * ************************ 4691 * obsoleted dma routines 4692 * ************************ 4693 */ 4694 4695 /* 4696 * rootnex_dma_map() 4697 * called from ddi_dma_setup() 4698 * NO IOMMU in 32 bit mode. The below routines doesn't work in 64 bit mode. 4699 */ 4700 /* ARGSUSED */ 4701 static int 4702 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4703 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4704 { 4705 #if defined(__amd64) 4706 /* 4707 * this interface is not supported in 64-bit x86 kernel. See comment in 4708 * rootnex_dma_mctl() 4709 */ 4710 return (DDI_DMA_NORESOURCES); 4711 4712 #else /* 32-bit x86 kernel */ 4713 ddi_dma_handle_t *lhandlep; 4714 ddi_dma_handle_t lhandle; 4715 ddi_dma_cookie_t cookie; 4716 ddi_dma_attr_t dma_attr; 4717 ddi_dma_lim_t *dma_lim; 4718 uint_t ccnt; 4719 int e; 4720 4721 4722 /* 4723 * if the driver is just testing to see if it's possible to do the bind, 4724 * we'll use local state. Otherwise, use the handle pointer passed in. 4725 */ 4726 if (handlep == NULL) { 4727 lhandlep = &lhandle; 4728 } else { 4729 lhandlep = handlep; 4730 } 4731 4732 /* convert the limit structure to a dma_attr one */ 4733 dma_lim = dmareq->dmar_limits; 4734 dma_attr.dma_attr_version = DMA_ATTR_V0; 4735 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4736 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4737 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4738 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4739 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4740 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4741 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4742 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4743 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4744 dma_attr.dma_attr_align = MMU_PAGESIZE; 4745 dma_attr.dma_attr_flags = 0; 4746 4747 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4748 dmareq->dmar_arg, lhandlep); 4749 if (e != DDI_SUCCESS) { 4750 return (e); 4751 } 4752 4753 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4754 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4755 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4756 return (e); 4757 } 4758 4759 /* 4760 * if the driver is just testing to see if it's possible to do the bind, 4761 * free up the local state and return the result. 4762 */ 4763 if (handlep == NULL) { 4764 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4765 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4766 if (e == DDI_DMA_MAPPED) { 4767 return (DDI_DMA_MAPOK); 4768 } else { 4769 return (DDI_DMA_NOMAPPING); 4770 } 4771 } 4772 4773 return (e); 4774 #endif /* defined(__amd64) */ 4775 } 4776 4777 /* 4778 * rootnex_dma_mctl() 4779 * 4780 * No IOMMU in 32 bit mode. The below routine doesn't work in 64 bit mode. 4781 */ 4782 /* ARGSUSED */ 4783 static int 4784 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4785 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4786 uint_t cache_flags) 4787 { 4788 #if defined(__amd64) 4789 /* 4790 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4791 * common implementation in genunix, so they no longer have x86 4792 * specific functionality which called into dma_ctl. 4793 * 4794 * The rest of the obsoleted interfaces were never supported in the 4795 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4796 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4797 * implementation issues. 4798 * 4799 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4800 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4801 * reflect that now too... 4802 * 4803 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4804 * not going to put this functionality into the 64-bit x86 kernel now. 4805 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4806 * that in a future release. 4807 */ 4808 return (DDI_FAILURE); 4809 4810 #else /* 32-bit x86 kernel */ 4811 ddi_dma_cookie_t lcookie; 4812 ddi_dma_cookie_t *cookie; 4813 rootnex_window_t *window; 4814 ddi_dma_impl_t *hp; 4815 rootnex_dma_t *dma; 4816 uint_t nwin; 4817 uint_t ccnt; 4818 size_t len; 4819 off_t off; 4820 int e; 4821 4822 4823 /* 4824 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4825 * hacky since were optimizing for the current interfaces and so we can 4826 * cleanup the mess in genunix. Hopefully we will remove the this 4827 * obsoleted routines someday soon. 4828 */ 4829 4830 switch (request) { 4831 4832 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4833 hp = (ddi_dma_impl_t *)handle; 4834 cookie = (ddi_dma_cookie_t *)objpp; 4835 4836 /* 4837 * convert segment to cookie. We don't distinguish between the 4838 * two :-) 4839 */ 4840 *cookie = *hp->dmai_cookie; 4841 *lenp = cookie->dmac_size; 4842 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4843 return (DDI_SUCCESS); 4844 4845 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4846 hp = (ddi_dma_impl_t *)handle; 4847 dma = (rootnex_dma_t *)hp->dmai_private; 4848 4849 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4850 return (DDI_DMA_STALE); 4851 } 4852 4853 /* handle the case where we don't have any windows */ 4854 if (dma->dp_window == NULL) { 4855 /* 4856 * if seg == NULL, and we don't have any windows, 4857 * return the first cookie in the sgl. 4858 */ 4859 if (*lenp == NULL) { 4860 dma->dp_current_cookie = 0; 4861 hp->dmai_cookie = dma->dp_cookies; 4862 *objpp = (caddr_t)handle; 4863 return (DDI_SUCCESS); 4864 4865 /* if we have more cookies, go to the next cookie */ 4866 } else { 4867 if ((dma->dp_current_cookie + 1) >= 4868 dma->dp_sglinfo.si_sgl_size) { 4869 return (DDI_DMA_DONE); 4870 } 4871 dma->dp_current_cookie++; 4872 hp->dmai_cookie++; 4873 return (DDI_SUCCESS); 4874 } 4875 } 4876 4877 /* We have one or more windows */ 4878 window = &dma->dp_window[dma->dp_current_win]; 4879 4880 /* 4881 * if seg == NULL, return the first cookie in the current 4882 * window 4883 */ 4884 if (*lenp == NULL) { 4885 dma->dp_current_cookie = 0; 4886 hp->dmai_cookie = window->wd_first_cookie; 4887 4888 /* 4889 * go to the next cookie in the window then see if we done with 4890 * this window. 4891 */ 4892 } else { 4893 if ((dma->dp_current_cookie + 1) >= 4894 window->wd_cookie_cnt) { 4895 return (DDI_DMA_DONE); 4896 } 4897 dma->dp_current_cookie++; 4898 hp->dmai_cookie++; 4899 } 4900 *objpp = (caddr_t)handle; 4901 return (DDI_SUCCESS); 4902 4903 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4904 hp = (ddi_dma_impl_t *)handle; 4905 dma = (rootnex_dma_t *)hp->dmai_private; 4906 4907 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4908 return (DDI_DMA_STALE); 4909 } 4910 4911 /* if win == NULL, return the first window in the bind */ 4912 if (*offp == NULL) { 4913 nwin = 0; 4914 4915 /* 4916 * else, go to the next window then see if we're done with all 4917 * the windows. 4918 */ 4919 } else { 4920 nwin = dma->dp_current_win + 1; 4921 if (nwin >= hp->dmai_nwin) { 4922 return (DDI_DMA_DONE); 4923 } 4924 } 4925 4926 /* switch to the next window */ 4927 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4928 &lcookie, &ccnt); 4929 ASSERT(e == DDI_SUCCESS); 4930 if (e != DDI_SUCCESS) { 4931 return (DDI_DMA_STALE); 4932 } 4933 4934 /* reset the cookie back to the first cookie in the window */ 4935 if (dma->dp_window != NULL) { 4936 window = &dma->dp_window[dma->dp_current_win]; 4937 hp->dmai_cookie = window->wd_first_cookie; 4938 } else { 4939 hp->dmai_cookie = dma->dp_cookies; 4940 } 4941 4942 *objpp = (caddr_t)handle; 4943 return (DDI_SUCCESS); 4944 4945 case DDI_DMA_FREE: /* ddi_dma_free() */ 4946 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4947 (void) rootnex_dma_freehdl(dip, rdip, handle); 4948 if (rootnex_state->r_dvma_call_list_id) { 4949 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4950 } 4951 return (DDI_SUCCESS); 4952 4953 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4954 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4955 /* should never get here, handled in genunix */ 4956 ASSERT(0); 4957 return (DDI_FAILURE); 4958 4959 case DDI_DMA_KVADDR: 4960 case DDI_DMA_GETERR: 4961 case DDI_DMA_COFF: 4962 return (DDI_FAILURE); 4963 } 4964 4965 return (DDI_FAILURE); 4966 #endif /* defined(__amd64) */ 4967 } 4968 4969 /* 4970 * ********* 4971 * FMA Code 4972 * ********* 4973 */ 4974 4975 /* 4976 * rootnex_fm_init() 4977 * FMA init busop 4978 */ 4979 /* ARGSUSED */ 4980 static int 4981 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4982 ddi_iblock_cookie_t *ibc) 4983 { 4984 *ibc = rootnex_state->r_err_ibc; 4985 4986 return (ddi_system_fmcap); 4987 } 4988 4989 /* 4990 * rootnex_dma_check() 4991 * Function called after a dma fault occurred to find out whether the 4992 * fault address is associated with a driver that is able to handle faults 4993 * and recover from faults. 4994 */ 4995 /* ARGSUSED */ 4996 static int 4997 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4998 const void *not_used) 4999 { 5000 rootnex_window_t *window; 5001 uint64_t start_addr; 5002 uint64_t fault_addr; 5003 ddi_dma_impl_t *hp; 5004 rootnex_dma_t *dma; 5005 uint64_t end_addr; 5006 size_t csize; 5007 int i; 5008 int j; 5009 5010 5011 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 5012 hp = (ddi_dma_impl_t *)handle; 5013 ASSERT(hp); 5014 5015 dma = (rootnex_dma_t *)hp->dmai_private; 5016 5017 /* Get the address that we need to search for */ 5018 fault_addr = *(uint64_t *)addr; 5019 5020 /* 5021 * if we don't have any windows, we can just walk through all the 5022 * cookies. 5023 */ 5024 if (dma->dp_window == NULL) { 5025 /* for each cookie */ 5026 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 5027 /* 5028 * if the faulted address is within the physical address 5029 * range of the cookie, return DDI_FM_NONFATAL. 5030 */ 5031 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 5032 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 5033 dma->dp_cookies[i].dmac_size))) { 5034 return (DDI_FM_NONFATAL); 5035 } 5036 } 5037 5038 /* fault_addr not within this DMA handle */ 5039 return (DDI_FM_UNKNOWN); 5040 } 5041 5042 /* we have mutiple windows, walk through each window */ 5043 for (i = 0; i < hp->dmai_nwin; i++) { 5044 window = &dma->dp_window[i]; 5045 5046 /* Go through all the cookies in the window */ 5047 for (j = 0; j < window->wd_cookie_cnt; j++) { 5048 5049 start_addr = window->wd_first_cookie[j].dmac_laddress; 5050 csize = window->wd_first_cookie[j].dmac_size; 5051 5052 /* 5053 * if we are trimming the first cookie in the window, 5054 * and this is the first cookie, adjust the start 5055 * address and size of the cookie to account for the 5056 * trim. 5057 */ 5058 if (window->wd_trim.tr_trim_first && (j == 0)) { 5059 start_addr = window->wd_trim.tr_first_paddr; 5060 csize = window->wd_trim.tr_first_size; 5061 } 5062 5063 /* 5064 * if we are trimming the last cookie in the window, 5065 * and this is the last cookie, adjust the start 5066 * address and size of the cookie to account for the 5067 * trim. 5068 */ 5069 if (window->wd_trim.tr_trim_last && 5070 (j == (window->wd_cookie_cnt - 1))) { 5071 start_addr = window->wd_trim.tr_last_paddr; 5072 csize = window->wd_trim.tr_last_size; 5073 } 5074 5075 end_addr = start_addr + csize; 5076 5077 /* 5078 * if the faulted address is within the physical 5079 * address of the cookie, return DDI_FM_NONFATAL. 5080 */ 5081 if ((fault_addr >= start_addr) && 5082 (fault_addr <= end_addr)) { 5083 return (DDI_FM_NONFATAL); 5084 } 5085 } 5086 } 5087 5088 /* fault_addr not within this DMA handle */ 5089 return (DDI_FM_UNKNOWN); 5090 } 5091 5092 /*ARGSUSED*/ 5093 static int 5094 rootnex_quiesce(dev_info_t *dip) 5095 { 5096 #if defined(__amd64) && !defined(__xpv) 5097 return (immu_quiesce()); 5098 #else 5099 return (DDI_SUCCESS); 5100 #endif 5101 } 5102 5103 #if defined(__xpv) 5104 void 5105 immu_init(void) 5106 { 5107 ; 5108 } 5109 5110 void 5111 immu_startup(void) 5112 { 5113 ; 5114 } 5115 /*ARGSUSED*/ 5116 void 5117 immu_physmem_update(uint64_t addr, uint64_t size) 5118 { 5119 ; 5120 } 5121 #endif 5122