1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #else 71 #include <sys/intel_iommu.h> 72 #endif 73 74 75 /* 76 * enable/disable extra checking of function parameters. Useful for debugging 77 * drivers. 78 */ 79 #ifdef DEBUG 80 int rootnex_alloc_check_parms = 1; 81 int rootnex_bind_check_parms = 1; 82 int rootnex_bind_check_inuse = 1; 83 int rootnex_unbind_verify_buffer = 0; 84 int rootnex_sync_check_parms = 1; 85 #else 86 int rootnex_alloc_check_parms = 0; 87 int rootnex_bind_check_parms = 0; 88 int rootnex_bind_check_inuse = 0; 89 int rootnex_unbind_verify_buffer = 0; 90 int rootnex_sync_check_parms = 0; 91 #endif 92 93 /* Master Abort and Target Abort panic flag */ 94 int rootnex_fm_ma_ta_panic_flag = 0; 95 96 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 97 int rootnex_bind_fail = 1; 98 int rootnex_bind_warn = 1; 99 uint8_t *rootnex_warn_list; 100 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 101 #define ROOTNEX_BIND_WARNING (0x1 << 0) 102 103 /* 104 * revert back to old broken behavior of always sync'ing entire copy buffer. 105 * This is useful if be have a buggy driver which doesn't correctly pass in 106 * the offset and size into ddi_dma_sync(). 107 */ 108 int rootnex_sync_ignore_params = 0; 109 110 /* 111 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 112 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 113 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 114 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 115 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 116 * (< 8K). We will still need to allocate the copy buffer during bind though 117 * (if we need one). These can only be modified in /etc/system before rootnex 118 * attach. 119 */ 120 #if defined(__amd64) 121 int rootnex_prealloc_cookies = 65; 122 int rootnex_prealloc_windows = 4; 123 int rootnex_prealloc_copybuf = 2; 124 #else 125 int rootnex_prealloc_cookies = 33; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #endif 129 130 /* driver global state */ 131 static rootnex_state_t *rootnex_state; 132 133 /* shortcut to rootnex counters */ 134 static uint64_t *rootnex_cnt; 135 136 /* 137 * XXX - does x86 even need these or are they left over from the SPARC days? 138 */ 139 /* statically defined integer/boolean properties for the root node */ 140 static rootnex_intprop_t rootnex_intprp[] = { 141 { "PAGESIZE", PAGESIZE }, 142 { "MMU_PAGESIZE", MMU_PAGESIZE }, 143 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 144 { DDI_RELATIVE_ADDRESSING, 1 }, 145 }; 146 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 147 148 #ifdef __xpv 149 typedef maddr_t rootnex_addr_t; 150 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 151 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 152 #else 153 typedef paddr_t rootnex_addr_t; 154 #endif 155 156 #if !defined(__xpv) 157 char _depends_on[] = "mach/pcplusmp misc/iommulib"; 158 #endif 159 160 static struct cb_ops rootnex_cb_ops = { 161 nodev, /* open */ 162 nodev, /* close */ 163 nodev, /* strategy */ 164 nodev, /* print */ 165 nodev, /* dump */ 166 nodev, /* read */ 167 nodev, /* write */ 168 nodev, /* ioctl */ 169 nodev, /* devmap */ 170 nodev, /* mmap */ 171 nodev, /* segmap */ 172 nochpoll, /* chpoll */ 173 ddi_prop_op, /* cb_prop_op */ 174 NULL, /* struct streamtab */ 175 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 176 CB_REV, /* Rev */ 177 nodev, /* cb_aread */ 178 nodev /* cb_awrite */ 179 }; 180 181 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 182 off_t offset, off_t len, caddr_t *vaddrp); 183 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 184 struct hat *hat, struct seg *seg, caddr_t addr, 185 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 186 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 187 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 188 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 189 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 190 ddi_dma_handle_t *handlep); 191 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 192 ddi_dma_handle_t handle); 193 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 194 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 195 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 196 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 197 ddi_dma_handle_t handle); 198 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 200 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 202 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 203 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 205 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 206 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 207 ddi_ctl_enum_t ctlop, void *arg, void *result); 208 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 209 ddi_iblock_cookie_t *ibc); 210 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 211 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 212 213 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 214 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 215 ddi_dma_handle_t *handlep); 216 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 217 ddi_dma_handle_t handle); 218 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 219 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 220 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 221 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle); 223 #if !defined(__xpv) 224 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 225 ddi_dma_handle_t handle); 226 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 227 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 228 #endif 229 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 230 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 231 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 232 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 233 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 234 static int rootnex_coredma_map(dev_info_t *dip, dev_info_t *rdip, 235 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 236 static int rootnex_coredma_mctl(dev_info_t *dip, dev_info_t *rdip, 237 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, off_t *offp, 238 size_t *lenp, caddr_t *objpp, uint_t cache_flags); 239 240 static struct bus_ops rootnex_bus_ops = { 241 BUSO_REV, 242 rootnex_map, 243 NULL, 244 NULL, 245 NULL, 246 rootnex_map_fault, 247 rootnex_dma_map, 248 rootnex_dma_allochdl, 249 rootnex_dma_freehdl, 250 rootnex_dma_bindhdl, 251 rootnex_dma_unbindhdl, 252 rootnex_dma_sync, 253 rootnex_dma_win, 254 rootnex_dma_mctl, 255 rootnex_ctlops, 256 ddi_bus_prop_op, 257 i_ddi_rootnex_get_eventcookie, 258 i_ddi_rootnex_add_eventcall, 259 i_ddi_rootnex_remove_eventcall, 260 i_ddi_rootnex_post_event, 261 0, /* bus_intr_ctl */ 262 0, /* bus_config */ 263 0, /* bus_unconfig */ 264 rootnex_fm_init, /* bus_fm_init */ 265 NULL, /* bus_fm_fini */ 266 NULL, /* bus_fm_access_enter */ 267 NULL, /* bus_fm_access_exit */ 268 NULL, /* bus_powr */ 269 rootnex_intr_ops /* bus_intr_op */ 270 }; 271 272 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 273 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 274 275 static struct dev_ops rootnex_ops = { 276 DEVO_REV, 277 0, 278 ddi_no_info, 279 nulldev, 280 nulldev, 281 rootnex_attach, 282 rootnex_detach, 283 nulldev, 284 &rootnex_cb_ops, 285 &rootnex_bus_ops 286 }; 287 288 static struct modldrv rootnex_modldrv = { 289 &mod_driverops, 290 "i86pc root nexus", 291 &rootnex_ops 292 }; 293 294 static struct modlinkage rootnex_modlinkage = { 295 MODREV_1, 296 (void *)&rootnex_modldrv, 297 NULL 298 }; 299 300 #if !defined(__xpv) 301 static iommulib_nexops_t iommulib_nexops = { 302 IOMMU_NEXOPS_VERSION, 303 "Rootnex IOMMU ops Vers 1.1", 304 NULL, 305 rootnex_coredma_allochdl, 306 rootnex_coredma_freehdl, 307 rootnex_coredma_bindhdl, 308 rootnex_coredma_unbindhdl, 309 rootnex_coredma_reset_cookies, 310 rootnex_coredma_get_cookies, 311 rootnex_coredma_sync, 312 rootnex_coredma_win, 313 rootnex_coredma_map, 314 rootnex_coredma_mctl 315 }; 316 #endif 317 318 /* 319 * extern hacks 320 */ 321 extern struct seg_ops segdev_ops; 322 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 323 #ifdef DDI_MAP_DEBUG 324 extern int ddi_map_debug_flag; 325 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 326 #endif 327 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 328 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 329 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 330 psm_intr_op_t, int *); 331 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 332 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 333 334 /* 335 * Use device arena to use for device control register mappings. 336 * Various kernel memory walkers (debugger, dtrace) need to know 337 * to avoid this address range to prevent undesired device activity. 338 */ 339 extern void *device_arena_alloc(size_t size, int vm_flag); 340 extern void device_arena_free(void * vaddr, size_t size); 341 342 343 /* 344 * Internal functions 345 */ 346 static int rootnex_dma_init(); 347 static void rootnex_add_props(dev_info_t *); 348 static int rootnex_ctl_reportdev(dev_info_t *dip); 349 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 350 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 351 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 352 static int rootnex_map_handle(ddi_map_req_t *mp); 353 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 354 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 355 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 356 ddi_dma_attr_t *attr); 357 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 358 rootnex_sglinfo_t *sglinfo); 359 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 360 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 361 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 362 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 363 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 364 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 365 ddi_dma_attr_t *attr, int kmflag); 366 static void rootnex_teardown_windows(rootnex_dma_t *dma); 367 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 368 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 369 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 370 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 371 size_t *copybuf_used, page_t **cur_pp); 372 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 373 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 374 ddi_dma_attr_t *attr, off_t cur_offset); 375 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 376 rootnex_dma_t *dma, rootnex_window_t **windowp, 377 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 378 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 379 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 380 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 381 off_t offset, size_t size, uint_t cache_flags); 382 static int rootnex_verify_buffer(rootnex_dma_t *dma); 383 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 384 const void *comp_addr, const void *not_used); 385 386 /* 387 * _init() 388 * 389 */ 390 int 391 _init(void) 392 { 393 394 rootnex_state = NULL; 395 return (mod_install(&rootnex_modlinkage)); 396 } 397 398 399 /* 400 * _info() 401 * 402 */ 403 int 404 _info(struct modinfo *modinfop) 405 { 406 return (mod_info(&rootnex_modlinkage, modinfop)); 407 } 408 409 410 /* 411 * _fini() 412 * 413 */ 414 int 415 _fini(void) 416 { 417 return (EBUSY); 418 } 419 420 421 /* 422 * rootnex_attach() 423 * 424 */ 425 static int 426 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 427 { 428 int fmcap; 429 int e; 430 431 switch (cmd) { 432 case DDI_ATTACH: 433 break; 434 case DDI_RESUME: 435 return (DDI_SUCCESS); 436 default: 437 return (DDI_FAILURE); 438 } 439 440 /* 441 * We should only have one instance of rootnex. Save it away since we 442 * don't have an easy way to get it back later. 443 */ 444 ASSERT(rootnex_state == NULL); 445 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 446 447 rootnex_state->r_dip = dip; 448 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 449 rootnex_state->r_reserved_msg_printed = B_FALSE; 450 rootnex_cnt = &rootnex_state->r_counters[0]; 451 rootnex_state->r_intel_iommu_enabled = B_FALSE; 452 453 /* 454 * Set minimum fm capability level for i86pc platforms and then 455 * initialize error handling. Since we're the rootnex, we don't 456 * care what's returned in the fmcap field. 457 */ 458 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 459 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 460 fmcap = ddi_system_fmcap; 461 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 462 463 /* initialize DMA related state */ 464 e = rootnex_dma_init(); 465 if (e != DDI_SUCCESS) { 466 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 467 return (DDI_FAILURE); 468 } 469 470 /* Add static root node properties */ 471 rootnex_add_props(dip); 472 473 /* since we can't call ddi_report_dev() */ 474 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 475 476 /* Initialize rootnex event handle */ 477 i_ddi_rootnex_init_events(dip); 478 479 #if !defined(__xpv) 480 #if defined(__amd64) 481 /* probe intel iommu */ 482 intel_iommu_probe_and_parse(); 483 484 /* attach the iommu nodes */ 485 if (intel_iommu_support) { 486 if (intel_iommu_attach_dmar_nodes() == DDI_SUCCESS) { 487 rootnex_state->r_intel_iommu_enabled = B_TRUE; 488 } else { 489 intel_iommu_release_dmar_info(); 490 } 491 } 492 #endif 493 494 e = iommulib_nexus_register(dip, &iommulib_nexops, 495 &rootnex_state->r_iommulib_handle); 496 497 ASSERT(e == DDI_SUCCESS); 498 #endif 499 500 return (DDI_SUCCESS); 501 } 502 503 504 /* 505 * rootnex_detach() 506 * 507 */ 508 /*ARGSUSED*/ 509 static int 510 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 511 { 512 switch (cmd) { 513 case DDI_SUSPEND: 514 break; 515 default: 516 return (DDI_FAILURE); 517 } 518 519 return (DDI_SUCCESS); 520 } 521 522 523 /* 524 * rootnex_dma_init() 525 * 526 */ 527 /*ARGSUSED*/ 528 static int 529 rootnex_dma_init() 530 { 531 size_t bufsize; 532 533 534 /* 535 * size of our cookie/window/copybuf state needed in dma bind that we 536 * pre-alloc in dma_alloc_handle 537 */ 538 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 539 rootnex_state->r_prealloc_size = 540 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 541 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 542 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 543 544 /* 545 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 546 * allocate 16 extra bytes for struct pointer alignment 547 * (p->dmai_private & dma->dp_prealloc_buffer) 548 */ 549 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 550 rootnex_state->r_prealloc_size + 0x10; 551 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 552 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 553 if (rootnex_state->r_dmahdl_cache == NULL) { 554 return (DDI_FAILURE); 555 } 556 557 /* 558 * allocate array to track which major numbers we have printed warnings 559 * for. 560 */ 561 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 562 KM_SLEEP); 563 564 return (DDI_SUCCESS); 565 } 566 567 568 /* 569 * rootnex_add_props() 570 * 571 */ 572 static void 573 rootnex_add_props(dev_info_t *dip) 574 { 575 rootnex_intprop_t *rpp; 576 int i; 577 578 /* Add static integer/boolean properties to the root node */ 579 rpp = rootnex_intprp; 580 for (i = 0; i < NROOT_INTPROPS; i++) { 581 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 582 rpp[i].prop_name, rpp[i].prop_value); 583 } 584 } 585 586 587 588 /* 589 * ************************* 590 * ctlops related routines 591 * ************************* 592 */ 593 594 /* 595 * rootnex_ctlops() 596 * 597 */ 598 /*ARGSUSED*/ 599 static int 600 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 601 void *arg, void *result) 602 { 603 int n, *ptr; 604 struct ddi_parent_private_data *pdp; 605 606 switch (ctlop) { 607 case DDI_CTLOPS_DMAPMAPC: 608 /* 609 * Return 'partial' to indicate that dma mapping 610 * has to be done in the main MMU. 611 */ 612 return (DDI_DMA_PARTIAL); 613 614 case DDI_CTLOPS_BTOP: 615 /* 616 * Convert byte count input to physical page units. 617 * (byte counts that are not a page-size multiple 618 * are rounded down) 619 */ 620 *(ulong_t *)result = btop(*(ulong_t *)arg); 621 return (DDI_SUCCESS); 622 623 case DDI_CTLOPS_PTOB: 624 /* 625 * Convert size in physical pages to bytes 626 */ 627 *(ulong_t *)result = ptob(*(ulong_t *)arg); 628 return (DDI_SUCCESS); 629 630 case DDI_CTLOPS_BTOPR: 631 /* 632 * Convert byte count input to physical page units 633 * (byte counts that are not a page-size multiple 634 * are rounded up) 635 */ 636 *(ulong_t *)result = btopr(*(ulong_t *)arg); 637 return (DDI_SUCCESS); 638 639 case DDI_CTLOPS_INITCHILD: 640 return (impl_ddi_sunbus_initchild(arg)); 641 642 case DDI_CTLOPS_UNINITCHILD: 643 impl_ddi_sunbus_removechild(arg); 644 return (DDI_SUCCESS); 645 646 case DDI_CTLOPS_REPORTDEV: 647 return (rootnex_ctl_reportdev(rdip)); 648 649 case DDI_CTLOPS_IOMIN: 650 /* 651 * Nothing to do here but reflect back.. 652 */ 653 return (DDI_SUCCESS); 654 655 case DDI_CTLOPS_REGSIZE: 656 case DDI_CTLOPS_NREGS: 657 break; 658 659 case DDI_CTLOPS_SIDDEV: 660 if (ndi_dev_is_prom_node(rdip)) 661 return (DDI_SUCCESS); 662 if (ndi_dev_is_persistent_node(rdip)) 663 return (DDI_SUCCESS); 664 return (DDI_FAILURE); 665 666 case DDI_CTLOPS_POWER: 667 return ((*pm_platform_power)((power_req_t *)arg)); 668 669 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 670 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 671 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 672 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 673 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 674 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 675 if (!rootnex_state->r_reserved_msg_printed) { 676 rootnex_state->r_reserved_msg_printed = B_TRUE; 677 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 678 "1 or more reserved/obsolete operations."); 679 } 680 return (DDI_FAILURE); 681 682 default: 683 return (DDI_FAILURE); 684 } 685 /* 686 * The rest are for "hardware" properties 687 */ 688 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 689 return (DDI_FAILURE); 690 691 if (ctlop == DDI_CTLOPS_NREGS) { 692 ptr = (int *)result; 693 *ptr = pdp->par_nreg; 694 } else { 695 off_t *size = (off_t *)result; 696 697 ptr = (int *)arg; 698 n = *ptr; 699 if (n >= pdp->par_nreg) { 700 return (DDI_FAILURE); 701 } 702 *size = (off_t)pdp->par_reg[n].regspec_size; 703 } 704 return (DDI_SUCCESS); 705 } 706 707 708 /* 709 * rootnex_ctl_reportdev() 710 * 711 */ 712 static int 713 rootnex_ctl_reportdev(dev_info_t *dev) 714 { 715 int i, n, len, f_len = 0; 716 char *buf; 717 718 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 719 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 720 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 721 len = strlen(buf); 722 723 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 724 725 struct regspec *rp = sparc_pd_getreg(dev, i); 726 727 if (i == 0) 728 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 729 ": "); 730 else 731 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 732 " and "); 733 len = strlen(buf); 734 735 switch (rp->regspec_bustype) { 736 737 case BTEISA: 738 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 739 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 740 break; 741 742 case BTISA: 743 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 744 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 745 break; 746 747 default: 748 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 749 "space %x offset %x", 750 rp->regspec_bustype, rp->regspec_addr); 751 break; 752 } 753 len = strlen(buf); 754 } 755 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 756 int pri; 757 758 if (i != 0) { 759 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 760 ","); 761 len = strlen(buf); 762 } 763 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 764 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 765 " sparc ipl %d", pri); 766 len = strlen(buf); 767 } 768 #ifdef DEBUG 769 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 770 cmn_err(CE_NOTE, "next message is truncated: " 771 "printed length 1024, real length %d", f_len); 772 } 773 #endif /* DEBUG */ 774 cmn_err(CE_CONT, "?%s\n", buf); 775 kmem_free(buf, REPORTDEV_BUFSIZE); 776 return (DDI_SUCCESS); 777 } 778 779 780 /* 781 * ****************** 782 * map related code 783 * ****************** 784 */ 785 786 /* 787 * rootnex_map() 788 * 789 */ 790 static int 791 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 792 off_t len, caddr_t *vaddrp) 793 { 794 struct regspec *rp, tmp_reg; 795 ddi_map_req_t mr = *mp; /* Get private copy of request */ 796 int error; 797 798 mp = &mr; 799 800 switch (mp->map_op) { 801 case DDI_MO_MAP_LOCKED: 802 case DDI_MO_UNMAP: 803 case DDI_MO_MAP_HANDLE: 804 break; 805 default: 806 #ifdef DDI_MAP_DEBUG 807 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 808 mp->map_op); 809 #endif /* DDI_MAP_DEBUG */ 810 return (DDI_ME_UNIMPLEMENTED); 811 } 812 813 if (mp->map_flags & DDI_MF_USER_MAPPING) { 814 #ifdef DDI_MAP_DEBUG 815 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 816 #endif /* DDI_MAP_DEBUG */ 817 return (DDI_ME_UNIMPLEMENTED); 818 } 819 820 /* 821 * First, if given an rnumber, convert it to a regspec... 822 * (Presumably, this is on behalf of a child of the root node?) 823 */ 824 825 if (mp->map_type == DDI_MT_RNUMBER) { 826 827 int rnumber = mp->map_obj.rnumber; 828 #ifdef DDI_MAP_DEBUG 829 static char *out_of_range = 830 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 831 #endif /* DDI_MAP_DEBUG */ 832 833 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 834 if (rp == NULL) { 835 #ifdef DDI_MAP_DEBUG 836 cmn_err(CE_WARN, out_of_range, rnumber, 837 ddi_get_name(rdip)); 838 #endif /* DDI_MAP_DEBUG */ 839 return (DDI_ME_RNUMBER_RANGE); 840 } 841 842 /* 843 * Convert the given ddi_map_req_t from rnumber to regspec... 844 */ 845 846 mp->map_type = DDI_MT_REGSPEC; 847 mp->map_obj.rp = rp; 848 } 849 850 /* 851 * Adjust offset and length correspnding to called values... 852 * XXX: A non-zero length means override the one in the regspec 853 * XXX: (regardless of what's in the parent's range?) 854 */ 855 856 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 857 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 858 859 #ifdef DDI_MAP_DEBUG 860 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 861 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 862 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 863 len, mp->map_handlep); 864 #endif /* DDI_MAP_DEBUG */ 865 866 /* 867 * I/O or memory mapping: 868 * 869 * <bustype=0, addr=x, len=x>: memory 870 * <bustype=1, addr=x, len=x>: i/o 871 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 872 */ 873 874 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 875 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 876 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 877 ddi_get_name(rdip), rp->regspec_bustype, 878 rp->regspec_addr, rp->regspec_size); 879 return (DDI_ME_INVAL); 880 } 881 882 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 883 /* 884 * compatibility i/o mapping 885 */ 886 rp->regspec_bustype += (uint_t)offset; 887 } else { 888 /* 889 * Normal memory or i/o mapping 890 */ 891 rp->regspec_addr += (uint_t)offset; 892 } 893 894 if (len != 0) 895 rp->regspec_size = (uint_t)len; 896 897 #ifdef DDI_MAP_DEBUG 898 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 899 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 900 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 901 offset, len, mp->map_handlep); 902 #endif /* DDI_MAP_DEBUG */ 903 904 /* 905 * Apply any parent ranges at this level, if applicable. 906 * (This is where nexus specific regspec translation takes place. 907 * Use of this function is implicit agreement that translation is 908 * provided via ddi_apply_range.) 909 */ 910 911 #ifdef DDI_MAP_DEBUG 912 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 913 ddi_get_name(dip), ddi_get_name(rdip)); 914 #endif /* DDI_MAP_DEBUG */ 915 916 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 917 return (error); 918 919 switch (mp->map_op) { 920 case DDI_MO_MAP_LOCKED: 921 922 /* 923 * Set up the locked down kernel mapping to the regspec... 924 */ 925 926 return (rootnex_map_regspec(mp, vaddrp)); 927 928 case DDI_MO_UNMAP: 929 930 /* 931 * Release mapping... 932 */ 933 934 return (rootnex_unmap_regspec(mp, vaddrp)); 935 936 case DDI_MO_MAP_HANDLE: 937 938 return (rootnex_map_handle(mp)); 939 940 default: 941 return (DDI_ME_UNIMPLEMENTED); 942 } 943 } 944 945 946 /* 947 * rootnex_map_fault() 948 * 949 * fault in mappings for requestors 950 */ 951 /*ARGSUSED*/ 952 static int 953 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 954 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 955 uint_t lock) 956 { 957 958 #ifdef DDI_MAP_DEBUG 959 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 960 ddi_map_debug(" Seg <%s>\n", 961 seg->s_ops == &segdev_ops ? "segdev" : 962 seg == &kvseg ? "segkmem" : "NONE!"); 963 #endif /* DDI_MAP_DEBUG */ 964 965 /* 966 * This is all terribly broken, but it is a start 967 * 968 * XXX Note that this test means that segdev_ops 969 * must be exported from seg_dev.c. 970 * XXX What about devices with their own segment drivers? 971 */ 972 if (seg->s_ops == &segdev_ops) { 973 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 974 975 if (hat == NULL) { 976 /* 977 * This is one plausible interpretation of 978 * a null hat i.e. use the first hat on the 979 * address space hat list which by convention is 980 * the hat of the system MMU. At alternative 981 * would be to panic .. this might well be better .. 982 */ 983 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 984 hat = seg->s_as->a_hat; 985 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 986 } 987 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 988 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 989 } else if (seg == &kvseg && dp == NULL) { 990 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 991 HAT_LOAD_LOCK); 992 } else 993 return (DDI_FAILURE); 994 return (DDI_SUCCESS); 995 } 996 997 998 /* 999 * rootnex_map_regspec() 1000 * we don't support mapping of I/O cards above 4Gb 1001 */ 1002 static int 1003 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1004 { 1005 rootnex_addr_t rbase; 1006 void *cvaddr; 1007 uint_t npages, pgoffset; 1008 struct regspec *rp; 1009 ddi_acc_hdl_t *hp; 1010 ddi_acc_impl_t *ap; 1011 uint_t hat_acc_flags; 1012 paddr_t pbase; 1013 1014 rp = mp->map_obj.rp; 1015 hp = mp->map_handlep; 1016 1017 #ifdef DDI_MAP_DEBUG 1018 ddi_map_debug( 1019 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1020 rp->regspec_bustype, rp->regspec_addr, 1021 rp->regspec_size, mp->map_handlep); 1022 #endif /* DDI_MAP_DEBUG */ 1023 1024 /* 1025 * I/O or memory mapping 1026 * 1027 * <bustype=0, addr=x, len=x>: memory 1028 * <bustype=1, addr=x, len=x>: i/o 1029 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1030 */ 1031 1032 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1033 cmn_err(CE_WARN, "rootnex: invalid register spec" 1034 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1035 rp->regspec_addr, rp->regspec_size); 1036 return (DDI_FAILURE); 1037 } 1038 1039 if (rp->regspec_bustype != 0) { 1040 /* 1041 * I/O space - needs a handle. 1042 */ 1043 if (hp == NULL) { 1044 return (DDI_FAILURE); 1045 } 1046 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1047 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1048 impl_acc_hdl_init(hp); 1049 1050 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1051 #ifdef DDI_MAP_DEBUG 1052 ddi_map_debug("rootnex_map_regspec: mmap() " 1053 "to I/O space is not supported.\n"); 1054 #endif /* DDI_MAP_DEBUG */ 1055 return (DDI_ME_INVAL); 1056 } else { 1057 /* 1058 * 1275-compliant vs. compatibility i/o mapping 1059 */ 1060 *vaddrp = 1061 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1062 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1063 ((caddr_t)(uintptr_t)rp->regspec_addr); 1064 #ifdef __xpv 1065 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1066 hp->ah_pfn = xen_assign_pfn( 1067 mmu_btop((ulong_t)rp->regspec_addr & 1068 MMU_PAGEMASK)); 1069 } else { 1070 hp->ah_pfn = mmu_btop( 1071 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1072 } 1073 #else 1074 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1075 MMU_PAGEMASK); 1076 #endif 1077 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1078 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1079 } 1080 1081 #ifdef DDI_MAP_DEBUG 1082 ddi_map_debug( 1083 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1084 rp->regspec_size, *vaddrp); 1085 #endif /* DDI_MAP_DEBUG */ 1086 return (DDI_SUCCESS); 1087 } 1088 1089 /* 1090 * Memory space 1091 */ 1092 1093 if (hp != NULL) { 1094 /* 1095 * hat layer ignores 1096 * hp->ah_acc.devacc_attr_endian_flags. 1097 */ 1098 switch (hp->ah_acc.devacc_attr_dataorder) { 1099 case DDI_STRICTORDER_ACC: 1100 hat_acc_flags = HAT_STRICTORDER; 1101 break; 1102 case DDI_UNORDERED_OK_ACC: 1103 hat_acc_flags = HAT_UNORDERED_OK; 1104 break; 1105 case DDI_MERGING_OK_ACC: 1106 hat_acc_flags = HAT_MERGING_OK; 1107 break; 1108 case DDI_LOADCACHING_OK_ACC: 1109 hat_acc_flags = HAT_LOADCACHING_OK; 1110 break; 1111 case DDI_STORECACHING_OK_ACC: 1112 hat_acc_flags = HAT_STORECACHING_OK; 1113 break; 1114 } 1115 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1116 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1117 impl_acc_hdl_init(hp); 1118 hp->ah_hat_flags = hat_acc_flags; 1119 } else { 1120 hat_acc_flags = HAT_STRICTORDER; 1121 } 1122 1123 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1124 #ifdef __xpv 1125 /* 1126 * If we're dom0, we're using a real device so we need to translate 1127 * the MA to a PA. 1128 */ 1129 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1130 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1131 } else { 1132 pbase = rbase; 1133 } 1134 #else 1135 pbase = rbase; 1136 #endif 1137 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1138 1139 if (rp->regspec_size == 0) { 1140 #ifdef DDI_MAP_DEBUG 1141 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1142 #endif /* DDI_MAP_DEBUG */ 1143 return (DDI_ME_INVAL); 1144 } 1145 1146 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1147 /* extra cast to make gcc happy */ 1148 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1149 } else { 1150 npages = mmu_btopr(rp->regspec_size + pgoffset); 1151 1152 #ifdef DDI_MAP_DEBUG 1153 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1154 "physical %llx", npages, pbase); 1155 #endif /* DDI_MAP_DEBUG */ 1156 1157 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1158 if (cvaddr == NULL) 1159 return (DDI_ME_NORESOURCES); 1160 1161 /* 1162 * Now map in the pages we've allocated... 1163 */ 1164 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1165 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1166 HAT_LOAD_LOCK); 1167 *vaddrp = (caddr_t)cvaddr + pgoffset; 1168 1169 /* save away pfn and npages for FMA */ 1170 hp = mp->map_handlep; 1171 if (hp) { 1172 hp->ah_pfn = mmu_btop(pbase); 1173 hp->ah_pnum = npages; 1174 } 1175 } 1176 1177 #ifdef DDI_MAP_DEBUG 1178 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1179 #endif /* DDI_MAP_DEBUG */ 1180 return (DDI_SUCCESS); 1181 } 1182 1183 1184 /* 1185 * rootnex_unmap_regspec() 1186 * 1187 */ 1188 static int 1189 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1190 { 1191 caddr_t addr = (caddr_t)*vaddrp; 1192 uint_t npages, pgoffset; 1193 struct regspec *rp; 1194 1195 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1196 return (0); 1197 1198 rp = mp->map_obj.rp; 1199 1200 if (rp->regspec_size == 0) { 1201 #ifdef DDI_MAP_DEBUG 1202 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1203 #endif /* DDI_MAP_DEBUG */ 1204 return (DDI_ME_INVAL); 1205 } 1206 1207 /* 1208 * I/O or memory mapping: 1209 * 1210 * <bustype=0, addr=x, len=x>: memory 1211 * <bustype=1, addr=x, len=x>: i/o 1212 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1213 */ 1214 if (rp->regspec_bustype != 0) { 1215 /* 1216 * This is I/O space, which requires no particular 1217 * processing on unmap since it isn't mapped in the 1218 * first place. 1219 */ 1220 return (DDI_SUCCESS); 1221 } 1222 1223 /* 1224 * Memory space 1225 */ 1226 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1227 npages = mmu_btopr(rp->regspec_size + pgoffset); 1228 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1229 device_arena_free(addr - pgoffset, ptob(npages)); 1230 1231 /* 1232 * Destroy the pointer - the mapping has logically gone 1233 */ 1234 *vaddrp = NULL; 1235 1236 return (DDI_SUCCESS); 1237 } 1238 1239 1240 /* 1241 * rootnex_map_handle() 1242 * 1243 */ 1244 static int 1245 rootnex_map_handle(ddi_map_req_t *mp) 1246 { 1247 rootnex_addr_t rbase; 1248 ddi_acc_hdl_t *hp; 1249 uint_t pgoffset; 1250 struct regspec *rp; 1251 paddr_t pbase; 1252 1253 rp = mp->map_obj.rp; 1254 1255 #ifdef DDI_MAP_DEBUG 1256 ddi_map_debug( 1257 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1258 rp->regspec_bustype, rp->regspec_addr, 1259 rp->regspec_size, mp->map_handlep); 1260 #endif /* DDI_MAP_DEBUG */ 1261 1262 /* 1263 * I/O or memory mapping: 1264 * 1265 * <bustype=0, addr=x, len=x>: memory 1266 * <bustype=1, addr=x, len=x>: i/o 1267 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1268 */ 1269 if (rp->regspec_bustype != 0) { 1270 /* 1271 * This refers to I/O space, and we don't support "mapping" 1272 * I/O space to a user. 1273 */ 1274 return (DDI_FAILURE); 1275 } 1276 1277 /* 1278 * Set up the hat_flags for the mapping. 1279 */ 1280 hp = mp->map_handlep; 1281 1282 switch (hp->ah_acc.devacc_attr_endian_flags) { 1283 case DDI_NEVERSWAP_ACC: 1284 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1285 break; 1286 case DDI_STRUCTURE_LE_ACC: 1287 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1288 break; 1289 case DDI_STRUCTURE_BE_ACC: 1290 return (DDI_FAILURE); 1291 default: 1292 return (DDI_REGS_ACC_CONFLICT); 1293 } 1294 1295 switch (hp->ah_acc.devacc_attr_dataorder) { 1296 case DDI_STRICTORDER_ACC: 1297 break; 1298 case DDI_UNORDERED_OK_ACC: 1299 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1300 break; 1301 case DDI_MERGING_OK_ACC: 1302 hp->ah_hat_flags |= HAT_MERGING_OK; 1303 break; 1304 case DDI_LOADCACHING_OK_ACC: 1305 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1306 break; 1307 case DDI_STORECACHING_OK_ACC: 1308 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1309 break; 1310 default: 1311 return (DDI_FAILURE); 1312 } 1313 1314 rbase = (rootnex_addr_t)rp->regspec_addr & 1315 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1316 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1317 1318 if (rp->regspec_size == 0) 1319 return (DDI_ME_INVAL); 1320 1321 #ifdef __xpv 1322 /* 1323 * If we're dom0, we're using a real device so we need to translate 1324 * the MA to a PA. 1325 */ 1326 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1327 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1328 (rbase & MMU_PAGEOFFSET); 1329 } else { 1330 pbase = rbase; 1331 } 1332 #else 1333 pbase = rbase; 1334 #endif 1335 1336 hp->ah_pfn = mmu_btop(pbase); 1337 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1338 1339 return (DDI_SUCCESS); 1340 } 1341 1342 1343 1344 /* 1345 * ************************ 1346 * interrupt related code 1347 * ************************ 1348 */ 1349 1350 /* 1351 * rootnex_intr_ops() 1352 * bus_intr_op() function for interrupt support 1353 */ 1354 /* ARGSUSED */ 1355 static int 1356 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1357 ddi_intr_handle_impl_t *hdlp, void *result) 1358 { 1359 struct intrspec *ispec; 1360 struct ddi_parent_private_data *pdp; 1361 1362 DDI_INTR_NEXDBG((CE_CONT, 1363 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1364 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1365 1366 /* Process the interrupt operation */ 1367 switch (intr_op) { 1368 case DDI_INTROP_GETCAP: 1369 /* First check with pcplusmp */ 1370 if (psm_intr_ops == NULL) 1371 return (DDI_FAILURE); 1372 1373 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1374 *(int *)result = 0; 1375 return (DDI_FAILURE); 1376 } 1377 break; 1378 case DDI_INTROP_SETCAP: 1379 if (psm_intr_ops == NULL) 1380 return (DDI_FAILURE); 1381 1382 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1383 return (DDI_FAILURE); 1384 break; 1385 case DDI_INTROP_ALLOC: 1386 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1387 return (DDI_FAILURE); 1388 hdlp->ih_pri = ispec->intrspec_pri; 1389 *(int *)result = hdlp->ih_scratch1; 1390 break; 1391 case DDI_INTROP_FREE: 1392 pdp = ddi_get_parent_data(rdip); 1393 /* 1394 * Special case for 'pcic' driver' only. 1395 * If an intrspec was created for it, clean it up here 1396 * See detailed comments on this in the function 1397 * rootnex_get_ispec(). 1398 */ 1399 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1400 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1401 pdp->par_nintr); 1402 /* 1403 * Set it to zero; so that 1404 * DDI framework doesn't free it again 1405 */ 1406 pdp->par_intr = NULL; 1407 pdp->par_nintr = 0; 1408 } 1409 break; 1410 case DDI_INTROP_GETPRI: 1411 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1412 return (DDI_FAILURE); 1413 *(int *)result = ispec->intrspec_pri; 1414 break; 1415 case DDI_INTROP_SETPRI: 1416 /* Validate the interrupt priority passed to us */ 1417 if (*(int *)result > LOCK_LEVEL) 1418 return (DDI_FAILURE); 1419 1420 /* Ensure that PSM is all initialized and ispec is ok */ 1421 if ((psm_intr_ops == NULL) || 1422 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1423 return (DDI_FAILURE); 1424 1425 /* Change the priority */ 1426 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1427 PSM_FAILURE) 1428 return (DDI_FAILURE); 1429 1430 /* update the ispec with the new priority */ 1431 ispec->intrspec_pri = *(int *)result; 1432 break; 1433 case DDI_INTROP_ADDISR: 1434 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1435 return (DDI_FAILURE); 1436 ispec->intrspec_func = hdlp->ih_cb_func; 1437 break; 1438 case DDI_INTROP_REMISR: 1439 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1440 return (DDI_FAILURE); 1441 ispec->intrspec_func = (uint_t (*)()) 0; 1442 break; 1443 case DDI_INTROP_ENABLE: 1444 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1445 return (DDI_FAILURE); 1446 1447 /* Call psmi to translate irq with the dip */ 1448 if (psm_intr_ops == NULL) 1449 return (DDI_FAILURE); 1450 1451 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1452 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1453 (int *)&hdlp->ih_vector); 1454 1455 /* Add the interrupt handler */ 1456 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1457 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1458 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1459 return (DDI_FAILURE); 1460 break; 1461 case DDI_INTROP_DISABLE: 1462 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1463 return (DDI_FAILURE); 1464 1465 /* Call psm_ops() to translate irq with the dip */ 1466 if (psm_intr_ops == NULL) 1467 return (DDI_FAILURE); 1468 1469 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1470 (void) (*psm_intr_ops)(rdip, hdlp, 1471 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1472 1473 /* Remove the interrupt handler */ 1474 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1475 hdlp->ih_cb_func, hdlp->ih_vector); 1476 break; 1477 case DDI_INTROP_SETMASK: 1478 if (psm_intr_ops == NULL) 1479 return (DDI_FAILURE); 1480 1481 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1482 return (DDI_FAILURE); 1483 break; 1484 case DDI_INTROP_CLRMASK: 1485 if (psm_intr_ops == NULL) 1486 return (DDI_FAILURE); 1487 1488 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1489 return (DDI_FAILURE); 1490 break; 1491 case DDI_INTROP_GETPENDING: 1492 if (psm_intr_ops == NULL) 1493 return (DDI_FAILURE); 1494 1495 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1496 result)) { 1497 *(int *)result = 0; 1498 return (DDI_FAILURE); 1499 } 1500 break; 1501 case DDI_INTROP_NAVAIL: 1502 case DDI_INTROP_NINTRS: 1503 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1504 if (*(int *)result == 0) { 1505 /* 1506 * Special case for 'pcic' driver' only. This driver 1507 * driver is a child of 'isa' and 'rootnex' drivers. 1508 * 1509 * See detailed comments on this in the function 1510 * rootnex_get_ispec(). 1511 * 1512 * Children of 'pcic' send 'NINITR' request all the 1513 * way to rootnex driver. But, the 'pdp->par_nintr' 1514 * field may not initialized. So, we fake it here 1515 * to return 1 (a la what PCMCIA nexus does). 1516 */ 1517 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1518 *(int *)result = 1; 1519 else 1520 return (DDI_FAILURE); 1521 } 1522 break; 1523 case DDI_INTROP_SUPPORTED_TYPES: 1524 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1525 break; 1526 default: 1527 return (DDI_FAILURE); 1528 } 1529 1530 return (DDI_SUCCESS); 1531 } 1532 1533 1534 /* 1535 * rootnex_get_ispec() 1536 * convert an interrupt number to an interrupt specification. 1537 * The interrupt number determines which interrupt spec will be 1538 * returned if more than one exists. 1539 * 1540 * Look into the parent private data area of the 'rdip' to find out 1541 * the interrupt specification. First check to make sure there is 1542 * one that matchs "inumber" and then return a pointer to it. 1543 * 1544 * Return NULL if one could not be found. 1545 * 1546 * NOTE: This is needed for rootnex_intr_ops() 1547 */ 1548 static struct intrspec * 1549 rootnex_get_ispec(dev_info_t *rdip, int inum) 1550 { 1551 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1552 1553 /* 1554 * Special case handling for drivers that provide their own 1555 * intrspec structures instead of relying on the DDI framework. 1556 * 1557 * A broken hardware driver in ON could potentially provide its 1558 * own intrspec structure, instead of relying on the hardware. 1559 * If these drivers are children of 'rootnex' then we need to 1560 * continue to provide backward compatibility to them here. 1561 * 1562 * Following check is a special case for 'pcic' driver which 1563 * was found to have broken hardwre andby provides its own intrspec. 1564 * 1565 * Verbatim comments from this driver are shown here: 1566 * "Don't use the ddi_add_intr since we don't have a 1567 * default intrspec in all cases." 1568 * 1569 * Since an 'ispec' may not be always created for it, 1570 * check for that and create one if so. 1571 * 1572 * NOTE: Currently 'pcic' is the only driver found to do this. 1573 */ 1574 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1575 pdp->par_nintr = 1; 1576 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1577 pdp->par_nintr, KM_SLEEP); 1578 } 1579 1580 /* Validate the interrupt number */ 1581 if (inum >= pdp->par_nintr) 1582 return (NULL); 1583 1584 /* Get the interrupt structure pointer and return that */ 1585 return ((struct intrspec *)&pdp->par_intr[inum]); 1586 } 1587 1588 1589 /* 1590 * ****************** 1591 * dma related code 1592 * ****************** 1593 */ 1594 1595 /*ARGSUSED*/ 1596 static int 1597 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1598 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1599 ddi_dma_handle_t *handlep) 1600 { 1601 uint64_t maxsegmentsize_ll; 1602 uint_t maxsegmentsize; 1603 ddi_dma_impl_t *hp; 1604 rootnex_dma_t *dma; 1605 uint64_t count_max; 1606 uint64_t seg; 1607 int kmflag; 1608 int e; 1609 1610 1611 /* convert our sleep flags */ 1612 if (waitfp == DDI_DMA_SLEEP) { 1613 kmflag = KM_SLEEP; 1614 } else { 1615 kmflag = KM_NOSLEEP; 1616 } 1617 1618 /* 1619 * We try to do only one memory allocation here. We'll do a little 1620 * pointer manipulation later. If the bind ends up taking more than 1621 * our prealloc's space, we'll have to allocate more memory in the 1622 * bind operation. Not great, but much better than before and the 1623 * best we can do with the current bind interfaces. 1624 */ 1625 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1626 if (hp == NULL) { 1627 if (waitfp != DDI_DMA_DONTWAIT) { 1628 ddi_set_callback(waitfp, arg, 1629 &rootnex_state->r_dvma_call_list_id); 1630 } 1631 return (DDI_DMA_NORESOURCES); 1632 } 1633 1634 /* Do our pointer manipulation now, align the structures */ 1635 hp->dmai_private = (void *)(((uintptr_t)hp + 1636 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1637 dma = (rootnex_dma_t *)hp->dmai_private; 1638 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1639 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1640 1641 /* setup the handle */ 1642 rootnex_clean_dmahdl(hp); 1643 dma->dp_dip = rdip; 1644 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1645 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1646 hp->dmai_minxfer = attr->dma_attr_minxfer; 1647 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1648 hp->dmai_rdip = rdip; 1649 hp->dmai_attr = *attr; 1650 1651 /* we don't need to worry about the SPL since we do a tryenter */ 1652 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1653 1654 /* 1655 * Figure out our maximum segment size. If the segment size is greater 1656 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1657 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1658 * dma_attr_count_max are size-1 type values. 1659 * 1660 * Maximum segment size is the largest physically contiguous chunk of 1661 * memory that we can return from a bind (i.e. the maximum size of a 1662 * single cookie). 1663 */ 1664 1665 /* handle the rollover cases */ 1666 seg = attr->dma_attr_seg + 1; 1667 if (seg < attr->dma_attr_seg) { 1668 seg = attr->dma_attr_seg; 1669 } 1670 count_max = attr->dma_attr_count_max + 1; 1671 if (count_max < attr->dma_attr_count_max) { 1672 count_max = attr->dma_attr_count_max; 1673 } 1674 1675 /* 1676 * granularity may or may not be a power of two. If it isn't, we can't 1677 * use a simple mask. 1678 */ 1679 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1680 dma->dp_granularity_power_2 = B_FALSE; 1681 } else { 1682 dma->dp_granularity_power_2 = B_TRUE; 1683 } 1684 1685 /* 1686 * maxxfer should be a whole multiple of granularity. If we're going to 1687 * break up a window because we're greater than maxxfer, we might as 1688 * well make sure it's maxxfer is a whole multiple so we don't have to 1689 * worry about triming the window later on for this case. 1690 */ 1691 if (attr->dma_attr_granular > 1) { 1692 if (dma->dp_granularity_power_2) { 1693 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1694 (attr->dma_attr_maxxfer & 1695 (attr->dma_attr_granular - 1)); 1696 } else { 1697 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1698 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1699 } 1700 } else { 1701 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1702 } 1703 1704 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1705 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1706 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1707 maxsegmentsize = 0xFFFFFFFF; 1708 } else { 1709 maxsegmentsize = maxsegmentsize_ll; 1710 } 1711 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1712 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1713 1714 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1715 if (rootnex_alloc_check_parms) { 1716 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1717 if (e != DDI_SUCCESS) { 1718 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1719 (void) rootnex_dma_freehdl(dip, rdip, 1720 (ddi_dma_handle_t)hp); 1721 return (e); 1722 } 1723 } 1724 1725 *handlep = (ddi_dma_handle_t)hp; 1726 1727 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1728 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1729 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1730 1731 return (DDI_SUCCESS); 1732 } 1733 1734 1735 /* 1736 * rootnex_dma_allochdl() 1737 * called from ddi_dma_alloc_handle(). 1738 */ 1739 static int 1740 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1741 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1742 { 1743 #if !defined(__xpv) 1744 uint_t error = ENOTSUP; 1745 int retval; 1746 1747 retval = iommulib_nex_open(rdip, &error); 1748 1749 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1750 /* No IOMMU */ 1751 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1752 handlep)); 1753 } else if (retval != DDI_SUCCESS) { 1754 return (DDI_FAILURE); 1755 } 1756 1757 ASSERT(IOMMU_USED(rdip)); 1758 1759 /* has an IOMMU */ 1760 return (iommulib_nexdma_allochdl(dip, rdip, attr, 1761 waitfp, arg, handlep)); 1762 #else 1763 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1764 handlep)); 1765 #endif 1766 } 1767 1768 /*ARGSUSED*/ 1769 static int 1770 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1771 ddi_dma_handle_t handle) 1772 { 1773 ddi_dma_impl_t *hp; 1774 rootnex_dma_t *dma; 1775 1776 1777 hp = (ddi_dma_impl_t *)handle; 1778 dma = (rootnex_dma_t *)hp->dmai_private; 1779 1780 /* unbind should have been called first */ 1781 ASSERT(!dma->dp_inuse); 1782 1783 mutex_destroy(&dma->dp_mutex); 1784 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1785 1786 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1787 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1788 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1789 1790 if (rootnex_state->r_dvma_call_list_id) 1791 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1792 1793 return (DDI_SUCCESS); 1794 } 1795 1796 /* 1797 * rootnex_dma_freehdl() 1798 * called from ddi_dma_free_handle(). 1799 */ 1800 static int 1801 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1802 { 1803 #if !defined(__xpv) 1804 if (IOMMU_USED(rdip)) { 1805 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1806 } 1807 #endif 1808 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1809 } 1810 1811 1812 /*ARGSUSED*/ 1813 static int 1814 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1815 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1816 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1817 { 1818 rootnex_sglinfo_t *sinfo; 1819 ddi_dma_attr_t *attr; 1820 ddi_dma_impl_t *hp; 1821 rootnex_dma_t *dma; 1822 int kmflag; 1823 int e; 1824 1825 1826 hp = (ddi_dma_impl_t *)handle; 1827 dma = (rootnex_dma_t *)hp->dmai_private; 1828 sinfo = &dma->dp_sglinfo; 1829 attr = &hp->dmai_attr; 1830 1831 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1832 1833 /* 1834 * This is useful for debugging a driver. Not as useful in a production 1835 * system. The only time this will fail is if you have a driver bug. 1836 */ 1837 if (rootnex_bind_check_inuse) { 1838 /* 1839 * No one else should ever have this lock unless someone else 1840 * is trying to use this handle. So contention on the lock 1841 * is the same as inuse being set. 1842 */ 1843 e = mutex_tryenter(&dma->dp_mutex); 1844 if (e == 0) { 1845 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1846 return (DDI_DMA_INUSE); 1847 } 1848 if (dma->dp_inuse) { 1849 mutex_exit(&dma->dp_mutex); 1850 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1851 return (DDI_DMA_INUSE); 1852 } 1853 dma->dp_inuse = B_TRUE; 1854 mutex_exit(&dma->dp_mutex); 1855 } 1856 1857 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1858 if (rootnex_bind_check_parms) { 1859 e = rootnex_valid_bind_parms(dmareq, attr); 1860 if (e != DDI_SUCCESS) { 1861 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1862 rootnex_clean_dmahdl(hp); 1863 return (e); 1864 } 1865 } 1866 1867 /* save away the original bind info */ 1868 dma->dp_dma = dmareq->dmar_object; 1869 1870 #if !defined(__xpv) 1871 if (rootnex_state->r_intel_iommu_enabled) { 1872 e = intel_iommu_map_sgl(handle, dmareq, 1873 rootnex_state->r_prealloc_cookies); 1874 1875 switch (e) { 1876 case IOMMU_SGL_SUCCESS: 1877 goto rootnex_sgl_end; 1878 1879 case IOMMU_SGL_DISABLE: 1880 goto rootnex_sgl_start; 1881 1882 case IOMMU_SGL_NORESOURCES: 1883 cmn_err(CE_WARN, "iommu map sgl failed for %s", 1884 ddi_node_name(dma->dp_dip)); 1885 rootnex_clean_dmahdl(hp); 1886 return (DDI_DMA_NORESOURCES); 1887 1888 default: 1889 cmn_err(CE_WARN, 1890 "undefined value returned from" 1891 " intel_iommu_map_sgl: %d", 1892 e); 1893 rootnex_clean_dmahdl(hp); 1894 return (DDI_DMA_NORESOURCES); 1895 } 1896 } 1897 #endif 1898 1899 rootnex_sgl_start: 1900 /* 1901 * Figure out a rough estimate of what maximum number of pages this 1902 * buffer could use (a high estimate of course). 1903 */ 1904 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1905 1906 /* 1907 * We'll use the pre-allocated cookies for any bind that will *always* 1908 * fit (more important to be consistent, we don't want to create 1909 * additional degenerate cases). 1910 */ 1911 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1912 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1913 dma->dp_need_to_free_cookie = B_FALSE; 1914 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1915 uint_t, sinfo->si_max_pages); 1916 1917 /* 1918 * For anything larger than that, we'll go ahead and allocate the 1919 * maximum number of pages we expect to see. Hopefuly, we won't be 1920 * seeing this path in the fast path for high performance devices very 1921 * frequently. 1922 * 1923 * a ddi bind interface that allowed the driver to provide storage to 1924 * the bind interface would speed this case up. 1925 */ 1926 } else { 1927 /* convert the sleep flags */ 1928 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1929 kmflag = KM_SLEEP; 1930 } else { 1931 kmflag = KM_NOSLEEP; 1932 } 1933 1934 /* 1935 * Save away how much memory we allocated. If we're doing a 1936 * nosleep, the alloc could fail... 1937 */ 1938 dma->dp_cookie_size = sinfo->si_max_pages * 1939 sizeof (ddi_dma_cookie_t); 1940 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1941 if (dma->dp_cookies == NULL) { 1942 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1943 rootnex_clean_dmahdl(hp); 1944 return (DDI_DMA_NORESOURCES); 1945 } 1946 dma->dp_need_to_free_cookie = B_TRUE; 1947 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1948 sinfo->si_max_pages); 1949 } 1950 hp->dmai_cookie = dma->dp_cookies; 1951 1952 /* 1953 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1954 * looking at the contraints in the dma structure. It will then put some 1955 * additional state about the sgl in the dma struct (i.e. is the sgl 1956 * clean, or do we need to do some munging; how many pages need to be 1957 * copied, etc.) 1958 */ 1959 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1960 &dma->dp_sglinfo); 1961 1962 rootnex_sgl_end: 1963 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1964 /* if we don't need a copy buffer, we don't need to sync */ 1965 if (sinfo->si_copybuf_req == 0) { 1966 hp->dmai_rflags |= DMP_NOSYNC; 1967 } 1968 1969 /* 1970 * if we don't need the copybuf and we don't need to do a partial, we 1971 * hit the fast path. All the high performance devices should be trying 1972 * to hit this path. To hit this path, a device should be able to reach 1973 * all of memory, shouldn't try to bind more than it can transfer, and 1974 * the buffer shouldn't require more cookies than the driver/device can 1975 * handle [sgllen]). 1976 */ 1977 if ((sinfo->si_copybuf_req == 0) && 1978 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1979 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1980 /* 1981 * If the driver supports FMA, insert the handle in the FMA DMA 1982 * handle cache. 1983 */ 1984 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1985 hp->dmai_error.err_cf = rootnex_dma_check; 1986 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1987 } 1988 1989 /* 1990 * copy out the first cookie and ccountp, set the cookie 1991 * pointer to the second cookie. The first cookie is passed 1992 * back on the stack. Additional cookies are accessed via 1993 * ddi_dma_nextcookie() 1994 */ 1995 *cookiep = dma->dp_cookies[0]; 1996 *ccountp = sinfo->si_sgl_size; 1997 hp->dmai_cookie++; 1998 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1999 hp->dmai_nwin = 1; 2000 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2001 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 2002 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2003 dma->dp_dma.dmao_size); 2004 return (DDI_DMA_MAPPED); 2005 } 2006 2007 /* 2008 * go to the slow path, we may need to alloc more memory, create 2009 * multiple windows, and munge up a sgl to make the device happy. 2010 */ 2011 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2012 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2013 if (dma->dp_need_to_free_cookie) { 2014 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2015 } 2016 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2017 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2018 return (e); 2019 } 2020 2021 /* 2022 * If the driver supports FMA, insert the handle in the FMA DMA handle 2023 * cache. 2024 */ 2025 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 2026 hp->dmai_error.err_cf = rootnex_dma_check; 2027 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 2028 } 2029 2030 /* if the first window uses the copy buffer, sync it for the device */ 2031 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2032 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2033 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2034 DDI_DMA_SYNC_FORDEV); 2035 } 2036 2037 /* 2038 * copy out the first cookie and ccountp, set the cookie pointer to the 2039 * second cookie. Make sure the partial flag is set/cleared correctly. 2040 * If we have a partial map (i.e. multiple windows), the number of 2041 * cookies we return is the number of cookies in the first window. 2042 */ 2043 if (e == DDI_DMA_MAPPED) { 2044 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2045 *ccountp = sinfo->si_sgl_size; 2046 } else { 2047 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2048 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2049 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2050 } 2051 *cookiep = dma->dp_cookies[0]; 2052 hp->dmai_cookie++; 2053 2054 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2055 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2056 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2057 dma->dp_dma.dmao_size); 2058 return (e); 2059 } 2060 2061 2062 /* 2063 * rootnex_dma_bindhdl() 2064 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2065 */ 2066 static int 2067 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2068 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2069 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2070 { 2071 #if !defined(__xpv) 2072 if (IOMMU_USED(rdip)) { 2073 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2074 cookiep, ccountp)); 2075 } 2076 #endif 2077 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2078 cookiep, ccountp)); 2079 } 2080 2081 /*ARGSUSED*/ 2082 static int 2083 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2084 ddi_dma_handle_t handle) 2085 { 2086 ddi_dma_impl_t *hp; 2087 rootnex_dma_t *dma; 2088 int e; 2089 2090 hp = (ddi_dma_impl_t *)handle; 2091 dma = (rootnex_dma_t *)hp->dmai_private; 2092 2093 /* make sure the buffer wasn't free'd before calling unbind */ 2094 if (rootnex_unbind_verify_buffer) { 2095 e = rootnex_verify_buffer(dma); 2096 if (e != DDI_SUCCESS) { 2097 ASSERT(0); 2098 return (DDI_FAILURE); 2099 } 2100 } 2101 2102 /* sync the current window before unbinding the buffer */ 2103 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2104 (hp->dmai_rflags & DDI_DMA_READ)) { 2105 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2106 DDI_DMA_SYNC_FORCPU); 2107 } 2108 2109 /* 2110 * If the driver supports FMA, remove the handle in the FMA DMA handle 2111 * cache. 2112 */ 2113 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 2114 if ((DEVI(rdip)->devi_fmhdl != NULL) && 2115 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 2116 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 2117 } 2118 } 2119 2120 /* 2121 * cleanup and copy buffer or window state. if we didn't use the copy 2122 * buffer or windows, there won't be much to do :-) 2123 */ 2124 rootnex_teardown_copybuf(dma); 2125 rootnex_teardown_windows(dma); 2126 2127 #if !defined(__xpv) 2128 /* 2129 * If intel iommu enabled, clean up the page tables and free the dvma 2130 */ 2131 if (rootnex_state->r_intel_iommu_enabled) { 2132 intel_iommu_unmap_sgl(handle); 2133 } 2134 #endif 2135 2136 /* 2137 * If we had to allocate space to for the worse case sgl (it didn't 2138 * fit into our pre-allocate buffer), free that up now 2139 */ 2140 if (dma->dp_need_to_free_cookie) { 2141 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2142 } 2143 2144 /* 2145 * clean up the handle so it's ready for the next bind (i.e. if the 2146 * handle is reused). 2147 */ 2148 rootnex_clean_dmahdl(hp); 2149 2150 if (rootnex_state->r_dvma_call_list_id) 2151 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2152 2153 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2154 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2155 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2156 2157 return (DDI_SUCCESS); 2158 } 2159 2160 /* 2161 * rootnex_dma_unbindhdl() 2162 * called from ddi_dma_unbind_handle() 2163 */ 2164 /*ARGSUSED*/ 2165 static int 2166 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2167 ddi_dma_handle_t handle) 2168 { 2169 #if !defined(__xpv) 2170 if (IOMMU_USED(rdip)) { 2171 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2172 } 2173 #endif 2174 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2175 } 2176 2177 #if !defined(__xpv) 2178 /*ARGSUSED*/ 2179 static void 2180 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2181 { 2182 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2183 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2184 2185 hp->dmai_cookie = &dma->dp_cookies[0]; 2186 hp->dmai_cookie++; 2187 } 2188 2189 /*ARGSUSED*/ 2190 static int 2191 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2192 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2193 { 2194 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2195 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2196 2197 2198 if (hp->dmai_rflags & DDI_DMA_PARTIAL) { 2199 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2200 } else { 2201 *ccountp = dma->dp_sglinfo.si_sgl_size; 2202 } 2203 *cookiep = dma->dp_cookies[0]; 2204 2205 /* reset the cookies */ 2206 hp->dmai_cookie = &dma->dp_cookies[0]; 2207 hp->dmai_cookie++; 2208 2209 return (DDI_SUCCESS); 2210 } 2211 #endif 2212 2213 /* 2214 * rootnex_verify_buffer() 2215 * verify buffer wasn't free'd 2216 */ 2217 static int 2218 rootnex_verify_buffer(rootnex_dma_t *dma) 2219 { 2220 page_t **pplist; 2221 caddr_t vaddr; 2222 uint_t pcnt; 2223 uint_t poff; 2224 page_t *pp; 2225 char b; 2226 int i; 2227 2228 /* Figure out how many pages this buffer occupies */ 2229 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2230 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2231 } else { 2232 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2233 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2234 } 2235 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2236 2237 switch (dma->dp_dma.dmao_type) { 2238 case DMA_OTYP_PAGES: 2239 /* 2240 * for a linked list of pp's walk through them to make sure 2241 * they're locked and not free. 2242 */ 2243 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2244 for (i = 0; i < pcnt; i++) { 2245 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2246 return (DDI_FAILURE); 2247 } 2248 pp = pp->p_next; 2249 } 2250 break; 2251 2252 case DMA_OTYP_VADDR: 2253 case DMA_OTYP_BUFVADDR: 2254 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2255 /* 2256 * for an array of pp's walk through them to make sure they're 2257 * not free. It's possible that they may not be locked. 2258 */ 2259 if (pplist) { 2260 for (i = 0; i < pcnt; i++) { 2261 if (PP_ISFREE(pplist[i])) { 2262 return (DDI_FAILURE); 2263 } 2264 } 2265 2266 /* For a virtual address, try to peek at each page */ 2267 } else { 2268 if (dma->dp_sglinfo.si_asp == &kas) { 2269 for (i = 0; i < pcnt; i++) { 2270 if (ddi_peek8(NULL, vaddr, &b) == 2271 DDI_FAILURE) 2272 return (DDI_FAILURE); 2273 vaddr += MMU_PAGESIZE; 2274 } 2275 } 2276 } 2277 break; 2278 2279 default: 2280 ASSERT(0); 2281 break; 2282 } 2283 2284 return (DDI_SUCCESS); 2285 } 2286 2287 2288 /* 2289 * rootnex_clean_dmahdl() 2290 * Clean the dma handle. This should be called on a handle alloc and an 2291 * unbind handle. Set the handle state to the default settings. 2292 */ 2293 static void 2294 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2295 { 2296 rootnex_dma_t *dma; 2297 2298 2299 dma = (rootnex_dma_t *)hp->dmai_private; 2300 2301 hp->dmai_nwin = 0; 2302 dma->dp_current_cookie = 0; 2303 dma->dp_copybuf_size = 0; 2304 dma->dp_window = NULL; 2305 dma->dp_cbaddr = NULL; 2306 dma->dp_inuse = B_FALSE; 2307 dma->dp_need_to_free_cookie = B_FALSE; 2308 dma->dp_need_to_free_window = B_FALSE; 2309 dma->dp_partial_required = B_FALSE; 2310 dma->dp_trim_required = B_FALSE; 2311 dma->dp_sglinfo.si_copybuf_req = 0; 2312 #if !defined(__amd64) 2313 dma->dp_cb_remaping = B_FALSE; 2314 dma->dp_kva = NULL; 2315 #endif 2316 2317 /* FMA related initialization */ 2318 hp->dmai_fault = 0; 2319 hp->dmai_fault_check = NULL; 2320 hp->dmai_fault_notify = NULL; 2321 hp->dmai_error.err_ena = 0; 2322 hp->dmai_error.err_status = DDI_FM_OK; 2323 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2324 hp->dmai_error.err_ontrap = NULL; 2325 hp->dmai_error.err_fep = NULL; 2326 hp->dmai_error.err_cf = NULL; 2327 } 2328 2329 2330 /* 2331 * rootnex_valid_alloc_parms() 2332 * Called in ddi_dma_alloc_handle path to validate its parameters. 2333 */ 2334 static int 2335 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2336 { 2337 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2338 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2339 (attr->dma_attr_granular > MMU_PAGESIZE) || 2340 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2341 return (DDI_DMA_BADATTR); 2342 } 2343 2344 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2345 return (DDI_DMA_BADATTR); 2346 } 2347 2348 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2349 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2350 attr->dma_attr_sgllen <= 0) { 2351 return (DDI_DMA_BADATTR); 2352 } 2353 2354 /* We should be able to DMA into every byte offset in a page */ 2355 if (maxsegmentsize < MMU_PAGESIZE) { 2356 return (DDI_DMA_BADATTR); 2357 } 2358 2359 return (DDI_SUCCESS); 2360 } 2361 2362 2363 /* 2364 * rootnex_valid_bind_parms() 2365 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2366 */ 2367 /* ARGSUSED */ 2368 static int 2369 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2370 { 2371 #if !defined(__amd64) 2372 /* 2373 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2374 * we can track the offset for the obsoleted interfaces. 2375 */ 2376 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2377 return (DDI_DMA_TOOBIG); 2378 } 2379 #endif 2380 2381 return (DDI_SUCCESS); 2382 } 2383 2384 2385 /* 2386 * rootnex_get_sgl() 2387 * Called in bind fastpath to get the sgl. Most of this will be replaced 2388 * with a call to the vm layer when vm2.0 comes around... 2389 */ 2390 static void 2391 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2392 rootnex_sglinfo_t *sglinfo) 2393 { 2394 ddi_dma_atyp_t buftype; 2395 rootnex_addr_t raddr; 2396 uint64_t last_page; 2397 uint64_t offset; 2398 uint64_t addrhi; 2399 uint64_t addrlo; 2400 uint64_t maxseg; 2401 page_t **pplist; 2402 uint64_t paddr; 2403 uint32_t psize; 2404 uint32_t size; 2405 caddr_t vaddr; 2406 uint_t pcnt; 2407 page_t *pp; 2408 uint_t cnt; 2409 2410 2411 /* shortcuts */ 2412 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2413 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2414 maxseg = sglinfo->si_max_cookie_size; 2415 buftype = dmar_object->dmao_type; 2416 addrhi = sglinfo->si_max_addr; 2417 addrlo = sglinfo->si_min_addr; 2418 size = dmar_object->dmao_size; 2419 2420 pcnt = 0; 2421 cnt = 0; 2422 2423 /* 2424 * if we were passed down a linked list of pages, i.e. pointer to 2425 * page_t, use this to get our physical address and buf offset. 2426 */ 2427 if (buftype == DMA_OTYP_PAGES) { 2428 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2429 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2430 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2431 MMU_PAGEOFFSET; 2432 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2433 psize = MIN(size, (MMU_PAGESIZE - offset)); 2434 pp = pp->p_next; 2435 sglinfo->si_asp = NULL; 2436 2437 /* 2438 * We weren't passed down a linked list of pages, but if we were passed 2439 * down an array of pages, use this to get our physical address and buf 2440 * offset. 2441 */ 2442 } else if (pplist != NULL) { 2443 ASSERT((buftype == DMA_OTYP_VADDR) || 2444 (buftype == DMA_OTYP_BUFVADDR)); 2445 2446 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2447 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2448 if (sglinfo->si_asp == NULL) { 2449 sglinfo->si_asp = &kas; 2450 } 2451 2452 ASSERT(!PP_ISFREE(pplist[pcnt])); 2453 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2454 paddr += offset; 2455 psize = MIN(size, (MMU_PAGESIZE - offset)); 2456 pcnt++; 2457 2458 /* 2459 * All we have is a virtual address, we'll need to call into the VM 2460 * to get the physical address. 2461 */ 2462 } else { 2463 ASSERT((buftype == DMA_OTYP_VADDR) || 2464 (buftype == DMA_OTYP_BUFVADDR)); 2465 2466 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2467 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2468 if (sglinfo->si_asp == NULL) { 2469 sglinfo->si_asp = &kas; 2470 } 2471 2472 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2473 paddr += offset; 2474 psize = MIN(size, (MMU_PAGESIZE - offset)); 2475 vaddr += psize; 2476 } 2477 2478 #ifdef __xpv 2479 /* 2480 * If we're dom0, we're using a real device so we need to load 2481 * the cookies with MFNs instead of PFNs. 2482 */ 2483 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2484 #else 2485 raddr = paddr; 2486 #endif 2487 2488 /* 2489 * Setup the first cookie with the physical address of the page and the 2490 * size of the page (which takes into account the initial offset into 2491 * the page. 2492 */ 2493 sgl[cnt].dmac_laddress = raddr; 2494 sgl[cnt].dmac_size = psize; 2495 sgl[cnt].dmac_type = 0; 2496 2497 /* 2498 * Save away the buffer offset into the page. We'll need this later in 2499 * the copy buffer code to help figure out the page index within the 2500 * buffer and the offset into the current page. 2501 */ 2502 sglinfo->si_buf_offset = offset; 2503 2504 /* 2505 * If the DMA engine can't reach the physical address, increase how 2506 * much copy buffer we need. We always increase by pagesize so we don't 2507 * have to worry about converting offsets. Set a flag in the cookies 2508 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2509 * last cookie, go to the next cookie (since we separate each page which 2510 * uses the copy buffer in case the copy buffer is not physically 2511 * contiguous. 2512 */ 2513 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2514 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2515 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2516 if ((cnt + 1) < sglinfo->si_max_pages) { 2517 cnt++; 2518 sgl[cnt].dmac_laddress = 0; 2519 sgl[cnt].dmac_size = 0; 2520 sgl[cnt].dmac_type = 0; 2521 } 2522 } 2523 2524 /* 2525 * save this page's physical address so we can figure out if the next 2526 * page is physically contiguous. Keep decrementing size until we are 2527 * done with the buffer. 2528 */ 2529 last_page = raddr & MMU_PAGEMASK; 2530 size -= psize; 2531 2532 while (size > 0) { 2533 /* Get the size for this page (i.e. partial or full page) */ 2534 psize = MIN(size, MMU_PAGESIZE); 2535 2536 if (buftype == DMA_OTYP_PAGES) { 2537 /* get the paddr from the page_t */ 2538 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2539 paddr = pfn_to_pa(pp->p_pagenum); 2540 pp = pp->p_next; 2541 } else if (pplist != NULL) { 2542 /* index into the array of page_t's to get the paddr */ 2543 ASSERT(!PP_ISFREE(pplist[pcnt])); 2544 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2545 pcnt++; 2546 } else { 2547 /* call into the VM to get the paddr */ 2548 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2549 vaddr)); 2550 vaddr += psize; 2551 } 2552 2553 #ifdef __xpv 2554 /* 2555 * If we're dom0, we're using a real device so we need to load 2556 * the cookies with MFNs instead of PFNs. 2557 */ 2558 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2559 #else 2560 raddr = paddr; 2561 #endif 2562 2563 /* check to see if this page needs the copy buffer */ 2564 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2565 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2566 2567 /* 2568 * if there is something in the current cookie, go to 2569 * the next one. We only want one page in a cookie which 2570 * uses the copybuf since the copybuf doesn't have to 2571 * be physically contiguous. 2572 */ 2573 if (sgl[cnt].dmac_size != 0) { 2574 cnt++; 2575 } 2576 sgl[cnt].dmac_laddress = raddr; 2577 sgl[cnt].dmac_size = psize; 2578 #if defined(__amd64) 2579 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2580 #else 2581 /* 2582 * save the buf offset for 32-bit kernel. used in the 2583 * obsoleted interfaces. 2584 */ 2585 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2586 (dmar_object->dmao_size - size); 2587 #endif 2588 /* if this isn't the last cookie, go to the next one */ 2589 if ((cnt + 1) < sglinfo->si_max_pages) { 2590 cnt++; 2591 sgl[cnt].dmac_laddress = 0; 2592 sgl[cnt].dmac_size = 0; 2593 sgl[cnt].dmac_type = 0; 2594 } 2595 2596 /* 2597 * this page didn't need the copy buffer, if it's not physically 2598 * contiguous, or it would put us over a segment boundary, or it 2599 * puts us over the max cookie size, or the current sgl doesn't 2600 * have anything in it. 2601 */ 2602 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2603 !(raddr & sglinfo->si_segmask) || 2604 ((sgl[cnt].dmac_size + psize) > maxseg) || 2605 (sgl[cnt].dmac_size == 0)) { 2606 /* 2607 * if we're not already in a new cookie, go to the next 2608 * cookie. 2609 */ 2610 if (sgl[cnt].dmac_size != 0) { 2611 cnt++; 2612 } 2613 2614 /* save the cookie information */ 2615 sgl[cnt].dmac_laddress = raddr; 2616 sgl[cnt].dmac_size = psize; 2617 #if defined(__amd64) 2618 sgl[cnt].dmac_type = 0; 2619 #else 2620 /* 2621 * save the buf offset for 32-bit kernel. used in the 2622 * obsoleted interfaces. 2623 */ 2624 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2625 #endif 2626 2627 /* 2628 * this page didn't need the copy buffer, it is physically 2629 * contiguous with the last page, and it's <= the max cookie 2630 * size. 2631 */ 2632 } else { 2633 sgl[cnt].dmac_size += psize; 2634 2635 /* 2636 * if this exactly == the maximum cookie size, and 2637 * it isn't the last cookie, go to the next cookie. 2638 */ 2639 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2640 ((cnt + 1) < sglinfo->si_max_pages)) { 2641 cnt++; 2642 sgl[cnt].dmac_laddress = 0; 2643 sgl[cnt].dmac_size = 0; 2644 sgl[cnt].dmac_type = 0; 2645 } 2646 } 2647 2648 /* 2649 * save this page's physical address so we can figure out if the 2650 * next page is physically contiguous. Keep decrementing size 2651 * until we are done with the buffer. 2652 */ 2653 last_page = raddr; 2654 size -= psize; 2655 } 2656 2657 /* we're done, save away how many cookies the sgl has */ 2658 if (sgl[cnt].dmac_size == 0) { 2659 ASSERT(cnt < sglinfo->si_max_pages); 2660 sglinfo->si_sgl_size = cnt; 2661 } else { 2662 sglinfo->si_sgl_size = cnt + 1; 2663 } 2664 } 2665 2666 2667 /* 2668 * rootnex_bind_slowpath() 2669 * Call in the bind path if the calling driver can't use the sgl without 2670 * modifying it. We either need to use the copy buffer and/or we will end up 2671 * with a partial bind. 2672 */ 2673 static int 2674 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2675 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2676 { 2677 rootnex_sglinfo_t *sinfo; 2678 rootnex_window_t *window; 2679 ddi_dma_cookie_t *cookie; 2680 size_t copybuf_used; 2681 size_t dmac_size; 2682 boolean_t partial; 2683 off_t cur_offset; 2684 page_t *cur_pp; 2685 major_t mnum; 2686 int e; 2687 int i; 2688 2689 2690 sinfo = &dma->dp_sglinfo; 2691 copybuf_used = 0; 2692 partial = B_FALSE; 2693 2694 /* 2695 * If we're using the copybuf, set the copybuf state in dma struct. 2696 * Needs to be first since it sets the copy buffer size. 2697 */ 2698 if (sinfo->si_copybuf_req != 0) { 2699 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2700 if (e != DDI_SUCCESS) { 2701 return (e); 2702 } 2703 } else { 2704 dma->dp_copybuf_size = 0; 2705 } 2706 2707 /* 2708 * Figure out if we need to do a partial mapping. If so, figure out 2709 * if we need to trim the buffers when we munge the sgl. 2710 */ 2711 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2712 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2713 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2714 dma->dp_partial_required = B_TRUE; 2715 if (attr->dma_attr_granular != 1) { 2716 dma->dp_trim_required = B_TRUE; 2717 } 2718 } else { 2719 dma->dp_partial_required = B_FALSE; 2720 dma->dp_trim_required = B_FALSE; 2721 } 2722 2723 /* If we need to do a partial bind, make sure the driver supports it */ 2724 if (dma->dp_partial_required && 2725 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2726 2727 mnum = ddi_driver_major(dma->dp_dip); 2728 /* 2729 * patchable which allows us to print one warning per major 2730 * number. 2731 */ 2732 if ((rootnex_bind_warn) && 2733 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2734 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2735 cmn_err(CE_WARN, "!%s: coding error detected, the " 2736 "driver is using ddi_dma_attr(9S) incorrectly. " 2737 "There is a small risk of data corruption in " 2738 "particular with large I/Os. The driver should be " 2739 "replaced with a corrected version for proper " 2740 "system operation. To disable this warning, add " 2741 "'set rootnex:rootnex_bind_warn=0' to " 2742 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2743 } 2744 return (DDI_DMA_TOOBIG); 2745 } 2746 2747 /* 2748 * we might need multiple windows, setup state to handle them. In this 2749 * code path, we will have at least one window. 2750 */ 2751 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2752 if (e != DDI_SUCCESS) { 2753 rootnex_teardown_copybuf(dma); 2754 return (e); 2755 } 2756 2757 window = &dma->dp_window[0]; 2758 cookie = &dma->dp_cookies[0]; 2759 cur_offset = 0; 2760 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2761 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2762 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2763 } 2764 2765 /* loop though all the cookies we got back from get_sgl() */ 2766 for (i = 0; i < sinfo->si_sgl_size; i++) { 2767 /* 2768 * If we're using the copy buffer, check this cookie and setup 2769 * its associated copy buffer state. If this cookie uses the 2770 * copy buffer, make sure we sync this window during dma_sync. 2771 */ 2772 if (dma->dp_copybuf_size > 0) { 2773 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2774 cur_offset, ©buf_used, &cur_pp); 2775 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2776 window->wd_dosync = B_TRUE; 2777 } 2778 } 2779 2780 /* 2781 * save away the cookie size, since it could be modified in 2782 * the windowing code. 2783 */ 2784 dmac_size = cookie->dmac_size; 2785 2786 /* if we went over max copybuf size */ 2787 if (dma->dp_copybuf_size && 2788 (copybuf_used > dma->dp_copybuf_size)) { 2789 partial = B_TRUE; 2790 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2791 cookie, cur_offset, ©buf_used); 2792 if (e != DDI_SUCCESS) { 2793 rootnex_teardown_copybuf(dma); 2794 rootnex_teardown_windows(dma); 2795 return (e); 2796 } 2797 2798 /* 2799 * if the coookie uses the copy buffer, make sure the 2800 * new window we just moved to is set to sync. 2801 */ 2802 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2803 window->wd_dosync = B_TRUE; 2804 } 2805 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2806 dma->dp_dip); 2807 2808 /* if the cookie cnt == max sgllen, move to the next window */ 2809 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2810 partial = B_TRUE; 2811 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2812 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2813 cookie, attr, cur_offset); 2814 if (e != DDI_SUCCESS) { 2815 rootnex_teardown_copybuf(dma); 2816 rootnex_teardown_windows(dma); 2817 return (e); 2818 } 2819 2820 /* 2821 * if the coookie uses the copy buffer, make sure the 2822 * new window we just moved to is set to sync. 2823 */ 2824 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2825 window->wd_dosync = B_TRUE; 2826 } 2827 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2828 dma->dp_dip); 2829 2830 /* else if we will be over maxxfer */ 2831 } else if ((window->wd_size + dmac_size) > 2832 dma->dp_maxxfer) { 2833 partial = B_TRUE; 2834 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2835 cookie); 2836 if (e != DDI_SUCCESS) { 2837 rootnex_teardown_copybuf(dma); 2838 rootnex_teardown_windows(dma); 2839 return (e); 2840 } 2841 2842 /* 2843 * if the coookie uses the copy buffer, make sure the 2844 * new window we just moved to is set to sync. 2845 */ 2846 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2847 window->wd_dosync = B_TRUE; 2848 } 2849 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2850 dma->dp_dip); 2851 2852 /* else this cookie fits in the current window */ 2853 } else { 2854 window->wd_cookie_cnt++; 2855 window->wd_size += dmac_size; 2856 } 2857 2858 /* track our offset into the buffer, go to the next cookie */ 2859 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2860 ASSERT(cookie->dmac_size <= dmac_size); 2861 cur_offset += dmac_size; 2862 cookie++; 2863 } 2864 2865 /* if we ended up with a zero sized window in the end, clean it up */ 2866 if (window->wd_size == 0) { 2867 hp->dmai_nwin--; 2868 window--; 2869 } 2870 2871 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2872 2873 if (!partial) { 2874 return (DDI_DMA_MAPPED); 2875 } 2876 2877 ASSERT(dma->dp_partial_required); 2878 return (DDI_DMA_PARTIAL_MAP); 2879 } 2880 2881 2882 /* 2883 * rootnex_setup_copybuf() 2884 * Called in bind slowpath. Figures out if we're going to use the copy 2885 * buffer, and if we do, sets up the basic state to handle it. 2886 */ 2887 static int 2888 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2889 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2890 { 2891 rootnex_sglinfo_t *sinfo; 2892 ddi_dma_attr_t lattr; 2893 size_t max_copybuf; 2894 int cansleep; 2895 int e; 2896 #if !defined(__amd64) 2897 int vmflag; 2898 #endif 2899 2900 2901 sinfo = &dma->dp_sglinfo; 2902 2903 /* read this first so it's consistent through the routine */ 2904 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 2905 2906 /* We need to call into the rootnex on ddi_dma_sync() */ 2907 hp->dmai_rflags &= ~DMP_NOSYNC; 2908 2909 /* make sure the copybuf size <= the max size */ 2910 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2911 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2912 2913 #if !defined(__amd64) 2914 /* 2915 * if we don't have kva space to copy to/from, allocate the KVA space 2916 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2917 * the 64-bit kernel. 2918 */ 2919 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2920 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2921 2922 /* convert the sleep flags */ 2923 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2924 vmflag = VM_SLEEP; 2925 } else { 2926 vmflag = VM_NOSLEEP; 2927 } 2928 2929 /* allocate Kernel VA space that we can bcopy to/from */ 2930 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2931 vmflag); 2932 if (dma->dp_kva == NULL) { 2933 return (DDI_DMA_NORESOURCES); 2934 } 2935 } 2936 #endif 2937 2938 /* convert the sleep flags */ 2939 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2940 cansleep = 1; 2941 } else { 2942 cansleep = 0; 2943 } 2944 2945 /* 2946 * Allocate the actual copy buffer. This needs to fit within the DMA 2947 * engine limits, so we can't use kmem_alloc... We don't need 2948 * contiguous memory (sgllen) since we will be forcing windows on 2949 * sgllen anyway. 2950 */ 2951 lattr = *attr; 2952 lattr.dma_attr_align = MMU_PAGESIZE; 2953 /* 2954 * this should be < 0 to indicate no limit, but due to a bug in 2955 * the rootnex, we'll set it to the maximum positive int. 2956 */ 2957 lattr.dma_attr_sgllen = 0x7fffffff; 2958 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2959 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2960 if (e != DDI_SUCCESS) { 2961 #if !defined(__amd64) 2962 if (dma->dp_kva != NULL) { 2963 vmem_free(heap_arena, dma->dp_kva, 2964 dma->dp_copybuf_size); 2965 } 2966 #endif 2967 return (DDI_DMA_NORESOURCES); 2968 } 2969 2970 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2971 size_t, dma->dp_copybuf_size); 2972 2973 return (DDI_SUCCESS); 2974 } 2975 2976 2977 /* 2978 * rootnex_setup_windows() 2979 * Called in bind slowpath to setup the window state. We always have windows 2980 * in the slowpath. Even if the window count = 1. 2981 */ 2982 static int 2983 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2984 ddi_dma_attr_t *attr, int kmflag) 2985 { 2986 rootnex_window_t *windowp; 2987 rootnex_sglinfo_t *sinfo; 2988 size_t copy_state_size; 2989 size_t win_state_size; 2990 size_t state_available; 2991 size_t space_needed; 2992 uint_t copybuf_win; 2993 uint_t maxxfer_win; 2994 size_t space_used; 2995 uint_t sglwin; 2996 2997 2998 sinfo = &dma->dp_sglinfo; 2999 3000 dma->dp_current_win = 0; 3001 hp->dmai_nwin = 0; 3002 3003 /* If we don't need to do a partial, we only have one window */ 3004 if (!dma->dp_partial_required) { 3005 dma->dp_max_win = 1; 3006 3007 /* 3008 * we need multiple windows, need to figure out the worse case number 3009 * of windows. 3010 */ 3011 } else { 3012 /* 3013 * if we need windows because we need more copy buffer that 3014 * we allow, the worse case number of windows we could need 3015 * here would be (copybuf space required / copybuf space that 3016 * we have) plus one for remainder, and plus 2 to handle the 3017 * extra pages on the trim for the first and last pages of the 3018 * buffer (a page is the minimum window size so under the right 3019 * attr settings, you could have a window for each page). 3020 * The last page will only be hit here if the size is not a 3021 * multiple of the granularity (which theoretically shouldn't 3022 * be the case but never has been enforced, so we could have 3023 * broken things without it). 3024 */ 3025 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3026 ASSERT(dma->dp_copybuf_size > 0); 3027 copybuf_win = (sinfo->si_copybuf_req / 3028 dma->dp_copybuf_size) + 1 + 2; 3029 } else { 3030 copybuf_win = 0; 3031 } 3032 3033 /* 3034 * if we need windows because we have more cookies than the H/W 3035 * can handle, the number of windows we would need here would 3036 * be (cookie count / cookies count H/W supports) plus one for 3037 * remainder, and plus 2 to handle the extra pages on the trim 3038 * (see above comment about trim) 3039 */ 3040 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3041 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 3042 + 1) + 2; 3043 } else { 3044 sglwin = 0; 3045 } 3046 3047 /* 3048 * if we need windows because we're binding more memory than the 3049 * H/W can transfer at once, the number of windows we would need 3050 * here would be (xfer count / max xfer H/W supports) plus one 3051 * for remainder, and plus 2 to handle the extra pages on the 3052 * trim (see above comment about trim) 3053 */ 3054 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3055 maxxfer_win = (dma->dp_dma.dmao_size / 3056 dma->dp_maxxfer) + 1 + 2; 3057 } else { 3058 maxxfer_win = 0; 3059 } 3060 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3061 ASSERT(dma->dp_max_win > 0); 3062 } 3063 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3064 3065 /* 3066 * Get space for window and potential copy buffer state. Before we 3067 * go and allocate memory, see if we can get away with using what's 3068 * left in the pre-allocted state or the dynamically allocated sgl. 3069 */ 3070 space_used = (uintptr_t)(sinfo->si_sgl_size * 3071 sizeof (ddi_dma_cookie_t)); 3072 3073 /* if we dynamically allocated space for the cookies */ 3074 if (dma->dp_need_to_free_cookie) { 3075 /* if we have more space in the pre-allocted buffer, use it */ 3076 ASSERT(space_used <= dma->dp_cookie_size); 3077 if ((dma->dp_cookie_size - space_used) <= 3078 rootnex_state->r_prealloc_size) { 3079 state_available = rootnex_state->r_prealloc_size; 3080 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3081 3082 /* 3083 * else, we have more free space in the dynamically allocated 3084 * buffer, i.e. the buffer wasn't worse case fragmented so we 3085 * didn't need a lot of cookies. 3086 */ 3087 } else { 3088 state_available = dma->dp_cookie_size - space_used; 3089 windowp = (rootnex_window_t *) 3090 &dma->dp_cookies[sinfo->si_sgl_size]; 3091 } 3092 3093 /* we used the pre-alloced buffer */ 3094 } else { 3095 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3096 state_available = rootnex_state->r_prealloc_size - space_used; 3097 windowp = (rootnex_window_t *) 3098 &dma->dp_cookies[sinfo->si_sgl_size]; 3099 } 3100 3101 /* 3102 * figure out how much state we need to track the copy buffer. Add an 3103 * addition 8 bytes for pointer alignemnt later. 3104 */ 3105 if (dma->dp_copybuf_size > 0) { 3106 copy_state_size = sinfo->si_max_pages * 3107 sizeof (rootnex_pgmap_t); 3108 } else { 3109 copy_state_size = 0; 3110 } 3111 /* add an additional 8 bytes for pointer alignment */ 3112 space_needed = win_state_size + copy_state_size + 0x8; 3113 3114 /* if we have enough space already, use it */ 3115 if (state_available >= space_needed) { 3116 dma->dp_window = windowp; 3117 dma->dp_need_to_free_window = B_FALSE; 3118 3119 /* not enough space, need to allocate more. */ 3120 } else { 3121 dma->dp_window = kmem_alloc(space_needed, kmflag); 3122 if (dma->dp_window == NULL) { 3123 return (DDI_DMA_NORESOURCES); 3124 } 3125 dma->dp_need_to_free_window = B_TRUE; 3126 dma->dp_window_size = space_needed; 3127 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3128 dma->dp_dip, size_t, space_needed); 3129 } 3130 3131 /* 3132 * we allocate copy buffer state and window state at the same time. 3133 * setup our copy buffer state pointers. Make sure it's aligned. 3134 */ 3135 if (dma->dp_copybuf_size > 0) { 3136 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3137 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3138 3139 #if !defined(__amd64) 3140 /* 3141 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3142 * false/NULL. Should be quicker to bzero vs loop and set. 3143 */ 3144 bzero(dma->dp_pgmap, copy_state_size); 3145 #endif 3146 } else { 3147 dma->dp_pgmap = NULL; 3148 } 3149 3150 return (DDI_SUCCESS); 3151 } 3152 3153 3154 /* 3155 * rootnex_teardown_copybuf() 3156 * cleans up after rootnex_setup_copybuf() 3157 */ 3158 static void 3159 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3160 { 3161 #if !defined(__amd64) 3162 int i; 3163 3164 /* 3165 * if we allocated kernel heap VMEM space, go through all the pages and 3166 * map out any of the ones that we're mapped into the kernel heap VMEM 3167 * arena. Then free the VMEM space. 3168 */ 3169 if (dma->dp_kva != NULL) { 3170 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3171 if (dma->dp_pgmap[i].pm_mapped) { 3172 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3173 MMU_PAGESIZE, HAT_UNLOAD); 3174 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3175 } 3176 } 3177 3178 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3179 } 3180 3181 #endif 3182 3183 /* if we allocated a copy buffer, free it */ 3184 if (dma->dp_cbaddr != NULL) { 3185 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3186 } 3187 } 3188 3189 3190 /* 3191 * rootnex_teardown_windows() 3192 * cleans up after rootnex_setup_windows() 3193 */ 3194 static void 3195 rootnex_teardown_windows(rootnex_dma_t *dma) 3196 { 3197 /* 3198 * if we had to allocate window state on the last bind (because we 3199 * didn't have enough pre-allocated space in the handle), free it. 3200 */ 3201 if (dma->dp_need_to_free_window) { 3202 kmem_free(dma->dp_window, dma->dp_window_size); 3203 } 3204 } 3205 3206 3207 /* 3208 * rootnex_init_win() 3209 * Called in bind slow path during creation of a new window. Initializes 3210 * window state to default values. 3211 */ 3212 /*ARGSUSED*/ 3213 static void 3214 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3215 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3216 { 3217 hp->dmai_nwin++; 3218 window->wd_dosync = B_FALSE; 3219 window->wd_offset = cur_offset; 3220 window->wd_size = 0; 3221 window->wd_first_cookie = cookie; 3222 window->wd_cookie_cnt = 0; 3223 window->wd_trim.tr_trim_first = B_FALSE; 3224 window->wd_trim.tr_trim_last = B_FALSE; 3225 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3226 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3227 #if !defined(__amd64) 3228 window->wd_remap_copybuf = dma->dp_cb_remaping; 3229 #endif 3230 } 3231 3232 3233 /* 3234 * rootnex_setup_cookie() 3235 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3236 * the sgl uses the copy buffer, we need to go through each cookie, figure 3237 * out if it uses the copy buffer, and if it does, save away everything we'll 3238 * need during sync. 3239 */ 3240 static void 3241 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3242 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3243 page_t **cur_pp) 3244 { 3245 boolean_t copybuf_sz_power_2; 3246 rootnex_sglinfo_t *sinfo; 3247 paddr_t paddr; 3248 uint_t pidx; 3249 uint_t pcnt; 3250 off_t poff; 3251 #if defined(__amd64) 3252 pfn_t pfn; 3253 #else 3254 page_t **pplist; 3255 #endif 3256 3257 sinfo = &dma->dp_sglinfo; 3258 3259 /* 3260 * Calculate the page index relative to the start of the buffer. The 3261 * index to the current page for our buffer is the offset into the 3262 * first page of the buffer plus our current offset into the buffer 3263 * itself, shifted of course... 3264 */ 3265 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3266 ASSERT(pidx < sinfo->si_max_pages); 3267 3268 /* if this cookie uses the copy buffer */ 3269 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3270 /* 3271 * NOTE: we know that since this cookie uses the copy buffer, it 3272 * is <= MMU_PAGESIZE. 3273 */ 3274 3275 /* 3276 * get the offset into the page. For the 64-bit kernel, get the 3277 * pfn which we'll use with seg kpm. 3278 */ 3279 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3280 #if defined(__amd64) 3281 /* mfn_to_pfn() is a NOP on i86pc */ 3282 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3283 #endif /* __amd64 */ 3284 3285 /* figure out if the copybuf size is a power of 2 */ 3286 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3287 copybuf_sz_power_2 = B_FALSE; 3288 } else { 3289 copybuf_sz_power_2 = B_TRUE; 3290 } 3291 3292 /* This page uses the copy buffer */ 3293 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3294 3295 /* 3296 * save the copy buffer KVA that we'll use with this page. 3297 * if we still fit within the copybuf, it's a simple add. 3298 * otherwise, we need to wrap over using & or % accordingly. 3299 */ 3300 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3301 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3302 *copybuf_used; 3303 } else { 3304 if (copybuf_sz_power_2) { 3305 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3306 (uintptr_t)dma->dp_cbaddr + 3307 (*copybuf_used & 3308 (dma->dp_copybuf_size - 1))); 3309 } else { 3310 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3311 (uintptr_t)dma->dp_cbaddr + 3312 (*copybuf_used % dma->dp_copybuf_size)); 3313 } 3314 } 3315 3316 /* 3317 * over write the cookie physical address with the address of 3318 * the physical address of the copy buffer page that we will 3319 * use. 3320 */ 3321 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3322 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3323 3324 #ifdef __xpv 3325 /* 3326 * If we're dom0, we're using a real device so we need to load 3327 * the cookies with MAs instead of PAs. 3328 */ 3329 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3330 #else 3331 cookie->dmac_laddress = paddr; 3332 #endif 3333 3334 /* if we have a kernel VA, it's easy, just save that address */ 3335 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3336 (sinfo->si_asp == &kas)) { 3337 /* 3338 * save away the page aligned virtual address of the 3339 * driver buffer. Offsets are handled in the sync code. 3340 */ 3341 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3342 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3343 & MMU_PAGEMASK); 3344 #if !defined(__amd64) 3345 /* 3346 * we didn't need to, and will never need to map this 3347 * page. 3348 */ 3349 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3350 #endif 3351 3352 /* we don't have a kernel VA. We need one for the bcopy. */ 3353 } else { 3354 #if defined(__amd64) 3355 /* 3356 * for the 64-bit kernel, it's easy. We use seg kpm to 3357 * get a Kernel VA for the corresponding pfn. 3358 */ 3359 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3360 #else 3361 /* 3362 * for the 32-bit kernel, this is a pain. First we'll 3363 * save away the page_t or user VA for this page. This 3364 * is needed in rootnex_dma_win() when we switch to a 3365 * new window which requires us to re-map the copy 3366 * buffer. 3367 */ 3368 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3369 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3370 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3371 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3372 } else if (pplist != NULL) { 3373 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3374 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3375 } else { 3376 dma->dp_pgmap[pidx].pm_pp = NULL; 3377 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3378 (((uintptr_t) 3379 dmar_object->dmao_obj.virt_obj.v_addr + 3380 cur_offset) & MMU_PAGEMASK); 3381 } 3382 3383 /* 3384 * save away the page aligned virtual address which was 3385 * allocated from the kernel heap arena (taking into 3386 * account if we need more copy buffer than we alloced 3387 * and use multiple windows to handle this, i.e. &,%). 3388 * NOTE: there isn't and physical memory backing up this 3389 * virtual address space currently. 3390 */ 3391 if ((*copybuf_used + MMU_PAGESIZE) <= 3392 dma->dp_copybuf_size) { 3393 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3394 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3395 MMU_PAGEMASK); 3396 } else { 3397 if (copybuf_sz_power_2) { 3398 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3399 (((uintptr_t)dma->dp_kva + 3400 (*copybuf_used & 3401 (dma->dp_copybuf_size - 1))) & 3402 MMU_PAGEMASK); 3403 } else { 3404 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3405 (((uintptr_t)dma->dp_kva + 3406 (*copybuf_used % 3407 dma->dp_copybuf_size)) & 3408 MMU_PAGEMASK); 3409 } 3410 } 3411 3412 /* 3413 * if we haven't used up the available copy buffer yet, 3414 * map the kva to the physical page. 3415 */ 3416 if (!dma->dp_cb_remaping && ((*copybuf_used + 3417 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3418 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3419 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3420 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3421 dma->dp_pgmap[pidx].pm_kaddr); 3422 } else { 3423 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3424 sinfo->si_asp, 3425 dma->dp_pgmap[pidx].pm_kaddr); 3426 } 3427 3428 /* 3429 * we've used up the available copy buffer, this page 3430 * will have to be mapped during rootnex_dma_win() when 3431 * we switch to a new window which requires a re-map 3432 * the copy buffer. (32-bit kernel only) 3433 */ 3434 } else { 3435 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3436 } 3437 #endif 3438 /* go to the next page_t */ 3439 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3440 *cur_pp = (*cur_pp)->p_next; 3441 } 3442 } 3443 3444 /* add to the copy buffer count */ 3445 *copybuf_used += MMU_PAGESIZE; 3446 3447 /* 3448 * This cookie doesn't use the copy buffer. Walk through the pages this 3449 * cookie occupies to reflect this. 3450 */ 3451 } else { 3452 /* 3453 * figure out how many pages the cookie occupies. We need to 3454 * use the original page offset of the buffer and the cookies 3455 * offset in the buffer to do this. 3456 */ 3457 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3458 pcnt = mmu_btopr(cookie->dmac_size + poff); 3459 3460 while (pcnt > 0) { 3461 #if !defined(__amd64) 3462 /* 3463 * the 32-bit kernel doesn't have seg kpm, so we need 3464 * to map in the driver buffer (if it didn't come down 3465 * with a kernel VA) on the fly. Since this page doesn't 3466 * use the copy buffer, it's not, or will it ever, have 3467 * to be mapped in. 3468 */ 3469 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3470 #endif 3471 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3472 3473 /* 3474 * we need to update pidx and cur_pp or we'll loose 3475 * track of where we are. 3476 */ 3477 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3478 *cur_pp = (*cur_pp)->p_next; 3479 } 3480 pidx++; 3481 pcnt--; 3482 } 3483 } 3484 } 3485 3486 3487 /* 3488 * rootnex_sgllen_window_boundary() 3489 * Called in the bind slow path when the next cookie causes us to exceed (in 3490 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3491 * length supported by the DMA H/W. 3492 */ 3493 static int 3494 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3495 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3496 off_t cur_offset) 3497 { 3498 off_t new_offset; 3499 size_t trim_sz; 3500 off_t coffset; 3501 3502 3503 /* 3504 * if we know we'll never have to trim, it's pretty easy. Just move to 3505 * the next window and init it. We're done. 3506 */ 3507 if (!dma->dp_trim_required) { 3508 (*windowp)++; 3509 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3510 (*windowp)->wd_cookie_cnt++; 3511 (*windowp)->wd_size = cookie->dmac_size; 3512 return (DDI_SUCCESS); 3513 } 3514 3515 /* figure out how much we need to trim from the window */ 3516 ASSERT(attr->dma_attr_granular != 0); 3517 if (dma->dp_granularity_power_2) { 3518 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3519 } else { 3520 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3521 } 3522 3523 /* The window's a whole multiple of granularity. We're done */ 3524 if (trim_sz == 0) { 3525 (*windowp)++; 3526 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3527 (*windowp)->wd_cookie_cnt++; 3528 (*windowp)->wd_size = cookie->dmac_size; 3529 return (DDI_SUCCESS); 3530 } 3531 3532 /* 3533 * The window's not a whole multiple of granularity, since we know this 3534 * is due to the sgllen, we need to go back to the last cookie and trim 3535 * that one, add the left over part of the old cookie into the new 3536 * window, and then add in the new cookie into the new window. 3537 */ 3538 3539 /* 3540 * make sure the driver isn't making us do something bad... Trimming and 3541 * sgllen == 1 don't go together. 3542 */ 3543 if (attr->dma_attr_sgllen == 1) { 3544 return (DDI_DMA_NOMAPPING); 3545 } 3546 3547 /* 3548 * first, setup the current window to account for the trim. Need to go 3549 * back to the last cookie for this. 3550 */ 3551 cookie--; 3552 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3553 (*windowp)->wd_trim.tr_last_cookie = cookie; 3554 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3555 ASSERT(cookie->dmac_size > trim_sz); 3556 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3557 (*windowp)->wd_size -= trim_sz; 3558 3559 /* save the buffer offsets for the next window */ 3560 coffset = cookie->dmac_size - trim_sz; 3561 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3562 3563 /* 3564 * set this now in case this is the first window. all other cases are 3565 * set in dma_win() 3566 */ 3567 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3568 3569 /* 3570 * initialize the next window using what's left over in the previous 3571 * cookie. 3572 */ 3573 (*windowp)++; 3574 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3575 (*windowp)->wd_cookie_cnt++; 3576 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3577 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3578 (*windowp)->wd_trim.tr_first_size = trim_sz; 3579 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3580 (*windowp)->wd_dosync = B_TRUE; 3581 } 3582 3583 /* 3584 * now go back to the current cookie and add it to the new window. set 3585 * the new window size to the what was left over from the previous 3586 * cookie and what's in the current cookie. 3587 */ 3588 cookie++; 3589 (*windowp)->wd_cookie_cnt++; 3590 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3591 3592 /* 3593 * trim plus the next cookie could put us over maxxfer (a cookie can be 3594 * a max size of maxxfer). Handle that case. 3595 */ 3596 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3597 /* 3598 * maxxfer is already a whole multiple of granularity, and this 3599 * trim will be <= the previous trim (since a cookie can't be 3600 * larger than maxxfer). Make things simple here. 3601 */ 3602 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3603 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3604 (*windowp)->wd_trim.tr_last_cookie = cookie; 3605 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3606 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3607 (*windowp)->wd_size -= trim_sz; 3608 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3609 3610 /* save the buffer offsets for the next window */ 3611 coffset = cookie->dmac_size - trim_sz; 3612 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3613 3614 /* setup the next window */ 3615 (*windowp)++; 3616 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3617 (*windowp)->wd_cookie_cnt++; 3618 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3619 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3620 coffset; 3621 (*windowp)->wd_trim.tr_first_size = trim_sz; 3622 } 3623 3624 return (DDI_SUCCESS); 3625 } 3626 3627 3628 /* 3629 * rootnex_copybuf_window_boundary() 3630 * Called in bind slowpath when we get to a window boundary because we used 3631 * up all the copy buffer that we have. 3632 */ 3633 static int 3634 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3635 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3636 size_t *copybuf_used) 3637 { 3638 rootnex_sglinfo_t *sinfo; 3639 off_t new_offset; 3640 size_t trim_sz; 3641 paddr_t paddr; 3642 off_t coffset; 3643 uint_t pidx; 3644 off_t poff; 3645 3646 3647 sinfo = &dma->dp_sglinfo; 3648 3649 /* 3650 * the copy buffer should be a whole multiple of page size. We know that 3651 * this cookie is <= MMU_PAGESIZE. 3652 */ 3653 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3654 3655 /* 3656 * from now on, all new windows in this bind need to be re-mapped during 3657 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3658 * space... 3659 */ 3660 #if !defined(__amd64) 3661 dma->dp_cb_remaping = B_TRUE; 3662 #endif 3663 3664 /* reset copybuf used */ 3665 *copybuf_used = 0; 3666 3667 /* 3668 * if we don't have to trim (since granularity is set to 1), go to the 3669 * next window and add the current cookie to it. We know the current 3670 * cookie uses the copy buffer since we're in this code path. 3671 */ 3672 if (!dma->dp_trim_required) { 3673 (*windowp)++; 3674 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3675 3676 /* Add this cookie to the new window */ 3677 (*windowp)->wd_cookie_cnt++; 3678 (*windowp)->wd_size += cookie->dmac_size; 3679 *copybuf_used += MMU_PAGESIZE; 3680 return (DDI_SUCCESS); 3681 } 3682 3683 /* 3684 * *** may need to trim, figure it out. 3685 */ 3686 3687 /* figure out how much we need to trim from the window */ 3688 if (dma->dp_granularity_power_2) { 3689 trim_sz = (*windowp)->wd_size & 3690 (hp->dmai_attr.dma_attr_granular - 1); 3691 } else { 3692 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3693 } 3694 3695 /* 3696 * if the window's a whole multiple of granularity, go to the next 3697 * window, init it, then add in the current cookie. We know the current 3698 * cookie uses the copy buffer since we're in this code path. 3699 */ 3700 if (trim_sz == 0) { 3701 (*windowp)++; 3702 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3703 3704 /* Add this cookie to the new window */ 3705 (*windowp)->wd_cookie_cnt++; 3706 (*windowp)->wd_size += cookie->dmac_size; 3707 *copybuf_used += MMU_PAGESIZE; 3708 return (DDI_SUCCESS); 3709 } 3710 3711 /* 3712 * *** We figured it out, we definitly need to trim 3713 */ 3714 3715 /* 3716 * make sure the driver isn't making us do something bad... 3717 * Trimming and sgllen == 1 don't go together. 3718 */ 3719 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3720 return (DDI_DMA_NOMAPPING); 3721 } 3722 3723 /* 3724 * first, setup the current window to account for the trim. Need to go 3725 * back to the last cookie for this. Some of the last cookie will be in 3726 * the current window, and some of the last cookie will be in the new 3727 * window. All of the current cookie will be in the new window. 3728 */ 3729 cookie--; 3730 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3731 (*windowp)->wd_trim.tr_last_cookie = cookie; 3732 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3733 ASSERT(cookie->dmac_size > trim_sz); 3734 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3735 (*windowp)->wd_size -= trim_sz; 3736 3737 /* 3738 * we're trimming the last cookie (not the current cookie). So that 3739 * last cookie may have or may not have been using the copy buffer ( 3740 * we know the cookie passed in uses the copy buffer since we're in 3741 * this code path). 3742 * 3743 * If the last cookie doesn't use the copy buffer, nothing special to 3744 * do. However, if it does uses the copy buffer, it will be both the 3745 * last page in the current window and the first page in the next 3746 * window. Since we are reusing the copy buffer (and KVA space on the 3747 * 32-bit kernel), this page will use the end of the copy buffer in the 3748 * current window, and the start of the copy buffer in the next window. 3749 * Track that info... The cookie physical address was already set to 3750 * the copy buffer physical address in setup_cookie.. 3751 */ 3752 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3753 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3754 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3755 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3756 (*windowp)->wd_trim.tr_last_pidx = pidx; 3757 (*windowp)->wd_trim.tr_last_cbaddr = 3758 dma->dp_pgmap[pidx].pm_cbaddr; 3759 #if !defined(__amd64) 3760 (*windowp)->wd_trim.tr_last_kaddr = 3761 dma->dp_pgmap[pidx].pm_kaddr; 3762 #endif 3763 } 3764 3765 /* save the buffer offsets for the next window */ 3766 coffset = cookie->dmac_size - trim_sz; 3767 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3768 3769 /* 3770 * set this now in case this is the first window. all other cases are 3771 * set in dma_win() 3772 */ 3773 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3774 3775 /* 3776 * initialize the next window using what's left over in the previous 3777 * cookie. 3778 */ 3779 (*windowp)++; 3780 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3781 (*windowp)->wd_cookie_cnt++; 3782 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3783 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3784 (*windowp)->wd_trim.tr_first_size = trim_sz; 3785 3786 /* 3787 * again, we're tracking if the last cookie uses the copy buffer. 3788 * read the comment above for more info on why we need to track 3789 * additional state. 3790 * 3791 * For the first cookie in the new window, we need reset the physical 3792 * address to DMA into to the start of the copy buffer plus any 3793 * initial page offset which may be present. 3794 */ 3795 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3796 (*windowp)->wd_dosync = B_TRUE; 3797 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3798 (*windowp)->wd_trim.tr_first_pidx = pidx; 3799 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3800 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3801 3802 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3803 poff; 3804 #ifdef __xpv 3805 /* 3806 * If we're dom0, we're using a real device so we need to load 3807 * the cookies with MAs instead of PAs. 3808 */ 3809 (*windowp)->wd_trim.tr_first_paddr = 3810 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3811 #else 3812 (*windowp)->wd_trim.tr_first_paddr = paddr; 3813 #endif 3814 3815 #if !defined(__amd64) 3816 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3817 #endif 3818 /* account for the cookie copybuf usage in the new window */ 3819 *copybuf_used += MMU_PAGESIZE; 3820 3821 /* 3822 * every piece of code has to have a hack, and here is this 3823 * ones :-) 3824 * 3825 * There is a complex interaction between setup_cookie and the 3826 * copybuf window boundary. The complexity had to be in either 3827 * the maxxfer window, or the copybuf window, and I chose the 3828 * copybuf code. 3829 * 3830 * So in this code path, we have taken the last cookie, 3831 * virtually broken it in half due to the trim, and it happens 3832 * to use the copybuf which further complicates life. At the 3833 * same time, we have already setup the current cookie, which 3834 * is now wrong. More background info: the current cookie uses 3835 * the copybuf, so it is only a page long max. So we need to 3836 * fix the current cookies copy buffer address, physical 3837 * address, and kva for the 32-bit kernel. We due this by 3838 * bumping them by page size (of course, we can't due this on 3839 * the physical address since the copy buffer may not be 3840 * physically contiguous). 3841 */ 3842 cookie++; 3843 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3844 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3845 3846 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3847 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3848 #ifdef __xpv 3849 /* 3850 * If we're dom0, we're using a real device so we need to load 3851 * the cookies with MAs instead of PAs. 3852 */ 3853 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3854 #else 3855 cookie->dmac_laddress = paddr; 3856 #endif 3857 3858 #if !defined(__amd64) 3859 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3860 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3861 #endif 3862 } else { 3863 /* go back to the current cookie */ 3864 cookie++; 3865 } 3866 3867 /* 3868 * add the current cookie to the new window. set the new window size to 3869 * the what was left over from the previous cookie and what's in the 3870 * current cookie. 3871 */ 3872 (*windowp)->wd_cookie_cnt++; 3873 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3874 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3875 3876 /* 3877 * we know that the cookie passed in always uses the copy buffer. We 3878 * wouldn't be here if it didn't. 3879 */ 3880 *copybuf_used += MMU_PAGESIZE; 3881 3882 return (DDI_SUCCESS); 3883 } 3884 3885 3886 /* 3887 * rootnex_maxxfer_window_boundary() 3888 * Called in bind slowpath when we get to a window boundary because we will 3889 * go over maxxfer. 3890 */ 3891 static int 3892 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3893 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3894 { 3895 size_t dmac_size; 3896 off_t new_offset; 3897 size_t trim_sz; 3898 off_t coffset; 3899 3900 3901 /* 3902 * calculate how much we have to trim off of the current cookie to equal 3903 * maxxfer. We don't have to account for granularity here since our 3904 * maxxfer already takes that into account. 3905 */ 3906 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3907 ASSERT(trim_sz <= cookie->dmac_size); 3908 ASSERT(trim_sz <= dma->dp_maxxfer); 3909 3910 /* save cookie size since we need it later and we might change it */ 3911 dmac_size = cookie->dmac_size; 3912 3913 /* 3914 * if we're not trimming the entire cookie, setup the current window to 3915 * account for the trim. 3916 */ 3917 if (trim_sz < cookie->dmac_size) { 3918 (*windowp)->wd_cookie_cnt++; 3919 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3920 (*windowp)->wd_trim.tr_last_cookie = cookie; 3921 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3922 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3923 (*windowp)->wd_size = dma->dp_maxxfer; 3924 3925 /* 3926 * set the adjusted cookie size now in case this is the first 3927 * window. All other windows are taken care of in get win 3928 */ 3929 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3930 } 3931 3932 /* 3933 * coffset is the current offset within the cookie, new_offset is the 3934 * current offset with the entire buffer. 3935 */ 3936 coffset = dmac_size - trim_sz; 3937 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3938 3939 /* initialize the next window */ 3940 (*windowp)++; 3941 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3942 (*windowp)->wd_cookie_cnt++; 3943 (*windowp)->wd_size = trim_sz; 3944 if (trim_sz < dmac_size) { 3945 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3946 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3947 coffset; 3948 (*windowp)->wd_trim.tr_first_size = trim_sz; 3949 } 3950 3951 return (DDI_SUCCESS); 3952 } 3953 3954 3955 /*ARGSUSED*/ 3956 static int 3957 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3958 off_t off, size_t len, uint_t cache_flags) 3959 { 3960 rootnex_sglinfo_t *sinfo; 3961 rootnex_pgmap_t *cbpage; 3962 rootnex_window_t *win; 3963 ddi_dma_impl_t *hp; 3964 rootnex_dma_t *dma; 3965 caddr_t fromaddr; 3966 caddr_t toaddr; 3967 uint_t psize; 3968 off_t offset; 3969 uint_t pidx; 3970 size_t size; 3971 off_t poff; 3972 int e; 3973 3974 3975 hp = (ddi_dma_impl_t *)handle; 3976 dma = (rootnex_dma_t *)hp->dmai_private; 3977 sinfo = &dma->dp_sglinfo; 3978 3979 /* 3980 * if we don't have any windows, we don't need to sync. A copybuf 3981 * will cause us to have at least one window. 3982 */ 3983 if (dma->dp_window == NULL) { 3984 return (DDI_SUCCESS); 3985 } 3986 3987 /* This window may not need to be sync'd */ 3988 win = &dma->dp_window[dma->dp_current_win]; 3989 if (!win->wd_dosync) { 3990 return (DDI_SUCCESS); 3991 } 3992 3993 /* handle off and len special cases */ 3994 if ((off == 0) || (rootnex_sync_ignore_params)) { 3995 offset = win->wd_offset; 3996 } else { 3997 offset = off; 3998 } 3999 if ((len == 0) || (rootnex_sync_ignore_params)) { 4000 size = win->wd_size; 4001 } else { 4002 size = len; 4003 } 4004 4005 /* check the sync args to make sure they make a little sense */ 4006 if (rootnex_sync_check_parms) { 4007 e = rootnex_valid_sync_parms(hp, win, offset, size, 4008 cache_flags); 4009 if (e != DDI_SUCCESS) { 4010 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4011 return (DDI_FAILURE); 4012 } 4013 } 4014 4015 /* 4016 * special case the first page to handle the offset into the page. The 4017 * offset to the current page for our buffer is the offset into the 4018 * first page of the buffer plus our current offset into the buffer 4019 * itself, masked of course. 4020 */ 4021 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4022 psize = MIN((MMU_PAGESIZE - poff), size); 4023 4024 /* go through all the pages that we want to sync */ 4025 while (size > 0) { 4026 /* 4027 * Calculate the page index relative to the start of the buffer. 4028 * The index to the current page for our buffer is the offset 4029 * into the first page of the buffer plus our current offset 4030 * into the buffer itself, shifted of course... 4031 */ 4032 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4033 ASSERT(pidx < sinfo->si_max_pages); 4034 4035 /* 4036 * if this page uses the copy buffer, we need to sync it, 4037 * otherwise, go on to the next page. 4038 */ 4039 cbpage = &dma->dp_pgmap[pidx]; 4040 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4041 (cbpage->pm_uses_copybuf == B_FALSE)); 4042 if (cbpage->pm_uses_copybuf) { 4043 /* cbaddr and kaddr should be page aligned */ 4044 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4045 MMU_PAGEOFFSET) == 0); 4046 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4047 MMU_PAGEOFFSET) == 0); 4048 4049 /* 4050 * if we're copying for the device, we are going to 4051 * copy from the drivers buffer and to the rootnex 4052 * allocated copy buffer. 4053 */ 4054 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4055 fromaddr = cbpage->pm_kaddr + poff; 4056 toaddr = cbpage->pm_cbaddr + poff; 4057 DTRACE_PROBE2(rootnex__sync__dev, 4058 dev_info_t *, dma->dp_dip, size_t, psize); 4059 4060 /* 4061 * if we're copying for the cpu/kernel, we are going to 4062 * copy from the rootnex allocated copy buffer to the 4063 * drivers buffer. 4064 */ 4065 } else { 4066 fromaddr = cbpage->pm_cbaddr + poff; 4067 toaddr = cbpage->pm_kaddr + poff; 4068 DTRACE_PROBE2(rootnex__sync__cpu, 4069 dev_info_t *, dma->dp_dip, size_t, psize); 4070 } 4071 4072 bcopy(fromaddr, toaddr, psize); 4073 } 4074 4075 /* 4076 * decrement size until we're done, update our offset into the 4077 * buffer, and get the next page size. 4078 */ 4079 size -= psize; 4080 offset += psize; 4081 psize = MIN(MMU_PAGESIZE, size); 4082 4083 /* page offset is zero for the rest of this loop */ 4084 poff = 0; 4085 } 4086 4087 return (DDI_SUCCESS); 4088 } 4089 4090 /* 4091 * rootnex_dma_sync() 4092 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4093 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4094 * is set, ddi_dma_sync() returns immediately passing back success. 4095 */ 4096 /*ARGSUSED*/ 4097 static int 4098 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4099 off_t off, size_t len, uint_t cache_flags) 4100 { 4101 #if !defined(__xpv) 4102 if (IOMMU_USED(rdip)) { 4103 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4104 cache_flags)); 4105 } 4106 #endif 4107 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4108 cache_flags)); 4109 } 4110 4111 /* 4112 * rootnex_valid_sync_parms() 4113 * checks the parameters passed to sync to verify they are correct. 4114 */ 4115 static int 4116 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4117 off_t offset, size_t size, uint_t cache_flags) 4118 { 4119 off_t woffset; 4120 4121 4122 /* 4123 * the first part of the test to make sure the offset passed in is 4124 * within the window. 4125 */ 4126 if (offset < win->wd_offset) { 4127 return (DDI_FAILURE); 4128 } 4129 4130 /* 4131 * second and last part of the test to make sure the offset and length 4132 * passed in is within the window. 4133 */ 4134 woffset = offset - win->wd_offset; 4135 if ((woffset + size) > win->wd_size) { 4136 return (DDI_FAILURE); 4137 } 4138 4139 /* 4140 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4141 * be set too. 4142 */ 4143 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4144 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4145 return (DDI_SUCCESS); 4146 } 4147 4148 /* 4149 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4150 * should be set. Also DDI_DMA_READ should be set in the flags. 4151 */ 4152 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4153 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4154 (hp->dmai_rflags & DDI_DMA_READ)) { 4155 return (DDI_SUCCESS); 4156 } 4157 4158 return (DDI_FAILURE); 4159 } 4160 4161 4162 /*ARGSUSED*/ 4163 static int 4164 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4165 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4166 uint_t *ccountp) 4167 { 4168 rootnex_window_t *window; 4169 rootnex_trim_t *trim; 4170 ddi_dma_impl_t *hp; 4171 rootnex_dma_t *dma; 4172 #if !defined(__amd64) 4173 rootnex_sglinfo_t *sinfo; 4174 rootnex_pgmap_t *pmap; 4175 uint_t pidx; 4176 uint_t pcnt; 4177 off_t poff; 4178 int i; 4179 #endif 4180 4181 4182 hp = (ddi_dma_impl_t *)handle; 4183 dma = (rootnex_dma_t *)hp->dmai_private; 4184 #if !defined(__amd64) 4185 sinfo = &dma->dp_sglinfo; 4186 #endif 4187 4188 /* If we try and get a window which doesn't exist, return failure */ 4189 if (win >= hp->dmai_nwin) { 4190 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4191 return (DDI_FAILURE); 4192 } 4193 4194 /* 4195 * if we don't have any windows, and they're asking for the first 4196 * window, setup the cookie pointer to the first cookie in the bind. 4197 * setup our return values, then increment the cookie since we return 4198 * the first cookie on the stack. 4199 */ 4200 if (dma->dp_window == NULL) { 4201 if (win != 0) { 4202 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4203 return (DDI_FAILURE); 4204 } 4205 hp->dmai_cookie = dma->dp_cookies; 4206 *offp = 0; 4207 *lenp = dma->dp_dma.dmao_size; 4208 *ccountp = dma->dp_sglinfo.si_sgl_size; 4209 *cookiep = hp->dmai_cookie[0]; 4210 hp->dmai_cookie++; 4211 return (DDI_SUCCESS); 4212 } 4213 4214 /* sync the old window before moving on to the new one */ 4215 window = &dma->dp_window[dma->dp_current_win]; 4216 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4217 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4218 DDI_DMA_SYNC_FORCPU); 4219 } 4220 4221 #if !defined(__amd64) 4222 /* 4223 * before we move to the next window, if we need to re-map, unmap all 4224 * the pages in this window. 4225 */ 4226 if (dma->dp_cb_remaping) { 4227 /* 4228 * If we switch to this window again, we'll need to map in 4229 * on the fly next time. 4230 */ 4231 window->wd_remap_copybuf = B_TRUE; 4232 4233 /* 4234 * calculate the page index into the buffer where this window 4235 * starts, and the number of pages this window takes up. 4236 */ 4237 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4238 MMU_PAGESHIFT; 4239 poff = (sinfo->si_buf_offset + window->wd_offset) & 4240 MMU_PAGEOFFSET; 4241 pcnt = mmu_btopr(window->wd_size + poff); 4242 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4243 4244 /* unmap pages which are currently mapped in this window */ 4245 for (i = 0; i < pcnt; i++) { 4246 if (dma->dp_pgmap[pidx].pm_mapped) { 4247 hat_unload(kas.a_hat, 4248 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4249 HAT_UNLOAD); 4250 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4251 } 4252 pidx++; 4253 } 4254 } 4255 #endif 4256 4257 /* 4258 * Move to the new window. 4259 * NOTE: current_win must be set for sync to work right 4260 */ 4261 dma->dp_current_win = win; 4262 window = &dma->dp_window[win]; 4263 4264 /* if needed, adjust the first and/or last cookies for trim */ 4265 trim = &window->wd_trim; 4266 if (trim->tr_trim_first) { 4267 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4268 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4269 #if !defined(__amd64) 4270 window->wd_first_cookie->dmac_type = 4271 (window->wd_first_cookie->dmac_type & 4272 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4273 #endif 4274 if (trim->tr_first_copybuf_win) { 4275 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4276 trim->tr_first_cbaddr; 4277 #if !defined(__amd64) 4278 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4279 trim->tr_first_kaddr; 4280 #endif 4281 } 4282 } 4283 if (trim->tr_trim_last) { 4284 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4285 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4286 if (trim->tr_last_copybuf_win) { 4287 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4288 trim->tr_last_cbaddr; 4289 #if !defined(__amd64) 4290 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4291 trim->tr_last_kaddr; 4292 #endif 4293 } 4294 } 4295 4296 /* 4297 * setup the cookie pointer to the first cookie in the window. setup 4298 * our return values, then increment the cookie since we return the 4299 * first cookie on the stack. 4300 */ 4301 hp->dmai_cookie = window->wd_first_cookie; 4302 *offp = window->wd_offset; 4303 *lenp = window->wd_size; 4304 *ccountp = window->wd_cookie_cnt; 4305 *cookiep = hp->dmai_cookie[0]; 4306 hp->dmai_cookie++; 4307 4308 #if !defined(__amd64) 4309 /* re-map copybuf if required for this window */ 4310 if (dma->dp_cb_remaping) { 4311 /* 4312 * calculate the page index into the buffer where this 4313 * window starts. 4314 */ 4315 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4316 MMU_PAGESHIFT; 4317 ASSERT(pidx < sinfo->si_max_pages); 4318 4319 /* 4320 * the first page can get unmapped if it's shared with the 4321 * previous window. Even if the rest of this window is already 4322 * mapped in, we need to still check this one. 4323 */ 4324 pmap = &dma->dp_pgmap[pidx]; 4325 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4326 if (pmap->pm_pp != NULL) { 4327 pmap->pm_mapped = B_TRUE; 4328 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4329 } else if (pmap->pm_vaddr != NULL) { 4330 pmap->pm_mapped = B_TRUE; 4331 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4332 pmap->pm_kaddr); 4333 } 4334 } 4335 pidx++; 4336 4337 /* map in the rest of the pages if required */ 4338 if (window->wd_remap_copybuf) { 4339 window->wd_remap_copybuf = B_FALSE; 4340 4341 /* figure out many pages this window takes up */ 4342 poff = (sinfo->si_buf_offset + window->wd_offset) & 4343 MMU_PAGEOFFSET; 4344 pcnt = mmu_btopr(window->wd_size + poff); 4345 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4346 4347 /* map pages which require it */ 4348 for (i = 1; i < pcnt; i++) { 4349 pmap = &dma->dp_pgmap[pidx]; 4350 if (pmap->pm_uses_copybuf) { 4351 ASSERT(pmap->pm_mapped == B_FALSE); 4352 if (pmap->pm_pp != NULL) { 4353 pmap->pm_mapped = B_TRUE; 4354 i86_pp_map(pmap->pm_pp, 4355 pmap->pm_kaddr); 4356 } else if (pmap->pm_vaddr != NULL) { 4357 pmap->pm_mapped = B_TRUE; 4358 i86_va_map(pmap->pm_vaddr, 4359 sinfo->si_asp, 4360 pmap->pm_kaddr); 4361 } 4362 } 4363 pidx++; 4364 } 4365 } 4366 } 4367 #endif 4368 4369 /* if the new window uses the copy buffer, sync it for the device */ 4370 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4371 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4372 DDI_DMA_SYNC_FORDEV); 4373 } 4374 4375 return (DDI_SUCCESS); 4376 } 4377 4378 /* 4379 * rootnex_dma_win() 4380 * called from ddi_dma_getwin() 4381 */ 4382 /*ARGSUSED*/ 4383 static int 4384 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4385 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4386 uint_t *ccountp) 4387 { 4388 #if !defined(__xpv) 4389 if (IOMMU_USED(rdip)) { 4390 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4391 cookiep, ccountp)); 4392 } 4393 #endif 4394 4395 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4396 cookiep, ccountp)); 4397 } 4398 4399 /* 4400 * ************************ 4401 * obsoleted dma routines 4402 * ************************ 4403 */ 4404 4405 /* ARGSUSED */ 4406 static int 4407 rootnex_coredma_map(dev_info_t *dip, dev_info_t *rdip, 4408 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4409 { 4410 #if defined(__amd64) 4411 /* 4412 * this interface is not supported in 64-bit x86 kernel. See comment in 4413 * rootnex_dma_mctl() 4414 */ 4415 return (DDI_DMA_NORESOURCES); 4416 4417 #else /* 32-bit x86 kernel */ 4418 ddi_dma_handle_t *lhandlep; 4419 ddi_dma_handle_t lhandle; 4420 ddi_dma_cookie_t cookie; 4421 ddi_dma_attr_t dma_attr; 4422 ddi_dma_lim_t *dma_lim; 4423 uint_t ccnt; 4424 int e; 4425 4426 4427 /* 4428 * if the driver is just testing to see if it's possible to do the bind, 4429 * we'll use local state. Otherwise, use the handle pointer passed in. 4430 */ 4431 if (handlep == NULL) { 4432 lhandlep = &lhandle; 4433 } else { 4434 lhandlep = handlep; 4435 } 4436 4437 /* convert the limit structure to a dma_attr one */ 4438 dma_lim = dmareq->dmar_limits; 4439 dma_attr.dma_attr_version = DMA_ATTR_V0; 4440 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4441 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4442 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4443 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4444 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4445 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4446 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4447 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4448 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4449 dma_attr.dma_attr_align = MMU_PAGESIZE; 4450 dma_attr.dma_attr_flags = 0; 4451 4452 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4453 dmareq->dmar_arg, lhandlep); 4454 if (e != DDI_SUCCESS) { 4455 return (e); 4456 } 4457 4458 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4459 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4460 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4461 return (e); 4462 } 4463 4464 /* 4465 * if the driver is just testing to see if it's possible to do the bind, 4466 * free up the local state and return the result. 4467 */ 4468 if (handlep == NULL) { 4469 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4470 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4471 if (e == DDI_DMA_MAPPED) { 4472 return (DDI_DMA_MAPOK); 4473 } else { 4474 return (DDI_DMA_NOMAPPING); 4475 } 4476 } 4477 4478 return (e); 4479 #endif /* defined(__amd64) */ 4480 } 4481 4482 /* 4483 * rootnex_dma_map() 4484 * called from ddi_dma_setup() 4485 */ 4486 /* ARGSUSED */ 4487 static int 4488 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4489 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4490 { 4491 #if !defined(__xpv) 4492 if (IOMMU_USED(rdip)) { 4493 return (iommulib_nexdma_map(dip, rdip, dmareq, handlep)); 4494 } 4495 #endif 4496 return (rootnex_coredma_map(dip, rdip, dmareq, handlep)); 4497 } 4498 4499 /* 4500 * rootnex_dma_mctl() 4501 * 4502 */ 4503 /* ARGSUSED */ 4504 static int 4505 rootnex_coredma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4506 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4507 uint_t cache_flags) 4508 { 4509 #if defined(__amd64) 4510 /* 4511 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4512 * common implementation in genunix, so they no longer have x86 4513 * specific functionality which called into dma_ctl. 4514 * 4515 * The rest of the obsoleted interfaces were never supported in the 4516 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4517 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4518 * implementation issues. 4519 * 4520 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4521 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4522 * reflect that now too... 4523 * 4524 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4525 * not going to put this functionality into the 64-bit x86 kernel now. 4526 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4527 * that in a future release. 4528 */ 4529 return (DDI_FAILURE); 4530 4531 #else /* 32-bit x86 kernel */ 4532 ddi_dma_cookie_t lcookie; 4533 ddi_dma_cookie_t *cookie; 4534 rootnex_window_t *window; 4535 ddi_dma_impl_t *hp; 4536 rootnex_dma_t *dma; 4537 uint_t nwin; 4538 uint_t ccnt; 4539 size_t len; 4540 off_t off; 4541 int e; 4542 4543 4544 /* 4545 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4546 * hacky since were optimizing for the current interfaces and so we can 4547 * cleanup the mess in genunix. Hopefully we will remove the this 4548 * obsoleted routines someday soon. 4549 */ 4550 4551 switch (request) { 4552 4553 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4554 hp = (ddi_dma_impl_t *)handle; 4555 cookie = (ddi_dma_cookie_t *)objpp; 4556 4557 /* 4558 * convert segment to cookie. We don't distinguish between the 4559 * two :-) 4560 */ 4561 *cookie = *hp->dmai_cookie; 4562 *lenp = cookie->dmac_size; 4563 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4564 return (DDI_SUCCESS); 4565 4566 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4567 hp = (ddi_dma_impl_t *)handle; 4568 dma = (rootnex_dma_t *)hp->dmai_private; 4569 4570 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4571 return (DDI_DMA_STALE); 4572 } 4573 4574 /* handle the case where we don't have any windows */ 4575 if (dma->dp_window == NULL) { 4576 /* 4577 * if seg == NULL, and we don't have any windows, 4578 * return the first cookie in the sgl. 4579 */ 4580 if (*lenp == NULL) { 4581 dma->dp_current_cookie = 0; 4582 hp->dmai_cookie = dma->dp_cookies; 4583 *objpp = (caddr_t)handle; 4584 return (DDI_SUCCESS); 4585 4586 /* if we have more cookies, go to the next cookie */ 4587 } else { 4588 if ((dma->dp_current_cookie + 1) >= 4589 dma->dp_sglinfo.si_sgl_size) { 4590 return (DDI_DMA_DONE); 4591 } 4592 dma->dp_current_cookie++; 4593 hp->dmai_cookie++; 4594 return (DDI_SUCCESS); 4595 } 4596 } 4597 4598 /* We have one or more windows */ 4599 window = &dma->dp_window[dma->dp_current_win]; 4600 4601 /* 4602 * if seg == NULL, return the first cookie in the current 4603 * window 4604 */ 4605 if (*lenp == NULL) { 4606 dma->dp_current_cookie = 0; 4607 hp->dmai_cookie = window->wd_first_cookie; 4608 4609 /* 4610 * go to the next cookie in the window then see if we done with 4611 * this window. 4612 */ 4613 } else { 4614 if ((dma->dp_current_cookie + 1) >= 4615 window->wd_cookie_cnt) { 4616 return (DDI_DMA_DONE); 4617 } 4618 dma->dp_current_cookie++; 4619 hp->dmai_cookie++; 4620 } 4621 *objpp = (caddr_t)handle; 4622 return (DDI_SUCCESS); 4623 4624 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4625 hp = (ddi_dma_impl_t *)handle; 4626 dma = (rootnex_dma_t *)hp->dmai_private; 4627 4628 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4629 return (DDI_DMA_STALE); 4630 } 4631 4632 /* if win == NULL, return the first window in the bind */ 4633 if (*offp == NULL) { 4634 nwin = 0; 4635 4636 /* 4637 * else, go to the next window then see if we're done with all 4638 * the windows. 4639 */ 4640 } else { 4641 nwin = dma->dp_current_win + 1; 4642 if (nwin >= hp->dmai_nwin) { 4643 return (DDI_DMA_DONE); 4644 } 4645 } 4646 4647 /* switch to the next window */ 4648 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4649 &lcookie, &ccnt); 4650 ASSERT(e == DDI_SUCCESS); 4651 if (e != DDI_SUCCESS) { 4652 return (DDI_DMA_STALE); 4653 } 4654 4655 /* reset the cookie back to the first cookie in the window */ 4656 if (dma->dp_window != NULL) { 4657 window = &dma->dp_window[dma->dp_current_win]; 4658 hp->dmai_cookie = window->wd_first_cookie; 4659 } else { 4660 hp->dmai_cookie = dma->dp_cookies; 4661 } 4662 4663 *objpp = (caddr_t)handle; 4664 return (DDI_SUCCESS); 4665 4666 case DDI_DMA_FREE: /* ddi_dma_free() */ 4667 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4668 (void) rootnex_dma_freehdl(dip, rdip, handle); 4669 if (rootnex_state->r_dvma_call_list_id) { 4670 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4671 } 4672 return (DDI_SUCCESS); 4673 4674 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4675 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4676 /* should never get here, handled in genunix */ 4677 ASSERT(0); 4678 return (DDI_FAILURE); 4679 4680 case DDI_DMA_KVADDR: 4681 case DDI_DMA_GETERR: 4682 case DDI_DMA_COFF: 4683 return (DDI_FAILURE); 4684 } 4685 4686 return (DDI_FAILURE); 4687 #endif /* defined(__amd64) */ 4688 } 4689 4690 /* 4691 * rootnex_dma_mctl() 4692 * 4693 */ 4694 /* ARGSUSED */ 4695 static int 4696 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4697 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4698 uint_t cache_flags) 4699 { 4700 #if !defined(__xpv) 4701 if (IOMMU_USED(rdip)) { 4702 return (iommulib_nexdma_mctl(dip, rdip, handle, request, offp, 4703 lenp, objpp, cache_flags)); 4704 } 4705 #endif 4706 4707 return (rootnex_coredma_mctl(dip, rdip, handle, request, offp, 4708 lenp, objpp, cache_flags)); 4709 } 4710 4711 /* 4712 * ********* 4713 * FMA Code 4714 * ********* 4715 */ 4716 4717 /* 4718 * rootnex_fm_init() 4719 * FMA init busop 4720 */ 4721 /* ARGSUSED */ 4722 static int 4723 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4724 ddi_iblock_cookie_t *ibc) 4725 { 4726 *ibc = rootnex_state->r_err_ibc; 4727 4728 return (ddi_system_fmcap); 4729 } 4730 4731 /* 4732 * rootnex_dma_check() 4733 * Function called after a dma fault occurred to find out whether the 4734 * fault address is associated with a driver that is able to handle faults 4735 * and recover from faults. 4736 */ 4737 /* ARGSUSED */ 4738 static int 4739 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4740 const void *not_used) 4741 { 4742 rootnex_window_t *window; 4743 uint64_t start_addr; 4744 uint64_t fault_addr; 4745 ddi_dma_impl_t *hp; 4746 rootnex_dma_t *dma; 4747 uint64_t end_addr; 4748 size_t csize; 4749 int i; 4750 int j; 4751 4752 4753 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4754 hp = (ddi_dma_impl_t *)handle; 4755 ASSERT(hp); 4756 4757 dma = (rootnex_dma_t *)hp->dmai_private; 4758 4759 /* Get the address that we need to search for */ 4760 fault_addr = *(uint64_t *)addr; 4761 4762 /* 4763 * if we don't have any windows, we can just walk through all the 4764 * cookies. 4765 */ 4766 if (dma->dp_window == NULL) { 4767 /* for each cookie */ 4768 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4769 /* 4770 * if the faulted address is within the physical address 4771 * range of the cookie, return DDI_FM_NONFATAL. 4772 */ 4773 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4774 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4775 dma->dp_cookies[i].dmac_size))) { 4776 return (DDI_FM_NONFATAL); 4777 } 4778 } 4779 4780 /* fault_addr not within this DMA handle */ 4781 return (DDI_FM_UNKNOWN); 4782 } 4783 4784 /* we have mutiple windows, walk through each window */ 4785 for (i = 0; i < hp->dmai_nwin; i++) { 4786 window = &dma->dp_window[i]; 4787 4788 /* Go through all the cookies in the window */ 4789 for (j = 0; j < window->wd_cookie_cnt; j++) { 4790 4791 start_addr = window->wd_first_cookie[j].dmac_laddress; 4792 csize = window->wd_first_cookie[j].dmac_size; 4793 4794 /* 4795 * if we are trimming the first cookie in the window, 4796 * and this is the first cookie, adjust the start 4797 * address and size of the cookie to account for the 4798 * trim. 4799 */ 4800 if (window->wd_trim.tr_trim_first && (j == 0)) { 4801 start_addr = window->wd_trim.tr_first_paddr; 4802 csize = window->wd_trim.tr_first_size; 4803 } 4804 4805 /* 4806 * if we are trimming the last cookie in the window, 4807 * and this is the last cookie, adjust the start 4808 * address and size of the cookie to account for the 4809 * trim. 4810 */ 4811 if (window->wd_trim.tr_trim_last && 4812 (j == (window->wd_cookie_cnt - 1))) { 4813 start_addr = window->wd_trim.tr_last_paddr; 4814 csize = window->wd_trim.tr_last_size; 4815 } 4816 4817 end_addr = start_addr + csize; 4818 4819 /* 4820 * if the faulted address is within the physical address 4821 * range of the cookie, return DDI_FM_NONFATAL. 4822 */ 4823 if ((fault_addr >= start_addr) && 4824 (fault_addr <= end_addr)) { 4825 return (DDI_FM_NONFATAL); 4826 } 4827 } 4828 } 4829 4830 /* fault_addr not within this DMA handle */ 4831 return (DDI_FM_UNKNOWN); 4832 } 4833