1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 * Copyright 2026 Oxide Computer Company 25 */ 26 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 28 /* All Rights Reserved */ 29 30 /* 31 * University Copyright- Copyright (c) 1982, 1986, 1988 32 * The Regents of the University of California 33 * All Rights Reserved 34 * 35 * University Acknowledgment- Portions of this document are derived from 36 * software developed by the University of California, Berkeley, and its 37 * contributors. 38 */ 39 40 #include <sys/types.h> 41 #include <sys/sysmacros.h> 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/uio.h> 45 #include <sys/errno.h> 46 #include <sys/vmsystm.h> 47 #include <sys/cmn_err.h> 48 #include <vm/as.h> 49 #include <vm/page.h> 50 51 #include <sys/dcopy.h> 52 53 int64_t uioa_maxpoll = -1; /* <0 = noblock, 0 = block, >0 = block after */ 54 #define UIO_DCOPY_CHANNEL 0 55 #define UIO_DCOPY_CMD 1 56 57 /* 58 * Move "n" bytes at byte address "p"; "rw" indicates the direction 59 * of the move, and the I/O parameters are provided in "uio", which is 60 * update to reflect the data which was moved. Returns 0 on success or 61 * a non-zero errno on failure. 62 */ 63 int 64 uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio) 65 { 66 struct iovec *iov; 67 ulong_t cnt; 68 int error; 69 70 while (n && uio->uio_resid) { 71 iov = uio->uio_iov; 72 cnt = MIN(iov->iov_len, n); 73 if (cnt == 0l) { 74 uio->uio_iov++; 75 uio->uio_iovcnt--; 76 continue; 77 } 78 switch (uio->uio_segflg) { 79 80 case UIO_USERSPACE: 81 case UIO_USERISPACE: 82 if (rw == UIO_READ) { 83 error = xcopyout_nta(p, iov->iov_base, cnt, 84 (uio->uio_extflg & UIO_COPY_CACHED)); 85 } else { 86 error = xcopyin_nta(iov->iov_base, p, cnt, 87 (uio->uio_extflg & UIO_COPY_CACHED)); 88 } 89 90 if (error) 91 return (error); 92 break; 93 94 case UIO_SYSSPACE: 95 if (rw == UIO_READ) 96 error = kcopy_nta(p, iov->iov_base, cnt, 97 (uio->uio_extflg & UIO_COPY_CACHED)); 98 else 99 error = kcopy_nta(iov->iov_base, p, cnt, 100 (uio->uio_extflg & UIO_COPY_CACHED)); 101 if (error) 102 return (error); 103 break; 104 } 105 iov->iov_base += cnt; 106 iov->iov_len -= cnt; 107 uio->uio_resid -= cnt; 108 uio->uio_loffset += cnt; 109 p = (caddr_t)p + cnt; 110 n -= cnt; 111 } 112 return (0); 113 } 114 115 /* 116 * Fault in the pages of the first n bytes specified by the uio structure. 117 * 1 byte in each page is touched and the uio struct is unmodified. Any 118 * error will terminate the process as this is only a best attempt to get 119 * the pages resident. 120 */ 121 void 122 uio_prefaultpages(ssize_t n, struct uio *uio) 123 { 124 struct iovec *iov; 125 ulong_t cnt, incr; 126 caddr_t p; 127 uint8_t tmp; 128 int iovcnt; 129 130 iov = uio->uio_iov; 131 iovcnt = uio->uio_iovcnt; 132 133 while ((n > 0) && (iovcnt > 0)) { 134 cnt = MIN(iov->iov_len, n); 135 if (cnt == 0) { 136 /* empty iov entry */ 137 iov++; 138 iovcnt--; 139 continue; 140 } 141 n -= cnt; 142 /* 143 * touch each page in this segment. 144 */ 145 p = iov->iov_base; 146 while (cnt) { 147 switch (uio->uio_segflg) { 148 case UIO_USERSPACE: 149 case UIO_USERISPACE: 150 if (fuword8(p, &tmp)) 151 return; 152 break; 153 case UIO_SYSSPACE: 154 if (kcopy(p, &tmp, 1)) 155 return; 156 break; 157 } 158 incr = MIN(cnt, PAGESIZE); 159 p += incr; 160 cnt -= incr; 161 } 162 /* 163 * touch the last byte in case it straddles a page. 164 */ 165 p--; 166 switch (uio->uio_segflg) { 167 case UIO_USERSPACE: 168 case UIO_USERISPACE: 169 if (fuword8(p, &tmp)) 170 return; 171 break; 172 case UIO_SYSSPACE: 173 if (kcopy(p, &tmp, 1)) 174 return; 175 break; 176 } 177 iov++; 178 iovcnt--; 179 } 180 } 181 182 /* 183 * same as uiomove() but doesn't modify uio structure. 184 * return in cbytes how many bytes were copied. 185 */ 186 int 187 uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes) 188 { 189 struct iovec *iov; 190 ulong_t cnt; 191 int error; 192 int iovcnt; 193 194 iovcnt = uio->uio_iovcnt; 195 *cbytes = 0; 196 197 for (iov = uio->uio_iov; n && iovcnt; iov++, iovcnt--) { 198 cnt = MIN(iov->iov_len, n); 199 if (cnt == 0) 200 continue; 201 202 switch (uio->uio_segflg) { 203 204 case UIO_USERSPACE: 205 case UIO_USERISPACE: 206 if (rw == UIO_READ) { 207 error = xcopyout_nta(p, iov->iov_base, cnt, 208 (uio->uio_extflg & UIO_COPY_CACHED)); 209 } else { 210 error = xcopyin_nta(iov->iov_base, p, cnt, 211 (uio->uio_extflg & UIO_COPY_CACHED)); 212 } 213 214 if (error) 215 return (error); 216 break; 217 218 case UIO_SYSSPACE: 219 if (rw == UIO_READ) 220 error = kcopy_nta(p, iov->iov_base, cnt, 221 (uio->uio_extflg & UIO_COPY_CACHED)); 222 else 223 error = kcopy_nta(iov->iov_base, p, cnt, 224 (uio->uio_extflg & UIO_COPY_CACHED)); 225 if (error) 226 return (error); 227 break; 228 } 229 p = (caddr_t)p + cnt; 230 n -= cnt; 231 *cbytes += cnt; 232 } 233 return (0); 234 } 235 236 /* 237 * transfer a character value into the address space 238 * delineated by a uio and update fields within the 239 * uio for next character. Return 0 for success, EFAULT 240 * for error. 241 */ 242 int 243 ureadc(int val, struct uio *uiop) 244 { 245 struct iovec *iovp; 246 unsigned char c; 247 248 /* 249 * first determine if uio is valid. uiop should be 250 * non-NULL and the resid count > 0. 251 */ 252 if (!(uiop && uiop->uio_resid > 0)) 253 return (EFAULT); 254 255 /* 256 * scan through iovecs until one is found that is non-empty. 257 * Return EFAULT if none found. 258 */ 259 while (uiop->uio_iovcnt > 0) { 260 iovp = uiop->uio_iov; 261 if (iovp->iov_len <= 0) { 262 uiop->uio_iovcnt--; 263 uiop->uio_iov++; 264 } else 265 break; 266 } 267 268 if (uiop->uio_iovcnt <= 0) 269 return (EFAULT); 270 271 /* 272 * Transfer character to uio space. 273 */ 274 275 c = (unsigned char) (val & 0xFF); 276 277 switch (uiop->uio_segflg) { 278 279 case UIO_USERISPACE: 280 case UIO_USERSPACE: 281 if (copyout(&c, iovp->iov_base, sizeof (unsigned char))) 282 return (EFAULT); 283 break; 284 285 case UIO_SYSSPACE: /* can do direct copy since kernel-kernel */ 286 *iovp->iov_base = c; 287 break; 288 289 default: 290 return (EFAULT); /* invalid segflg value */ 291 } 292 293 /* 294 * bump up/down iovec and uio members to reflect transfer. 295 */ 296 iovp->iov_base++; 297 iovp->iov_len--; 298 uiop->uio_resid--; 299 uiop->uio_loffset++; 300 return (0); /* success */ 301 } 302 303 /* 304 * return a character value from the address space 305 * delineated by a uio and update fields within the 306 * uio for next character. Return the character for success, 307 * -1 for error. 308 */ 309 int 310 uwritec(struct uio *uiop) 311 { 312 struct iovec *iovp; 313 unsigned char c; 314 315 /* 316 * verify we were passed a valid uio structure. 317 * (1) non-NULL uiop, (2) positive resid count 318 * (3) there is an iovec with positive length 319 */ 320 321 if (!(uiop && uiop->uio_resid > 0)) 322 return (-1); 323 324 while (uiop->uio_iovcnt > 0) { 325 iovp = uiop->uio_iov; 326 if (iovp->iov_len <= 0) { 327 uiop->uio_iovcnt--; 328 uiop->uio_iov++; 329 } else 330 break; 331 } 332 333 if (uiop->uio_iovcnt <= 0) 334 return (-1); 335 336 /* 337 * Get the character from the uio address space. 338 */ 339 switch (uiop->uio_segflg) { 340 341 case UIO_USERISPACE: 342 case UIO_USERSPACE: 343 if (copyin(iovp->iov_base, &c, sizeof (unsigned char))) 344 return (-1); 345 break; 346 347 case UIO_SYSSPACE: 348 c = *iovp->iov_base; 349 break; 350 351 default: 352 return (-1); /* invalid segflg */ 353 } 354 355 /* 356 * Adjust fields of iovec and uio appropriately. 357 */ 358 iovp->iov_base++; 359 iovp->iov_len--; 360 uiop->uio_resid--; 361 uiop->uio_loffset++; 362 return ((int)c & 0xFF); /* success */ 363 } 364 365 /* 366 * Drop the next n chars out of *uiop. 367 */ 368 void 369 uioskip(uio_t *uiop, size_t n) 370 { 371 if (n > uiop->uio_resid) 372 return; 373 while (n != 0) { 374 register iovec_t *iovp = uiop->uio_iov; 375 register size_t niovb = MIN(iovp->iov_len, n); 376 377 if (niovb == 0) { 378 uiop->uio_iov++; 379 uiop->uio_iovcnt--; 380 continue; 381 } 382 iovp->iov_base += niovb; 383 uiop->uio_loffset += niovb; 384 iovp->iov_len -= niovb; 385 uiop->uio_resid -= niovb; 386 n -= niovb; 387 } 388 } 389 390 /* 391 * Dup the suio into the duio and diovec of size diov_cnt. If diov 392 * is too small to dup suio then an error will be returned, else 0. 393 */ 394 int 395 uiodup(uio_t *suio, uio_t *duio, iovec_t *diov, int diov_cnt) 396 { 397 int ix; 398 iovec_t *siov = suio->uio_iov; 399 400 *duio = *suio; 401 for (ix = 0; ix < suio->uio_iovcnt; ix++) { 402 diov[ix] = siov[ix]; 403 if (ix >= diov_cnt) 404 return (1); 405 } 406 duio->uio_iov = diov; 407 return (0); 408 } 409 410 /* 411 * Copy n bytes from src to dst, where src lies in the address space 412 * indicated by seg and dst is in kernel space. This is the equivalent 413 * of ddi_copyin() for code which describes an address space with a 414 * uio_seg_t rather than with ioctl(9E) mode flags. As with copyin(), 415 * 0 is returned on success and -1 on failure. 416 */ 417 int 418 uio_copyin(const void *src, void *dst, size_t n, uio_seg_t seg) 419 { 420 switch (seg) { 421 case UIO_USERSPACE: 422 case UIO_USERISPACE: 423 return (copyin(src, dst, n)); 424 case UIO_SYSSPACE: 425 return (kcopy(src, dst, n) == 0 ? 0 : -1); 426 } 427 panic("uio_copyin: invalid segment type %d", seg); 428 } 429 430 /* 431 * The equivalent of ddi_copyout(), copying n bytes from the kernel 432 * address src to dst, which lies in the address space indicated by 433 * seg. Returns 0 on success and -1 on failure. 434 */ 435 int 436 uio_copyout(const void *src, void *dst, size_t n, uio_seg_t seg) 437 { 438 switch (seg) { 439 case UIO_USERSPACE: 440 case UIO_USERISPACE: 441 return (copyout(src, dst, n)); 442 case UIO_SYSSPACE: 443 return (kcopy(src, dst, n) == 0 ? 0 : -1); 444 } 445 panic("uio_copyout: invalid segment type %d", seg); 446 } 447 448 /* 449 * Shadow state for checking if a platform has hardware asynchronous 450 * copy capability and minimum copy size, e.g. Intel's I/OAT dma engine, 451 * 452 * Dcopy does a call-back to uioa_dcopy_enable() when a dma device calls 453 * into dcopy to register and uioa_dcopy_disable() when the device calls 454 * into dcopy to unregister. 455 */ 456 uioasync_t uioasync = {B_FALSE, 1024}; 457 458 void 459 uioa_dcopy_enable() 460 { 461 uioasync.enabled = B_TRUE; 462 } 463 464 void 465 uioa_dcopy_disable() 466 { 467 uioasync.enabled = B_FALSE; 468 } 469 470 /* 471 * Schedule an asynchronous move of "n" bytes at byte address "p", 472 * "rw" indicates the direction of the move, I/O parameters and 473 * async state are provided in "uioa" which is update to reflect 474 * the data which is to be moved. 475 * 476 * Returns 0 on success or a non-zero errno on failure. 477 * 478 * Note, while the uioasync APIs are general purpose in design 479 * the current implementation is Intel I/OAT specific. 480 */ 481 int 482 uioamove(void *p, size_t n, enum uio_rw rw, uioa_t *uioa) 483 { 484 int soff, doff; 485 uint64_t pa; 486 int cnt; 487 iovec_t *iov; 488 dcopy_handle_t channel; 489 dcopy_cmd_t cmd; 490 int ret = 0; 491 int dcopy_flags; 492 493 if (!(uioa->uioa_state & UIOA_ENABLED)) { 494 /* The uioa_t isn't enabled */ 495 return (ENXIO); 496 } 497 498 if (uioa->uio_segflg != UIO_USERSPACE || rw != UIO_READ) { 499 /* Only support to user-land from kernel */ 500 return (ENOTSUP); 501 } 502 503 504 channel = uioa->uioa_hwst[UIO_DCOPY_CHANNEL]; 505 cmd = uioa->uioa_hwst[UIO_DCOPY_CMD]; 506 dcopy_flags = DCOPY_NOSLEEP; 507 508 /* 509 * While source bytes and destination bytes. 510 */ 511 while (n > 0 && uioa->uio_resid > 0) { 512 iov = uioa->uio_iov; 513 if (iov->iov_len == 0l) { 514 uioa->uio_iov++; 515 uioa->uio_iovcnt--; 516 uioa->uioa_lcur++; 517 uioa->uioa_lppp = uioa->uioa_lcur->uioa_ppp; 518 continue; 519 } 520 /* 521 * While source bytes schedule an async 522 * dma for destination page by page. 523 */ 524 while (n > 0) { 525 /* Addr offset in page src/dst */ 526 soff = (uintptr_t)p & PAGEOFFSET; 527 doff = (uintptr_t)iov->iov_base & PAGEOFFSET; 528 /* Min copy count src and dst and page sized */ 529 cnt = MIN(n, iov->iov_len); 530 cnt = MIN(cnt, PAGESIZE - soff); 531 cnt = MIN(cnt, PAGESIZE - doff); 532 /* XXX if next page(s) contiguous could use multipage */ 533 534 /* 535 * if we have an old command, we want to link all 536 * other commands to the next command we alloced so 537 * we only need to track the last command but can 538 * still free them all. 539 */ 540 if (cmd != NULL) { 541 dcopy_flags |= DCOPY_ALLOC_LINK; 542 } 543 ret = dcopy_cmd_alloc(channel, dcopy_flags, &cmd); 544 if (ret != DCOPY_SUCCESS) { 545 /* Error of some sort */ 546 return (EIO); 547 } 548 uioa->uioa_hwst[UIO_DCOPY_CMD] = cmd; 549 550 ASSERT(cmd->dp_version == DCOPY_CMD_V0); 551 if (uioa_maxpoll >= 0) { 552 /* Blocking (>0 may be) used in uioafini() */ 553 cmd->dp_flags = DCOPY_CMD_INTR; 554 } else { 555 /* Non blocking uioafini() so no intr */ 556 cmd->dp_flags = DCOPY_CMD_NOFLAGS; 557 } 558 cmd->dp_cmd = DCOPY_CMD_COPY; 559 pa = ptob((uint64_t)hat_getpfnum(kas.a_hat, p)); 560 cmd->dp.copy.cc_source = pa + soff; 561 if (uioa->uioa_lcur->uioa_pfncnt == 0) { 562 /* Have a (page_t **) */ 563 pa = ptob((uint64_t)( 564 *(page_t **)uioa->uioa_lppp)->p_pagenum); 565 } else { 566 /* Have a (pfn_t *) */ 567 pa = ptob((uint64_t)( 568 *(pfn_t *)uioa->uioa_lppp)); 569 } 570 cmd->dp.copy.cc_dest = pa + doff; 571 cmd->dp.copy.cc_size = cnt; 572 ret = dcopy_cmd_post(cmd); 573 if (ret != DCOPY_SUCCESS) { 574 /* Error of some sort */ 575 return (EIO); 576 } 577 ret = 0; 578 579 /* If UIOA_POLL not set, set it */ 580 if (!(uioa->uioa_state & UIOA_POLL)) 581 uioa->uioa_state |= UIOA_POLL; 582 583 /* Update iov, uio, and local pointers/counters */ 584 iov->iov_base += cnt; 585 iov->iov_len -= cnt; 586 uioa->uio_resid -= cnt; 587 uioa->uioa_mbytes += cnt; 588 uioa->uio_loffset += cnt; 589 p = (caddr_t)p + cnt; 590 n -= cnt; 591 592 /* End of iovec? */ 593 if (iov->iov_len == 0) { 594 /* Yup, next iovec */ 595 break; 596 } 597 598 /* Next dst addr page? */ 599 if (doff + cnt == PAGESIZE) { 600 /* Yup, next page_t */ 601 uioa->uioa_lppp++; 602 } 603 } 604 } 605 606 return (ret); 607 } 608 609 /* 610 * Initialize a uioa_t for a given uio_t for the current user context, 611 * copy the common uio_t to the uioa_t, walk the shared iovec_t and 612 * lock down the user-land page(s) containing iovec_t data, then mapin 613 * user-land pages using segkpm. 614 */ 615 int 616 uioainit(uio_t *uiop, uioa_t *uioap) 617 { 618 caddr_t addr; 619 page_t **pages; 620 int off; 621 int len; 622 proc_t *procp = ttoproc(curthread); 623 struct as *as = procp->p_as; 624 iovec_t *iov = uiop->uio_iov; 625 int32_t iovcnt = uiop->uio_iovcnt; 626 uioa_page_t *locked = uioap->uioa_locked; 627 dcopy_handle_t channel; 628 int error; 629 630 if (! (uioap->uioa_state & UIOA_ALLOC)) { 631 /* Can only init() a freshly allocated uioa_t */ 632 return (EINVAL); 633 } 634 635 error = dcopy_alloc(DCOPY_NOSLEEP, &channel); 636 if (error == DCOPY_NORESOURCES) { 637 /* Turn off uioa */ 638 uioasync.enabled = B_FALSE; 639 return (ENODEV); 640 } 641 if (error != DCOPY_SUCCESS) { 642 /* Alloc failed */ 643 return (EIO); 644 } 645 646 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = channel; 647 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 648 649 /* Indicate uioa_t (will be) initialized */ 650 uioap->uioa_state = UIOA_INIT; 651 652 uioap->uioa_mbytes = 0; 653 654 /* uio_t/uioa_t uio_t common struct copy */ 655 *((uio_t *)uioap) = *uiop; 656 657 /* initialize *uiop->uio_iov */ 658 if (iovcnt > UIOA_IOV_MAX) { 659 /* Too big? */ 660 return (E2BIG); 661 } 662 uioap->uio_iov = iov; 663 uioap->uio_iovcnt = iovcnt; 664 665 /* Mark the uioap as such */ 666 uioap->uio_extflg |= UIO_ASYNC; 667 668 /* 669 * For each iovec_t, lock-down the page(s) backing the iovec_t 670 * and save the page_t list for phys addr use in uioamove(). 671 */ 672 iov = uiop->uio_iov; 673 iovcnt = uiop->uio_iovcnt; 674 while (iovcnt > 0) { 675 addr = iov->iov_base; 676 off = (uintptr_t)addr & PAGEOFFSET; 677 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 678 len = iov->iov_len + off; 679 680 /* Lock down page(s) for the iov span */ 681 if ((error = as_pagelock(as, &pages, 682 iov->iov_base, iov->iov_len, S_WRITE)) != 0) { 683 /* Error */ 684 goto cleanup; 685 } 686 687 if (pages == NULL) { 688 /* 689 * Need page_t list, really only need 690 * a pfn list so build one. 691 */ 692 pfn_t *pfnp; 693 int pcnt = len >> PAGESHIFT; 694 695 if (off) 696 pcnt++; 697 if ((pfnp = kmem_alloc(pcnt * sizeof (pfnp), 698 KM_NOSLEEP)) == NULL) { 699 error = ENOMEM; 700 goto cleanup; 701 } 702 locked->uioa_ppp = (void **)pfnp; 703 locked->uioa_pfncnt = pcnt; 704 AS_LOCK_ENTER(as, RW_READER); 705 while (pcnt-- > 0) { 706 *pfnp++ = hat_getpfnum(as->a_hat, addr); 707 addr += PAGESIZE; 708 } 709 AS_LOCK_EXIT(as); 710 } else { 711 /* Have a page_t list, save it */ 712 locked->uioa_ppp = (void **)pages; 713 locked->uioa_pfncnt = 0; 714 } 715 /* Save for as_pageunlock() in uioafini() */ 716 locked->uioa_base = iov->iov_base; 717 locked->uioa_len = iov->iov_len; 718 locked++; 719 720 /* Next iovec_t */ 721 iov++; 722 iovcnt--; 723 } 724 /* Initialize curret pointer into uioa_locked[] and it's uioa_ppp */ 725 uioap->uioa_lcur = uioap->uioa_locked; 726 uioap->uioa_lppp = uioap->uioa_lcur->uioa_ppp; 727 return (0); 728 729 cleanup: 730 /* Unlock any previously locked page_t(s) */ 731 while (locked > uioap->uioa_locked) { 732 locked--; 733 as_pageunlock(as, (page_t **)locked->uioa_ppp, 734 locked->uioa_base, locked->uioa_len, S_WRITE); 735 } 736 737 /* Last indicate uioa_t still in alloc state */ 738 uioap->uioa_state = UIOA_ALLOC; 739 uioap->uioa_mbytes = 0; 740 741 return (error); 742 } 743 744 /* 745 * Finish processing of a uioa_t by cleanup any pending "uioap" actions. 746 */ 747 int 748 uioafini(uio_t *uiop, uioa_t *uioap) 749 { 750 int32_t iovcnt = uiop->uio_iovcnt; 751 uioa_page_t *locked = uioap->uioa_locked; 752 struct as *as = ttoproc(curthread)->p_as; 753 dcopy_handle_t channel; 754 dcopy_cmd_t cmd; 755 int ret = 0; 756 757 ASSERT(uioap->uio_extflg & UIO_ASYNC); 758 759 if (!(uioap->uioa_state & (UIOA_ENABLED|UIOA_FINI))) { 760 /* Must be an active uioa_t */ 761 return (EINVAL); 762 } 763 764 channel = uioap->uioa_hwst[UIO_DCOPY_CHANNEL]; 765 cmd = uioap->uioa_hwst[UIO_DCOPY_CMD]; 766 767 /* XXX - why do we get cmd == NULL sometimes? */ 768 if (cmd != NULL) { 769 if (uioap->uioa_state & UIOA_POLL) { 770 /* Wait for last dcopy() to finish */ 771 int64_t poll = 1; 772 int poll_flag = DCOPY_POLL_NOFLAGS; 773 774 do { 775 if (uioa_maxpoll == 0 || 776 (uioa_maxpoll > 0 && 777 poll >= uioa_maxpoll)) { 778 /* Always block or after maxpoll */ 779 poll_flag = DCOPY_POLL_BLOCK; 780 } else { 781 /* No block, poll */ 782 poll++; 783 } 784 ret = dcopy_cmd_poll(cmd, poll_flag); 785 } while (ret == DCOPY_PENDING); 786 787 if (ret == DCOPY_COMPLETED) { 788 /* Poll/block succeeded */ 789 ret = 0; 790 } else { 791 /* Poll/block failed */ 792 ret = EIO; 793 } 794 } 795 dcopy_cmd_free(&cmd); 796 } 797 798 dcopy_free(&channel); 799 800 /* Unlock all page(s) iovec_t by iovec_t */ 801 while (iovcnt-- > 0) { 802 page_t **pages; 803 804 if (locked->uioa_pfncnt == 0) { 805 /* A as_pagelock() returned (page_t **) */ 806 pages = (page_t **)locked->uioa_ppp; 807 } else { 808 /* Our pfn_t array */ 809 pages = NULL; 810 kmem_free(locked->uioa_ppp, locked->uioa_pfncnt * 811 sizeof (pfn_t *)); 812 } 813 as_pageunlock(as, pages, locked->uioa_base, locked->uioa_len, 814 S_WRITE); 815 816 locked++; 817 } 818 /* uioa_t->uio_t common struct copy */ 819 *uiop = *((uio_t *)uioap); 820 821 /* 822 * Last, reset uioa state to alloc. 823 * 824 * Note, we only initialize the state here, all other members 825 * will be initialized in a subsequent uioainit(). 826 */ 827 uioap->uioa_state = UIOA_ALLOC; 828 uioap->uioa_mbytes = 0; 829 830 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 831 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = NULL; 832 833 return (ret); 834 } 835