1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 27 /* All Rights Reserved */ 28 29 /* 30 * University Copyright- Copyright (c) 1982, 1986, 1988 31 * The Regents of the University of California 32 * All Rights Reserved 33 * 34 * University Acknowledgment- Portions of this document are derived from 35 * software developed by the University of California, Berkeley, and its 36 * contributors. 37 */ 38 39 #include <sys/types.h> 40 #include <sys/sysmacros.h> 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/uio.h> 44 #include <sys/errno.h> 45 #include <sys/vmsystm.h> 46 #include <sys/cmn_err.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 50 #include <sys/dcopy.h> 51 52 int64_t uioa_maxpoll = -1; /* <0 = noblock, 0 = block, >0 = block after */ 53 #define UIO_DCOPY_CHANNEL 0 54 #define UIO_DCOPY_CMD 1 55 56 /* 57 * Move "n" bytes at byte address "p"; "rw" indicates the direction 58 * of the move, and the I/O parameters are provided in "uio", which is 59 * update to reflect the data which was moved. Returns 0 on success or 60 * a non-zero errno on failure. 61 */ 62 int 63 uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio) 64 { 65 struct iovec *iov; 66 ulong_t cnt; 67 int error; 68 69 while (n && uio->uio_resid) { 70 iov = uio->uio_iov; 71 cnt = MIN(iov->iov_len, n); 72 if (cnt == 0l) { 73 uio->uio_iov++; 74 uio->uio_iovcnt--; 75 continue; 76 } 77 switch (uio->uio_segflg) { 78 79 case UIO_USERSPACE: 80 case UIO_USERISPACE: 81 if (rw == UIO_READ) { 82 error = xcopyout_nta(p, iov->iov_base, cnt, 83 (uio->uio_extflg & UIO_COPY_CACHED)); 84 } else { 85 error = xcopyin_nta(iov->iov_base, p, cnt, 86 (uio->uio_extflg & UIO_COPY_CACHED)); 87 } 88 89 if (error) 90 return (error); 91 break; 92 93 case UIO_SYSSPACE: 94 if (rw == UIO_READ) 95 error = kcopy_nta(p, iov->iov_base, cnt, 96 (uio->uio_extflg & UIO_COPY_CACHED)); 97 else 98 error = kcopy_nta(iov->iov_base, p, cnt, 99 (uio->uio_extflg & UIO_COPY_CACHED)); 100 if (error) 101 return (error); 102 break; 103 } 104 iov->iov_base += cnt; 105 iov->iov_len -= cnt; 106 uio->uio_resid -= cnt; 107 uio->uio_loffset += cnt; 108 p = (caddr_t)p + cnt; 109 n -= cnt; 110 } 111 return (0); 112 } 113 114 /* 115 * Fault in the pages of the first n bytes specified by the uio structure. 116 * 1 byte in each page is touched and the uio struct is unmodified. Any 117 * error will terminate the process as this is only a best attempt to get 118 * the pages resident. 119 */ 120 void 121 uio_prefaultpages(ssize_t n, struct uio *uio) 122 { 123 struct iovec *iov; 124 ulong_t cnt, incr; 125 caddr_t p; 126 uint8_t tmp; 127 int iovcnt; 128 129 iov = uio->uio_iov; 130 iovcnt = uio->uio_iovcnt; 131 132 while ((n > 0) && (iovcnt > 0)) { 133 cnt = MIN(iov->iov_len, n); 134 if (cnt == 0) { 135 /* empty iov entry */ 136 iov++; 137 iovcnt--; 138 continue; 139 } 140 n -= cnt; 141 /* 142 * touch each page in this segment. 143 */ 144 p = iov->iov_base; 145 while (cnt) { 146 switch (uio->uio_segflg) { 147 case UIO_USERSPACE: 148 case UIO_USERISPACE: 149 if (fuword8(p, &tmp)) 150 return; 151 break; 152 case UIO_SYSSPACE: 153 if (kcopy(p, &tmp, 1)) 154 return; 155 break; 156 } 157 incr = MIN(cnt, PAGESIZE); 158 p += incr; 159 cnt -= incr; 160 } 161 /* 162 * touch the last byte in case it straddles a page. 163 */ 164 p--; 165 switch (uio->uio_segflg) { 166 case UIO_USERSPACE: 167 case UIO_USERISPACE: 168 if (fuword8(p, &tmp)) 169 return; 170 break; 171 case UIO_SYSSPACE: 172 if (kcopy(p, &tmp, 1)) 173 return; 174 break; 175 } 176 iov++; 177 iovcnt--; 178 } 179 } 180 181 /* 182 * same as uiomove() but doesn't modify uio structure. 183 * return in cbytes how many bytes were copied. 184 */ 185 int 186 uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes) 187 { 188 struct iovec *iov; 189 ulong_t cnt; 190 int error; 191 int iovcnt; 192 193 iovcnt = uio->uio_iovcnt; 194 *cbytes = 0; 195 196 for (iov = uio->uio_iov; n && iovcnt; iov++, iovcnt--) { 197 cnt = MIN(iov->iov_len, n); 198 if (cnt == 0) 199 continue; 200 201 switch (uio->uio_segflg) { 202 203 case UIO_USERSPACE: 204 case UIO_USERISPACE: 205 if (rw == UIO_READ) { 206 error = xcopyout_nta(p, iov->iov_base, cnt, 207 (uio->uio_extflg & UIO_COPY_CACHED)); 208 } else { 209 error = xcopyin_nta(iov->iov_base, p, cnt, 210 (uio->uio_extflg & UIO_COPY_CACHED)); 211 } 212 213 if (error) 214 return (error); 215 break; 216 217 case UIO_SYSSPACE: 218 if (rw == UIO_READ) 219 error = kcopy_nta(p, iov->iov_base, cnt, 220 (uio->uio_extflg & UIO_COPY_CACHED)); 221 else 222 error = kcopy_nta(iov->iov_base, p, cnt, 223 (uio->uio_extflg & UIO_COPY_CACHED)); 224 if (error) 225 return (error); 226 break; 227 } 228 p = (caddr_t)p + cnt; 229 n -= cnt; 230 *cbytes += cnt; 231 } 232 return (0); 233 } 234 235 /* 236 * transfer a character value into the address space 237 * delineated by a uio and update fields within the 238 * uio for next character. Return 0 for success, EFAULT 239 * for error. 240 */ 241 int 242 ureadc(int val, struct uio *uiop) 243 { 244 struct iovec *iovp; 245 unsigned char c; 246 247 /* 248 * first determine if uio is valid. uiop should be 249 * non-NULL and the resid count > 0. 250 */ 251 if (!(uiop && uiop->uio_resid > 0)) 252 return (EFAULT); 253 254 /* 255 * scan through iovecs until one is found that is non-empty. 256 * Return EFAULT if none found. 257 */ 258 while (uiop->uio_iovcnt > 0) { 259 iovp = uiop->uio_iov; 260 if (iovp->iov_len <= 0) { 261 uiop->uio_iovcnt--; 262 uiop->uio_iov++; 263 } else 264 break; 265 } 266 267 if (uiop->uio_iovcnt <= 0) 268 return (EFAULT); 269 270 /* 271 * Transfer character to uio space. 272 */ 273 274 c = (unsigned char) (val & 0xFF); 275 276 switch (uiop->uio_segflg) { 277 278 case UIO_USERISPACE: 279 case UIO_USERSPACE: 280 if (copyout(&c, iovp->iov_base, sizeof (unsigned char))) 281 return (EFAULT); 282 break; 283 284 case UIO_SYSSPACE: /* can do direct copy since kernel-kernel */ 285 *iovp->iov_base = c; 286 break; 287 288 default: 289 return (EFAULT); /* invalid segflg value */ 290 } 291 292 /* 293 * bump up/down iovec and uio members to reflect transfer. 294 */ 295 iovp->iov_base++; 296 iovp->iov_len--; 297 uiop->uio_resid--; 298 uiop->uio_loffset++; 299 return (0); /* success */ 300 } 301 302 /* 303 * return a character value from the address space 304 * delineated by a uio and update fields within the 305 * uio for next character. Return the character for success, 306 * -1 for error. 307 */ 308 int 309 uwritec(struct uio *uiop) 310 { 311 struct iovec *iovp; 312 unsigned char c; 313 314 /* 315 * verify we were passed a valid uio structure. 316 * (1) non-NULL uiop, (2) positive resid count 317 * (3) there is an iovec with positive length 318 */ 319 320 if (!(uiop && uiop->uio_resid > 0)) 321 return (-1); 322 323 while (uiop->uio_iovcnt > 0) { 324 iovp = uiop->uio_iov; 325 if (iovp->iov_len <= 0) { 326 uiop->uio_iovcnt--; 327 uiop->uio_iov++; 328 } else 329 break; 330 } 331 332 if (uiop->uio_iovcnt <= 0) 333 return (-1); 334 335 /* 336 * Get the character from the uio address space. 337 */ 338 switch (uiop->uio_segflg) { 339 340 case UIO_USERISPACE: 341 case UIO_USERSPACE: 342 if (copyin(iovp->iov_base, &c, sizeof (unsigned char))) 343 return (-1); 344 break; 345 346 case UIO_SYSSPACE: 347 c = *iovp->iov_base; 348 break; 349 350 default: 351 return (-1); /* invalid segflg */ 352 } 353 354 /* 355 * Adjust fields of iovec and uio appropriately. 356 */ 357 iovp->iov_base++; 358 iovp->iov_len--; 359 uiop->uio_resid--; 360 uiop->uio_loffset++; 361 return ((int)c & 0xFF); /* success */ 362 } 363 364 /* 365 * Drop the next n chars out of *uiop. 366 */ 367 void 368 uioskip(uio_t *uiop, size_t n) 369 { 370 if (n > uiop->uio_resid) 371 return; 372 while (n != 0) { 373 register iovec_t *iovp = uiop->uio_iov; 374 register size_t niovb = MIN(iovp->iov_len, n); 375 376 if (niovb == 0) { 377 uiop->uio_iov++; 378 uiop->uio_iovcnt--; 379 continue; 380 } 381 iovp->iov_base += niovb; 382 uiop->uio_loffset += niovb; 383 iovp->iov_len -= niovb; 384 uiop->uio_resid -= niovb; 385 n -= niovb; 386 } 387 } 388 389 /* 390 * Dup the suio into the duio and diovec of size diov_cnt. If diov 391 * is too small to dup suio then an error will be returned, else 0. 392 */ 393 int 394 uiodup(uio_t *suio, uio_t *duio, iovec_t *diov, int diov_cnt) 395 { 396 int ix; 397 iovec_t *siov = suio->uio_iov; 398 399 *duio = *suio; 400 for (ix = 0; ix < suio->uio_iovcnt; ix++) { 401 diov[ix] = siov[ix]; 402 if (ix >= diov_cnt) 403 return (1); 404 } 405 duio->uio_iov = diov; 406 return (0); 407 } 408 409 /* 410 * Shadow state for checking if a platform has hardware asynchronous 411 * copy capability and minimum copy size, e.g. Intel's I/OAT dma engine, 412 * 413 * Dcopy does a call-back to uioa_dcopy_enable() when a dma device calls 414 * into dcopy to register and uioa_dcopy_disable() when the device calls 415 * into dcopy to unregister. 416 */ 417 uioasync_t uioasync = {B_FALSE, 1024}; 418 419 void 420 uioa_dcopy_enable() 421 { 422 uioasync.enabled = B_TRUE; 423 } 424 425 void 426 uioa_dcopy_disable() 427 { 428 uioasync.enabled = B_FALSE; 429 } 430 431 /* 432 * Schedule an asynchronous move of "n" bytes at byte address "p", 433 * "rw" indicates the direction of the move, I/O parameters and 434 * async state are provided in "uioa" which is update to reflect 435 * the data which is to be moved. 436 * 437 * Returns 0 on success or a non-zero errno on failure. 438 * 439 * Note, while the uioasync APIs are general purpose in design 440 * the current implementation is Intel I/OAT specific. 441 */ 442 int 443 uioamove(void *p, size_t n, enum uio_rw rw, uioa_t *uioa) 444 { 445 int soff, doff; 446 uint64_t pa; 447 int cnt; 448 iovec_t *iov; 449 dcopy_handle_t channel; 450 dcopy_cmd_t cmd; 451 int ret = 0; 452 int dcopy_flags; 453 454 if (!(uioa->uioa_state & UIOA_ENABLED)) { 455 /* The uioa_t isn't enabled */ 456 return (ENXIO); 457 } 458 459 if (uioa->uio_segflg != UIO_USERSPACE || rw != UIO_READ) { 460 /* Only support to user-land from kernel */ 461 return (ENOTSUP); 462 } 463 464 465 channel = uioa->uioa_hwst[UIO_DCOPY_CHANNEL]; 466 cmd = uioa->uioa_hwst[UIO_DCOPY_CMD]; 467 dcopy_flags = DCOPY_NOSLEEP; 468 469 /* 470 * While source bytes and destination bytes. 471 */ 472 while (n > 0 && uioa->uio_resid > 0) { 473 iov = uioa->uio_iov; 474 if (iov->iov_len == 0l) { 475 uioa->uio_iov++; 476 uioa->uio_iovcnt--; 477 uioa->uioa_lcur++; 478 uioa->uioa_lppp = uioa->uioa_lcur->uioa_ppp; 479 continue; 480 } 481 /* 482 * While source bytes schedule an async 483 * dma for destination page by page. 484 */ 485 while (n > 0) { 486 /* Addr offset in page src/dst */ 487 soff = (uintptr_t)p & PAGEOFFSET; 488 doff = (uintptr_t)iov->iov_base & PAGEOFFSET; 489 /* Min copy count src and dst and page sized */ 490 cnt = MIN(n, iov->iov_len); 491 cnt = MIN(cnt, PAGESIZE - soff); 492 cnt = MIN(cnt, PAGESIZE - doff); 493 /* XXX if next page(s) contiguous could use multipage */ 494 495 /* 496 * if we have an old command, we want to link all 497 * other commands to the next command we alloced so 498 * we only need to track the last command but can 499 * still free them all. 500 */ 501 if (cmd != NULL) { 502 dcopy_flags |= DCOPY_ALLOC_LINK; 503 } 504 ret = dcopy_cmd_alloc(channel, dcopy_flags, &cmd); 505 if (ret != DCOPY_SUCCESS) { 506 /* Error of some sort */ 507 return (EIO); 508 } 509 uioa->uioa_hwst[UIO_DCOPY_CMD] = cmd; 510 511 ASSERT(cmd->dp_version == DCOPY_CMD_V0); 512 if (uioa_maxpoll >= 0) { 513 /* Blocking (>0 may be) used in uioafini() */ 514 cmd->dp_flags = DCOPY_CMD_INTR; 515 } else { 516 /* Non blocking uioafini() so no intr */ 517 cmd->dp_flags = DCOPY_CMD_NOFLAGS; 518 } 519 cmd->dp_cmd = DCOPY_CMD_COPY; 520 pa = ptob((uint64_t)hat_getpfnum(kas.a_hat, p)); 521 cmd->dp.copy.cc_source = pa + soff; 522 if (uioa->uioa_lcur->uioa_pfncnt == 0) { 523 /* Have a (page_t **) */ 524 pa = ptob((uint64_t)( 525 *(page_t **)uioa->uioa_lppp)->p_pagenum); 526 } else { 527 /* Have a (pfn_t *) */ 528 pa = ptob((uint64_t)( 529 *(pfn_t *)uioa->uioa_lppp)); 530 } 531 cmd->dp.copy.cc_dest = pa + doff; 532 cmd->dp.copy.cc_size = cnt; 533 ret = dcopy_cmd_post(cmd); 534 if (ret != DCOPY_SUCCESS) { 535 /* Error of some sort */ 536 return (EIO); 537 } 538 ret = 0; 539 540 /* If UIOA_POLL not set, set it */ 541 if (!(uioa->uioa_state & UIOA_POLL)) 542 uioa->uioa_state |= UIOA_POLL; 543 544 /* Update iov, uio, and local pointers/counters */ 545 iov->iov_base += cnt; 546 iov->iov_len -= cnt; 547 uioa->uio_resid -= cnt; 548 uioa->uioa_mbytes += cnt; 549 uioa->uio_loffset += cnt; 550 p = (caddr_t)p + cnt; 551 n -= cnt; 552 553 /* End of iovec? */ 554 if (iov->iov_len == 0) { 555 /* Yup, next iovec */ 556 break; 557 } 558 559 /* Next dst addr page? */ 560 if (doff + cnt == PAGESIZE) { 561 /* Yup, next page_t */ 562 uioa->uioa_lppp++; 563 } 564 } 565 } 566 567 return (ret); 568 } 569 570 /* 571 * Initialize a uioa_t for a given uio_t for the current user context, 572 * copy the common uio_t to the uioa_t, walk the shared iovec_t and 573 * lock down the user-land page(s) containing iovec_t data, then mapin 574 * user-land pages using segkpm. 575 */ 576 int 577 uioainit(uio_t *uiop, uioa_t *uioap) 578 { 579 caddr_t addr; 580 page_t **pages; 581 int off; 582 int len; 583 proc_t *procp = ttoproc(curthread); 584 struct as *as = procp->p_as; 585 iovec_t *iov = uiop->uio_iov; 586 int32_t iovcnt = uiop->uio_iovcnt; 587 uioa_page_t *locked = uioap->uioa_locked; 588 dcopy_handle_t channel; 589 int error; 590 591 if (! (uioap->uioa_state & UIOA_ALLOC)) { 592 /* Can only init() a freshly allocated uioa_t */ 593 return (EINVAL); 594 } 595 596 error = dcopy_alloc(DCOPY_NOSLEEP, &channel); 597 if (error == DCOPY_NORESOURCES) { 598 /* Turn off uioa */ 599 uioasync.enabled = B_FALSE; 600 return (ENODEV); 601 } 602 if (error != DCOPY_SUCCESS) { 603 /* Alloc failed */ 604 return (EIO); 605 } 606 607 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = channel; 608 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 609 610 /* Indicate uioa_t (will be) initialized */ 611 uioap->uioa_state = UIOA_INIT; 612 613 uioap->uioa_mbytes = 0; 614 615 /* uio_t/uioa_t uio_t common struct copy */ 616 *((uio_t *)uioap) = *uiop; 617 618 /* initialize *uiop->uio_iov */ 619 if (iovcnt > UIOA_IOV_MAX) { 620 /* Too big? */ 621 return (E2BIG); 622 } 623 uioap->uio_iov = iov; 624 uioap->uio_iovcnt = iovcnt; 625 626 /* Mark the uioap as such */ 627 uioap->uio_extflg |= UIO_ASYNC; 628 629 /* 630 * For each iovec_t, lock-down the page(s) backing the iovec_t 631 * and save the page_t list for phys addr use in uioamove(). 632 */ 633 iov = uiop->uio_iov; 634 iovcnt = uiop->uio_iovcnt; 635 while (iovcnt > 0) { 636 addr = iov->iov_base; 637 off = (uintptr_t)addr & PAGEOFFSET; 638 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK); 639 len = iov->iov_len + off; 640 641 /* Lock down page(s) for the iov span */ 642 if ((error = as_pagelock(as, &pages, 643 iov->iov_base, iov->iov_len, S_WRITE)) != 0) { 644 /* Error */ 645 goto cleanup; 646 } 647 648 if (pages == NULL) { 649 /* 650 * Need page_t list, really only need 651 * a pfn list so build one. 652 */ 653 pfn_t *pfnp; 654 int pcnt = len >> PAGESHIFT; 655 656 if (off) 657 pcnt++; 658 if ((pfnp = kmem_alloc(pcnt * sizeof (pfnp), 659 KM_NOSLEEP)) == NULL) { 660 error = ENOMEM; 661 goto cleanup; 662 } 663 locked->uioa_ppp = (void **)pfnp; 664 locked->uioa_pfncnt = pcnt; 665 AS_LOCK_ENTER(as, RW_READER); 666 while (pcnt-- > 0) { 667 *pfnp++ = hat_getpfnum(as->a_hat, addr); 668 addr += PAGESIZE; 669 } 670 AS_LOCK_EXIT(as); 671 } else { 672 /* Have a page_t list, save it */ 673 locked->uioa_ppp = (void **)pages; 674 locked->uioa_pfncnt = 0; 675 } 676 /* Save for as_pageunlock() in uioafini() */ 677 locked->uioa_base = iov->iov_base; 678 locked->uioa_len = iov->iov_len; 679 locked++; 680 681 /* Next iovec_t */ 682 iov++; 683 iovcnt--; 684 } 685 /* Initialize curret pointer into uioa_locked[] and it's uioa_ppp */ 686 uioap->uioa_lcur = uioap->uioa_locked; 687 uioap->uioa_lppp = uioap->uioa_lcur->uioa_ppp; 688 return (0); 689 690 cleanup: 691 /* Unlock any previously locked page_t(s) */ 692 while (locked > uioap->uioa_locked) { 693 locked--; 694 as_pageunlock(as, (page_t **)locked->uioa_ppp, 695 locked->uioa_base, locked->uioa_len, S_WRITE); 696 } 697 698 /* Last indicate uioa_t still in alloc state */ 699 uioap->uioa_state = UIOA_ALLOC; 700 uioap->uioa_mbytes = 0; 701 702 return (error); 703 } 704 705 /* 706 * Finish processing of a uioa_t by cleanup any pending "uioap" actions. 707 */ 708 int 709 uioafini(uio_t *uiop, uioa_t *uioap) 710 { 711 int32_t iovcnt = uiop->uio_iovcnt; 712 uioa_page_t *locked = uioap->uioa_locked; 713 struct as *as = ttoproc(curthread)->p_as; 714 dcopy_handle_t channel; 715 dcopy_cmd_t cmd; 716 int ret = 0; 717 718 ASSERT(uioap->uio_extflg & UIO_ASYNC); 719 720 if (!(uioap->uioa_state & (UIOA_ENABLED|UIOA_FINI))) { 721 /* Must be an active uioa_t */ 722 return (EINVAL); 723 } 724 725 channel = uioap->uioa_hwst[UIO_DCOPY_CHANNEL]; 726 cmd = uioap->uioa_hwst[UIO_DCOPY_CMD]; 727 728 /* XXX - why do we get cmd == NULL sometimes? */ 729 if (cmd != NULL) { 730 if (uioap->uioa_state & UIOA_POLL) { 731 /* Wait for last dcopy() to finish */ 732 int64_t poll = 1; 733 int poll_flag = DCOPY_POLL_NOFLAGS; 734 735 do { 736 if (uioa_maxpoll == 0 || 737 (uioa_maxpoll > 0 && 738 poll >= uioa_maxpoll)) { 739 /* Always block or after maxpoll */ 740 poll_flag = DCOPY_POLL_BLOCK; 741 } else { 742 /* No block, poll */ 743 poll++; 744 } 745 ret = dcopy_cmd_poll(cmd, poll_flag); 746 } while (ret == DCOPY_PENDING); 747 748 if (ret == DCOPY_COMPLETED) { 749 /* Poll/block succeeded */ 750 ret = 0; 751 } else { 752 /* Poll/block failed */ 753 ret = EIO; 754 } 755 } 756 dcopy_cmd_free(&cmd); 757 } 758 759 dcopy_free(&channel); 760 761 /* Unlock all page(s) iovec_t by iovec_t */ 762 while (iovcnt-- > 0) { 763 page_t **pages; 764 765 if (locked->uioa_pfncnt == 0) { 766 /* A as_pagelock() returned (page_t **) */ 767 pages = (page_t **)locked->uioa_ppp; 768 } else { 769 /* Our pfn_t array */ 770 pages = NULL; 771 kmem_free(locked->uioa_ppp, locked->uioa_pfncnt * 772 sizeof (pfn_t *)); 773 } 774 as_pageunlock(as, pages, locked->uioa_base, locked->uioa_len, 775 S_WRITE); 776 777 locked++; 778 } 779 /* uioa_t->uio_t common struct copy */ 780 *uiop = *((uio_t *)uioap); 781 782 /* 783 * Last, reset uioa state to alloc. 784 * 785 * Note, we only initialize the state here, all other members 786 * will be initialized in a subsequent uioainit(). 787 */ 788 uioap->uioa_state = UIOA_ALLOC; 789 uioap->uioa_mbytes = 0; 790 791 uioap->uioa_hwst[UIO_DCOPY_CMD] = NULL; 792 uioap->uioa_hwst[UIO_DCOPY_CHANNEL] = NULL; 793 794 return (ret); 795 } 796