1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/types.h> 27 #include <sys/t_lock.h> 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/buf.h> 31 #include <sys/conf.h> 32 #include <sys/cred.h> 33 #include <sys/kmem.h> 34 #include <sys/sysmacros.h> 35 #include <sys/vfs.h> 36 #include <sys/vnode.h> 37 #include <sys/debug.h> 38 #include <sys/errno.h> 39 #include <sys/time.h> 40 #include <sys/file.h> 41 #include <sys/open.h> 42 #include <sys/user.h> 43 #include <sys/termios.h> 44 #include <sys/stream.h> 45 #include <sys/strsubr.h> 46 #include <sys/sunddi.h> 47 #include <sys/esunddi.h> 48 #include <sys/flock.h> 49 #include <sys/modctl.h> 50 #include <sys/cmn_err.h> 51 #include <sys/vmsystm.h> 52 53 #include <sys/socket.h> 54 #include <sys/socketvar.h> 55 #include <fs/sockfs/sockcommon.h> 56 #include <fs/sockfs/socktpi.h> 57 58 #include <netinet/in.h> 59 #include <sys/sendfile.h> 60 #include <sys/un.h> 61 #include <sys/tihdr.h> 62 #include <sys/atomic.h> 63 64 #include <inet/common.h> 65 #include <inet/ip.h> 66 #include <inet/ip6.h> 67 #include <inet/tcp.h> 68 69 extern int sosendfile64(file_t *, file_t *, const struct ksendfilevec64 *, 70 ssize32_t *); 71 extern int nl7c_sendfilev(struct sonode *, u_offset_t *, struct sendfilevec *, 72 int, ssize_t *); 73 extern int snf_segmap(file_t *, vnode_t *, u_offset_t, u_offset_t, ssize_t *, 74 boolean_t); 75 extern sotpi_info_t *sotpi_sototpi(struct sonode *); 76 77 #define SEND_MAX_CHUNK 16 78 79 #if defined(_SYSCALL32_IMPL) || defined(_ILP32) 80 /* 81 * 64 bit offsets for 32 bit applications only running either on 82 * 64 bit kernel or 32 bit kernel. For 32 bit apps, we can't transfer 83 * more than 2GB of data. 84 */ 85 int 86 sendvec_chunk64(file_t *fp, u_offset_t *fileoff, struct ksendfilevec64 *sfv, 87 int copy_cnt, ssize32_t *count) 88 { 89 struct vnode *vp; 90 ushort_t fflag; 91 int ioflag; 92 size32_t cnt; 93 ssize32_t sfv_len; 94 ssize32_t tmpcount; 95 u_offset_t sfv_off; 96 struct uio auio; 97 struct iovec aiov; 98 int i, error; 99 100 fflag = fp->f_flag; 101 vp = fp->f_vnode; 102 for (i = 0; i < copy_cnt; i++) { 103 104 if (ISSIG(curthread, JUSTLOOKING)) 105 return (EINTR); 106 107 /* 108 * Do similar checks as "write" as we are writing 109 * sfv_len bytes into "vp". 110 */ 111 sfv_len = (ssize32_t)sfv->sfv_len; 112 113 if (sfv_len == 0) { 114 sfv++; 115 continue; 116 } 117 118 if (sfv_len < 0) 119 return (EINVAL); 120 121 if (vp->v_type == VREG) { 122 if (*fileoff >= curproc->p_fsz_ctl) { 123 mutex_enter(&curproc->p_lock); 124 (void) rctl_action( 125 rctlproc_legacy[RLIMIT_FSIZE], 126 curproc->p_rctls, curproc, RCA_SAFE); 127 mutex_exit(&curproc->p_lock); 128 return (EFBIG); 129 } 130 131 if (*fileoff >= OFFSET_MAX(fp)) 132 return (EFBIG); 133 134 if (*fileoff + sfv_len > OFFSET_MAX(fp)) 135 return (EINVAL); 136 } 137 138 tmpcount = *count + sfv_len; 139 if (tmpcount < 0) 140 return (EINVAL); 141 142 sfv_off = sfv->sfv_off; 143 144 auio.uio_extflg = UIO_COPY_DEFAULT; 145 if (sfv->sfv_fd == SFV_FD_SELF) { 146 aiov.iov_len = sfv_len; 147 aiov.iov_base = (caddr_t)(uintptr_t)sfv_off; 148 auio.uio_loffset = *fileoff; 149 auio.uio_iovcnt = 1; 150 auio.uio_resid = sfv_len; 151 auio.uio_iov = &aiov; 152 auio.uio_segflg = UIO_USERSPACE; 153 auio.uio_llimit = curproc->p_fsz_ctl; 154 auio.uio_fmode = fflag; 155 ioflag = auio.uio_fmode & (FAPPEND|FSYNC|FDSYNC|FRSYNC); 156 while (sfv_len > 0) { 157 error = VOP_WRITE(vp, &auio, ioflag, 158 fp->f_cred, NULL); 159 cnt = sfv_len - auio.uio_resid; 160 sfv_len -= cnt; 161 ttolwp(curthread)->lwp_ru.ioch += (ulong_t)cnt; 162 if (vp->v_type == VREG) 163 *fileoff += cnt; 164 *count += cnt; 165 if (error != 0) 166 return (error); 167 } 168 } else { 169 file_t *ffp; 170 vnode_t *readvp; 171 size_t size; 172 caddr_t ptr; 173 174 if ((ffp = getf(sfv->sfv_fd)) == NULL) 175 return (EBADF); 176 177 if ((ffp->f_flag & FREAD) == 0) { 178 releasef(sfv->sfv_fd); 179 return (EBADF); 180 } 181 182 readvp = ffp->f_vnode; 183 if (readvp->v_type != VREG) { 184 releasef(sfv->sfv_fd); 185 return (EINVAL); 186 } 187 188 /* 189 * No point reading and writing to same vp, 190 * as long as both are regular files. readvp is not 191 * locked; but since we got it from an open file the 192 * contents will be valid during the time of access. 193 */ 194 if (vn_compare(vp, readvp)) { 195 releasef(sfv->sfv_fd); 196 return (EINVAL); 197 } 198 199 /* 200 * Optimize the regular file over 201 * the socket case. 202 */ 203 if (vp->v_type == VSOCK) { 204 error = sosendfile64(fp, ffp, sfv, 205 (ssize32_t *)&cnt); 206 *count += cnt; 207 if (error) 208 return (error); 209 sfv++; 210 continue; 211 } 212 213 /* 214 * Note: we assume readvp != vp. "vp" is already 215 * locked, and "readvp" must not be. 216 */ 217 if (readvp < vp) { 218 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 219 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 220 NULL); 221 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 222 } else { 223 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 224 NULL); 225 } 226 227 /* 228 * Same checks as in pread64. 229 */ 230 if (sfv_off > MAXOFFSET_T) { 231 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 232 releasef(sfv->sfv_fd); 233 return (EINVAL); 234 } 235 236 if (sfv_off + sfv_len > MAXOFFSET_T) 237 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off); 238 239 /* Find the native blocksize to transfer data */ 240 size = MIN(vp->v_vfsp->vfs_bsize, 241 readvp->v_vfsp->vfs_bsize); 242 size = sfv_len < size ? sfv_len : size; 243 ptr = kmem_alloc(size, KM_NOSLEEP); 244 if (ptr == NULL) { 245 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 246 releasef(sfv->sfv_fd); 247 return (ENOMEM); 248 } 249 250 while (sfv_len > 0) { 251 size_t iov_len; 252 253 iov_len = MIN(size, sfv_len); 254 aiov.iov_base = ptr; 255 aiov.iov_len = iov_len; 256 auio.uio_loffset = sfv_off; 257 auio.uio_iov = &aiov; 258 auio.uio_iovcnt = 1; 259 auio.uio_resid = iov_len; 260 auio.uio_segflg = UIO_SYSSPACE; 261 auio.uio_llimit = MAXOFFSET_T; 262 auio.uio_fmode = ffp->f_flag; 263 ioflag = auio.uio_fmode & 264 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 265 266 /* 267 * If read sync is not asked for, 268 * filter sync flags 269 */ 270 if ((ioflag & FRSYNC) == 0) 271 ioflag &= ~(FSYNC|FDSYNC); 272 error = VOP_READ(readvp, &auio, ioflag, 273 fp->f_cred, NULL); 274 if (error) { 275 kmem_free(ptr, size); 276 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 277 NULL); 278 releasef(sfv->sfv_fd); 279 return (error); 280 } 281 282 /* 283 * Check how must data was really read. 284 * Decrement the 'len' and increment the 285 * 'off' appropriately. 286 */ 287 cnt = iov_len - auio.uio_resid; 288 if (cnt == 0) { 289 /* 290 * If we were reading a pipe (currently 291 * not implemented), we may now lose 292 * data. 293 */ 294 kmem_free(ptr, size); 295 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 296 NULL); 297 releasef(sfv->sfv_fd); 298 return (EINVAL); 299 } 300 sfv_len -= cnt; 301 sfv_off += cnt; 302 303 aiov.iov_base = ptr; 304 aiov.iov_len = cnt; 305 auio.uio_loffset = *fileoff; 306 auio.uio_iov = &aiov; 307 auio.uio_iovcnt = 1; 308 auio.uio_resid = cnt; 309 auio.uio_segflg = UIO_SYSSPACE; 310 auio.uio_llimit = curproc->p_fsz_ctl; 311 auio.uio_fmode = fflag; 312 ioflag = auio.uio_fmode & 313 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 314 error = VOP_WRITE(vp, &auio, ioflag, 315 fp->f_cred, NULL); 316 317 /* 318 * Check how much data was written. Increment 319 * the 'len' and decrement the 'off' if all 320 * the data was not written. 321 */ 322 cnt -= auio.uio_resid; 323 sfv_len += auio.uio_resid; 324 sfv_off -= auio.uio_resid; 325 ttolwp(curthread)->lwp_ru.ioch += (ulong_t)cnt; 326 if (vp->v_type == VREG) 327 *fileoff += cnt; 328 *count += cnt; 329 if (error != 0) { 330 kmem_free(ptr, size); 331 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 332 NULL); 333 releasef(sfv->sfv_fd); 334 return (error); 335 } 336 } 337 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 338 releasef(sfv->sfv_fd); 339 kmem_free(ptr, size); 340 } 341 sfv++; 342 } 343 return (0); 344 } 345 346 ssize32_t 347 sendvec64(file_t *fp, const struct ksendfilevec64 *vec, int sfvcnt, 348 size32_t *xferred, int fildes) 349 { 350 u_offset_t fileoff; 351 int copy_cnt; 352 const struct ksendfilevec64 *copy_vec; 353 struct ksendfilevec64 sfv[SEND_MAX_CHUNK]; 354 struct vnode *vp; 355 int error; 356 ssize32_t count = 0; 357 358 vp = fp->f_vnode; 359 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 360 361 copy_vec = vec; 362 fileoff = fp->f_offset; 363 364 do { 365 copy_cnt = MIN(sfvcnt, SEND_MAX_CHUNK); 366 if (copyin(copy_vec, sfv, copy_cnt * 367 sizeof (struct ksendfilevec64))) { 368 error = EFAULT; 369 break; 370 } 371 372 error = sendvec_chunk64(fp, &fileoff, sfv, copy_cnt, &count); 373 if (error != 0) 374 break; 375 376 copy_vec += copy_cnt; 377 sfvcnt -= copy_cnt; 378 } while (sfvcnt > 0); 379 380 if (vp->v_type == VREG) 381 fp->f_offset += count; 382 383 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 384 if (copyout(&count, xferred, sizeof (count))) 385 error = EFAULT; 386 releasef(fildes); 387 if (error != 0) 388 return (set_errno(error)); 389 return (count); 390 } 391 #endif 392 393 int 394 sendvec_small_chunk(file_t *fp, u_offset_t *fileoff, struct sendfilevec *sfv, 395 int copy_cnt, ssize_t total_size, int maxblk, ssize_t *count) 396 { 397 struct vnode *vp; 398 struct uio auio; 399 struct iovec aiov; 400 ushort_t fflag; 401 int ioflag; 402 int i, error; 403 size_t cnt; 404 ssize_t sfv_len; 405 u_offset_t sfv_off; 406 #ifdef _SYSCALL32_IMPL 407 model_t model = get_udatamodel(); 408 u_offset_t maxoff = (model == DATAMODEL_ILP32) ? 409 MAXOFF32_T : MAXOFFSET_T; 410 #else 411 const u_offset_t maxoff = MAXOFF32_T; 412 #endif 413 mblk_t *dmp = NULL; 414 int wroff; 415 int buf_left = 0; 416 size_t iov_len; 417 mblk_t *head, *tmp; 418 size_t size = total_size; 419 size_t extra; 420 int tail_len; 421 struct nmsghdr msg; 422 423 fflag = fp->f_flag; 424 vp = fp->f_vnode; 425 426 ASSERT(vp->v_type == VSOCK); 427 ASSERT(maxblk > 0); 428 429 /* If nothing to send, return */ 430 if (total_size == 0) 431 return (0); 432 433 if (vp->v_stream != NULL) { 434 wroff = (int)vp->v_stream->sd_wroff; 435 tail_len = (int)vp->v_stream->sd_tail; 436 } else { 437 struct sonode *so; 438 439 so = VTOSO(vp); 440 wroff = so->so_proto_props.sopp_wroff; 441 tail_len = so->so_proto_props.sopp_tail; 442 } 443 444 extra = wroff + tail_len; 445 446 buf_left = MIN(total_size, maxblk); 447 head = dmp = allocb(buf_left + extra, BPRI_HI); 448 if (head == NULL) 449 return (ENOMEM); 450 head->b_wptr = head->b_rptr = head->b_rptr + wroff; 451 bzero(&msg, sizeof (msg)); 452 453 auio.uio_extflg = UIO_COPY_DEFAULT; 454 for (i = 0; i < copy_cnt; i++) { 455 if (ISSIG(curthread, JUSTLOOKING)) { 456 freemsg(head); 457 return (EINTR); 458 } 459 460 /* 461 * Do similar checks as "write" as we are writing 462 * sfv_len bytes into "vp". 463 */ 464 sfv_len = (ssize_t)sfv->sfv_len; 465 466 if (sfv_len == 0) { 467 sfv++; 468 continue; 469 } 470 471 /* Check for overflow */ 472 #ifdef _SYSCALL32_IMPL 473 if (model == DATAMODEL_ILP32) { 474 if (((ssize32_t)(*count + sfv_len)) < 0) { 475 freemsg(head); 476 return (EINVAL); 477 } 478 } else 479 #endif 480 if ((*count + sfv_len) < 0) { 481 freemsg(head); 482 return (EINVAL); 483 } 484 485 sfv_off = (u_offset_t)(ulong_t)sfv->sfv_off; 486 487 if (sfv->sfv_fd == SFV_FD_SELF) { 488 while (sfv_len > 0) { 489 if (buf_left == 0) { 490 tmp = dmp; 491 buf_left = MIN(total_size, maxblk); 492 iov_len = MIN(buf_left, sfv_len); 493 dmp = allocb(buf_left + extra, BPRI_HI); 494 if (dmp == NULL) { 495 freemsg(head); 496 return (ENOMEM); 497 } 498 dmp->b_wptr = dmp->b_rptr = 499 dmp->b_rptr + wroff; 500 tmp->b_cont = dmp; 501 } else { 502 iov_len = MIN(buf_left, sfv_len); 503 } 504 505 aiov.iov_len = iov_len; 506 aiov.iov_base = (caddr_t)(uintptr_t)sfv_off; 507 auio.uio_loffset = *fileoff; 508 auio.uio_iovcnt = 1; 509 auio.uio_resid = iov_len; 510 auio.uio_iov = &aiov; 511 auio.uio_segflg = UIO_USERSPACE; 512 auio.uio_llimit = curproc->p_fsz_ctl; 513 auio.uio_fmode = fflag; 514 515 buf_left -= iov_len; 516 total_size -= iov_len; 517 sfv_len -= iov_len; 518 sfv_off += iov_len; 519 520 error = uiomove((caddr_t)dmp->b_wptr, 521 iov_len, UIO_WRITE, &auio); 522 if (error != 0) { 523 freemsg(head); 524 return (error); 525 } 526 dmp->b_wptr += iov_len; 527 } 528 } else { 529 file_t *ffp; 530 vnode_t *readvp; 531 532 if ((ffp = getf(sfv->sfv_fd)) == NULL) { 533 freemsg(head); 534 return (EBADF); 535 } 536 537 if ((ffp->f_flag & FREAD) == 0) { 538 releasef(sfv->sfv_fd); 539 freemsg(head); 540 return (EACCES); 541 } 542 543 readvp = ffp->f_vnode; 544 if (readvp->v_type != VREG) { 545 releasef(sfv->sfv_fd); 546 freemsg(head); 547 return (EINVAL); 548 } 549 550 /* 551 * No point reading and writing to same vp, 552 * as long as both are regular files. readvp is not 553 * locked; but since we got it from an open file the 554 * contents will be valid during the time of access. 555 */ 556 557 if (vn_compare(vp, readvp)) { 558 releasef(sfv->sfv_fd); 559 freemsg(head); 560 return (EINVAL); 561 } 562 563 /* 564 * Note: we assume readvp != vp. "vp" is already 565 * locked, and "readvp" must not be. 566 */ 567 568 if (readvp < vp) { 569 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 570 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 571 NULL); 572 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 573 } else { 574 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 575 NULL); 576 } 577 578 /* Same checks as in pread */ 579 if (sfv_off > maxoff) { 580 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 581 releasef(sfv->sfv_fd); 582 freemsg(head); 583 return (EINVAL); 584 } 585 if (sfv_off + sfv_len > maxoff) { 586 total_size -= (sfv_off + sfv_len - maxoff); 587 sfv_len = (ssize_t)((offset_t)maxoff - 588 sfv_off); 589 } 590 591 while (sfv_len > 0) { 592 if (buf_left == 0) { 593 tmp = dmp; 594 buf_left = MIN(total_size, maxblk); 595 iov_len = MIN(buf_left, sfv_len); 596 dmp = allocb(buf_left + extra, BPRI_HI); 597 if (dmp == NULL) { 598 VOP_RWUNLOCK(readvp, 599 V_WRITELOCK_FALSE, NULL); 600 releasef(sfv->sfv_fd); 601 freemsg(head); 602 return (ENOMEM); 603 } 604 dmp->b_wptr = dmp->b_rptr = 605 dmp->b_rptr + wroff; 606 tmp->b_cont = dmp; 607 } else { 608 iov_len = MIN(buf_left, sfv_len); 609 } 610 aiov.iov_base = (caddr_t)dmp->b_wptr; 611 aiov.iov_len = iov_len; 612 auio.uio_loffset = sfv_off; 613 auio.uio_iov = &aiov; 614 auio.uio_iovcnt = 1; 615 auio.uio_resid = iov_len; 616 auio.uio_segflg = UIO_SYSSPACE; 617 auio.uio_llimit = MAXOFFSET_T; 618 auio.uio_fmode = ffp->f_flag; 619 ioflag = auio.uio_fmode & 620 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 621 622 /* 623 * If read sync is not asked for, 624 * filter sync flags 625 */ 626 if ((ioflag & FRSYNC) == 0) 627 ioflag &= ~(FSYNC|FDSYNC); 628 error = VOP_READ(readvp, &auio, ioflag, 629 fp->f_cred, NULL); 630 if (error != 0) { 631 /* 632 * If we were reading a pipe (currently 633 * not implemented), we may now loose 634 * data. 635 */ 636 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 637 NULL); 638 releasef(sfv->sfv_fd); 639 freemsg(head); 640 return (error); 641 } 642 643 /* 644 * Check how much data was really read. 645 * Decrement the 'len' and increment the 646 * 'off' appropriately. 647 */ 648 cnt = iov_len - auio.uio_resid; 649 if (cnt == 0) { 650 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 651 NULL); 652 releasef(sfv->sfv_fd); 653 freemsg(head); 654 return (EINVAL); 655 } 656 sfv_len -= cnt; 657 sfv_off += cnt; 658 total_size -= cnt; 659 buf_left -= cnt; 660 661 dmp->b_wptr += cnt; 662 } 663 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 664 releasef(sfv->sfv_fd); 665 } 666 sfv++; 667 } 668 669 ASSERT(total_size == 0); 670 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &head); 671 if (error != 0) { 672 if (head != NULL) 673 freemsg(head); 674 return (error); 675 } 676 ttolwp(curthread)->lwp_ru.ioch += (ulong_t)size; 677 *count += size; 678 679 return (0); 680 } 681 682 683 int 684 sendvec_chunk(file_t *fp, u_offset_t *fileoff, struct sendfilevec *sfv, 685 int copy_cnt, ssize_t *count) 686 { 687 struct vnode *vp; 688 struct uio auio; 689 struct iovec aiov; 690 ushort_t fflag; 691 int ioflag; 692 int i, error; 693 size_t cnt; 694 ssize_t sfv_len; 695 u_offset_t sfv_off; 696 #ifdef _SYSCALL32_IMPL 697 model_t model = get_udatamodel(); 698 u_offset_t maxoff = (model == DATAMODEL_ILP32) ? 699 MAXOFF32_T : MAXOFFSET_T; 700 #else 701 const u_offset_t maxoff = MAXOFF32_T; 702 #endif 703 mblk_t *dmp = NULL; 704 char *buf = NULL; 705 size_t extra = 0; 706 int maxblk, wroff, tail_len; 707 struct sonode *so; 708 stdata_t *stp; 709 struct nmsghdr msg; 710 711 maxblk = 0; 712 wroff = 0; 713 fflag = fp->f_flag; 714 vp = fp->f_vnode; 715 so = NULL; 716 stp = NULL; 717 718 if (vp->v_type == VSOCK) { 719 so = VTOSO(vp); 720 if (vp->v_stream != NULL) { 721 stp = vp->v_stream; 722 wroff = (int)stp->sd_wroff; 723 tail_len = (int)stp->sd_tail; 724 maxblk = (int)stp->sd_maxblk; 725 } else { 726 stp = NULL; 727 wroff = so->so_proto_props.sopp_wroff; 728 tail_len = so->so_proto_props.sopp_tail; 729 maxblk = so->so_proto_props.sopp_maxblk; 730 } 731 extra = wroff + tail_len; 732 } 733 734 bzero(&msg, sizeof (msg)); 735 auio.uio_extflg = UIO_COPY_DEFAULT; 736 for (i = 0; i < copy_cnt; i++) { 737 if (ISSIG(curthread, JUSTLOOKING)) 738 return (EINTR); 739 740 /* 741 * Do similar checks as "write" as we are writing 742 * sfv_len bytes into "vp". 743 */ 744 sfv_len = (ssize_t)sfv->sfv_len; 745 746 if (sfv_len == 0) { 747 sfv++; 748 continue; 749 } 750 751 if (vp->v_type == VREG) { 752 if (*fileoff >= curproc->p_fsz_ctl) { 753 mutex_enter(&curproc->p_lock); 754 (void) rctl_action( 755 rctlproc_legacy[RLIMIT_FSIZE], 756 curproc->p_rctls, curproc, RCA_SAFE); 757 mutex_exit(&curproc->p_lock); 758 759 return (EFBIG); 760 } 761 762 if (*fileoff >= maxoff) 763 return (EFBIG); 764 765 if (*fileoff + sfv_len > maxoff) 766 return (EINVAL); 767 } 768 769 /* Check for overflow */ 770 #ifdef _SYSCALL32_IMPL 771 if (model == DATAMODEL_ILP32) { 772 if (((ssize32_t)(*count + sfv_len)) < 0) 773 return (EINVAL); 774 } else 775 #endif 776 if ((*count + sfv_len) < 0) 777 return (EINVAL); 778 779 sfv_off = (u_offset_t)(ulong_t)sfv->sfv_off; 780 781 if (sfv->sfv_fd == SFV_FD_SELF) { 782 if (vp->v_type == VSOCK) { 783 while (sfv_len > 0) { 784 size_t iov_len; 785 786 iov_len = sfv_len; 787 /* 788 * Socket filters can limit the mblk 789 * size, so limit reads to maxblk if 790 * there are filters present. 791 */ 792 if (so->so_filter_active > 0 && 793 maxblk != INFPSZ) 794 iov_len = MIN(iov_len, maxblk); 795 796 aiov.iov_len = iov_len; 797 aiov.iov_base = 798 (caddr_t)(uintptr_t)sfv_off; 799 800 auio.uio_iov = &aiov; 801 auio.uio_iovcnt = 1; 802 auio.uio_loffset = *fileoff; 803 auio.uio_segflg = UIO_USERSPACE; 804 auio.uio_fmode = fflag; 805 auio.uio_llimit = curproc->p_fsz_ctl; 806 auio.uio_resid = iov_len; 807 808 dmp = allocb(iov_len + extra, BPRI_HI); 809 if (dmp == NULL) 810 return (ENOMEM); 811 dmp->b_wptr = dmp->b_rptr = 812 dmp->b_rptr + wroff; 813 error = uiomove((caddr_t)dmp->b_wptr, 814 iov_len, UIO_WRITE, &auio); 815 if (error != 0) { 816 freeb(dmp); 817 return (error); 818 } 819 dmp->b_wptr += iov_len; 820 error = socket_sendmblk(VTOSO(vp), 821 &msg, fflag, CRED(), &dmp); 822 823 if (error != 0) { 824 if (dmp != NULL) 825 freeb(dmp); 826 return (error); 827 } 828 ttolwp(curthread)->lwp_ru.ioch += 829 (ulong_t)iov_len; 830 *count += iov_len; 831 sfv_len -= iov_len; 832 sfv_off += iov_len; 833 } 834 } else { 835 aiov.iov_len = sfv_len; 836 aiov.iov_base = (caddr_t)(uintptr_t)sfv_off; 837 838 auio.uio_iov = &aiov; 839 auio.uio_iovcnt = 1; 840 auio.uio_loffset = *fileoff; 841 auio.uio_segflg = UIO_USERSPACE; 842 auio.uio_fmode = fflag; 843 auio.uio_llimit = curproc->p_fsz_ctl; 844 auio.uio_resid = sfv_len; 845 846 ioflag = auio.uio_fmode & 847 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 848 while (sfv_len > 0) { 849 error = VOP_WRITE(vp, &auio, ioflag, 850 fp->f_cred, NULL); 851 cnt = sfv_len - auio.uio_resid; 852 sfv_len -= cnt; 853 ttolwp(curthread)->lwp_ru.ioch += 854 (ulong_t)cnt; 855 *fileoff += cnt; 856 *count += cnt; 857 if (error != 0) 858 return (error); 859 } 860 } 861 } else { 862 int segmapit = 0; 863 file_t *ffp; 864 vnode_t *readvp; 865 struct vnode *realvp; 866 size_t size; 867 caddr_t ptr; 868 869 if ((ffp = getf(sfv->sfv_fd)) == NULL) 870 return (EBADF); 871 872 if ((ffp->f_flag & FREAD) == 0) { 873 releasef(sfv->sfv_fd); 874 return (EBADF); 875 } 876 877 readvp = ffp->f_vnode; 878 if (VOP_REALVP(readvp, &realvp, NULL) == 0) 879 readvp = realvp; 880 if (readvp->v_type != VREG) { 881 releasef(sfv->sfv_fd); 882 return (EINVAL); 883 } 884 885 /* 886 * No point reading and writing to same vp, 887 * as long as both are regular files. readvp is not 888 * locked; but since we got it from an open file the 889 * contents will be valid during the time of access. 890 */ 891 if (vn_compare(vp, readvp)) { 892 releasef(sfv->sfv_fd); 893 return (EINVAL); 894 } 895 896 /* 897 * Note: we assume readvp != vp. "vp" is already 898 * locked, and "readvp" must not be. 899 */ 900 if (readvp < vp) { 901 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 902 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 903 NULL); 904 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 905 } else { 906 (void) VOP_RWLOCK(readvp, V_WRITELOCK_FALSE, 907 NULL); 908 } 909 910 /* Same checks as in pread */ 911 if (sfv_off > maxoff) { 912 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 913 releasef(sfv->sfv_fd); 914 return (EINVAL); 915 } 916 if (sfv_off + sfv_len > maxoff) { 917 sfv_len = (ssize_t)((offset_t)maxoff - 918 sfv_off); 919 } 920 /* Find the native blocksize to transfer data */ 921 size = MIN(vp->v_vfsp->vfs_bsize, 922 readvp->v_vfsp->vfs_bsize); 923 size = sfv_len < size ? sfv_len : size; 924 925 if (vp->v_type != VSOCK) { 926 segmapit = 0; 927 buf = kmem_alloc(size, KM_NOSLEEP); 928 if (buf == NULL) { 929 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 930 NULL); 931 releasef(sfv->sfv_fd); 932 return (ENOMEM); 933 } 934 } else { 935 uint_t copyflag; 936 937 copyflag = stp != NULL ? stp->sd_copyflag : 938 so->so_proto_props.sopp_zcopyflag; 939 940 /* 941 * Socket filters can limit the mblk size, 942 * so limit reads to maxblk if there are 943 * filters present. 944 */ 945 if (so->so_filter_active > 0 && 946 maxblk != INFPSZ) 947 size = MIN(size, maxblk); 948 949 if (vn_has_flocks(readvp) || 950 readvp->v_flag & VNOMAP || 951 copyflag & STZCVMUNSAFE) { 952 segmapit = 0; 953 } else if (copyflag & STZCVMSAFE) { 954 segmapit = 1; 955 } else { 956 int on = 1; 957 if (socket_setsockopt(VTOSO(vp), 958 SOL_SOCKET, SO_SND_COPYAVOID, 959 &on, sizeof (on), CRED()) == 0) 960 segmapit = 1; 961 } 962 } 963 964 if (segmapit) { 965 struct vattr va; 966 boolean_t nowait; 967 968 va.va_mask = AT_SIZE; 969 error = VOP_GETATTR(readvp, &va, 0, kcred, 970 NULL); 971 if (error != 0 || sfv_off >= va.va_size) { 972 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 973 NULL); 974 releasef(sfv->sfv_fd); 975 return (error); 976 } 977 /* Read as much as possible. */ 978 if (sfv_off + sfv_len > va.va_size) 979 sfv_len = va.va_size - sfv_off; 980 981 nowait = (sfv->sfv_flag & SFV_NOWAIT) != 0; 982 error = snf_segmap(fp, readvp, sfv_off, 983 (u_offset_t)sfv_len, (ssize_t *)&cnt, 984 nowait); 985 releasef(sfv->sfv_fd); 986 *count += cnt; 987 if (error) 988 return (error); 989 sfv++; 990 continue; 991 } 992 993 while (sfv_len > 0) { 994 size_t iov_len; 995 996 iov_len = MIN(size, sfv_len); 997 998 if (vp->v_type == VSOCK) { 999 dmp = allocb(iov_len + extra, BPRI_HI); 1000 if (dmp == NULL) { 1001 VOP_RWUNLOCK(readvp, 1002 V_WRITELOCK_FALSE, NULL); 1003 releasef(sfv->sfv_fd); 1004 return (ENOMEM); 1005 } 1006 dmp->b_wptr = dmp->b_rptr = 1007 dmp->b_rptr + wroff; 1008 ptr = (caddr_t)dmp->b_rptr; 1009 } else { 1010 ptr = buf; 1011 } 1012 1013 aiov.iov_base = ptr; 1014 aiov.iov_len = iov_len; 1015 auio.uio_loffset = sfv_off; 1016 auio.uio_iov = &aiov; 1017 auio.uio_iovcnt = 1; 1018 auio.uio_resid = iov_len; 1019 auio.uio_segflg = UIO_SYSSPACE; 1020 auio.uio_llimit = MAXOFFSET_T; 1021 auio.uio_fmode = ffp->f_flag; 1022 ioflag = auio.uio_fmode & 1023 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 1024 1025 /* 1026 * If read sync is not asked for, 1027 * filter sync flags 1028 */ 1029 if ((ioflag & FRSYNC) == 0) 1030 ioflag &= ~(FSYNC|FDSYNC); 1031 error = VOP_READ(readvp, &auio, ioflag, 1032 fp->f_cred, NULL); 1033 if (error != 0) { 1034 /* 1035 * If we were reading a pipe (currently 1036 * not implemented), we may now lose 1037 * data. 1038 */ 1039 if (vp->v_type == VSOCK) 1040 freeb(dmp); 1041 else 1042 kmem_free(buf, size); 1043 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 1044 NULL); 1045 releasef(sfv->sfv_fd); 1046 return (error); 1047 } 1048 1049 /* 1050 * Check how much data was really read. 1051 * Decrement the 'len' and increment the 1052 * 'off' appropriately. 1053 */ 1054 cnt = iov_len - auio.uio_resid; 1055 if (cnt == 0) { 1056 if (vp->v_type == VSOCK) 1057 freeb(dmp); 1058 else 1059 kmem_free(buf, size); 1060 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, 1061 NULL); 1062 releasef(sfv->sfv_fd); 1063 return (EINVAL); 1064 } 1065 sfv_len -= cnt; 1066 sfv_off += cnt; 1067 1068 if (vp->v_type == VSOCK) { 1069 dmp->b_wptr = dmp->b_rptr + cnt; 1070 1071 error = socket_sendmblk(VTOSO(vp), 1072 &msg, fflag, CRED(), &dmp); 1073 1074 if (error != 0) { 1075 if (dmp != NULL) 1076 freeb(dmp); 1077 VOP_RWUNLOCK(readvp, 1078 V_WRITELOCK_FALSE, NULL); 1079 releasef(sfv->sfv_fd); 1080 return (error); 1081 } 1082 1083 ttolwp(curthread)->lwp_ru.ioch += 1084 (ulong_t)cnt; 1085 *count += cnt; 1086 } else { 1087 1088 aiov.iov_base = ptr; 1089 aiov.iov_len = cnt; 1090 auio.uio_loffset = *fileoff; 1091 auio.uio_resid = cnt; 1092 auio.uio_iov = &aiov; 1093 auio.uio_iovcnt = 1; 1094 auio.uio_segflg = UIO_SYSSPACE; 1095 auio.uio_llimit = curproc->p_fsz_ctl; 1096 auio.uio_fmode = fflag; 1097 ioflag = auio.uio_fmode & 1098 (FAPPEND|FSYNC|FDSYNC|FRSYNC); 1099 error = VOP_WRITE(vp, &auio, ioflag, 1100 fp->f_cred, NULL); 1101 1102 /* 1103 * Check how much data was written. 1104 * Increment the 'len' and decrement the 1105 * 'off' if all the data was not 1106 * written. 1107 */ 1108 cnt -= auio.uio_resid; 1109 sfv_len += auio.uio_resid; 1110 sfv_off -= auio.uio_resid; 1111 ttolwp(curthread)->lwp_ru.ioch += 1112 (ulong_t)cnt; 1113 *fileoff += cnt; 1114 *count += cnt; 1115 if (error != 0) { 1116 kmem_free(buf, size); 1117 VOP_RWUNLOCK(readvp, 1118 V_WRITELOCK_FALSE, NULL); 1119 releasef(sfv->sfv_fd); 1120 return (error); 1121 } 1122 } 1123 } 1124 if (buf) { 1125 kmem_free(buf, size); 1126 buf = NULL; 1127 } 1128 VOP_RWUNLOCK(readvp, V_WRITELOCK_FALSE, NULL); 1129 releasef(sfv->sfv_fd); 1130 } 1131 sfv++; 1132 } 1133 return (0); 1134 } 1135 1136 ssize_t 1137 sendfilev(int opcode, int fildes, const struct sendfilevec *vec, int sfvcnt, 1138 size_t *xferred) 1139 { 1140 int error = 0; 1141 int first_vector_error = 0; 1142 file_t *fp; 1143 struct vnode *vp; 1144 struct sonode *so = NULL; 1145 u_offset_t fileoff; 1146 int copy_cnt; 1147 const struct sendfilevec *copy_vec; 1148 struct sendfilevec sfv[SEND_MAX_CHUNK]; 1149 ssize_t count = 0; 1150 #ifdef _SYSCALL32_IMPL 1151 struct ksendfilevec32 sfv32[SEND_MAX_CHUNK]; 1152 #endif 1153 ssize_t total_size; 1154 int i; 1155 boolean_t is_sock = B_FALSE; 1156 int maxblk = 0; 1157 1158 if (sfvcnt <= 0) 1159 return (set_errno(EINVAL)); 1160 1161 if ((fp = getf(fildes)) == NULL) 1162 return (set_errno(EBADF)); 1163 1164 if (((fp->f_flag) & FWRITE) == 0) { 1165 error = EBADF; 1166 goto err; 1167 } 1168 1169 fileoff = fp->f_offset; 1170 vp = fp->f_vnode; 1171 1172 switch (vp->v_type) { 1173 case VSOCK: 1174 so = VTOSO(vp); 1175 is_sock = B_TRUE; 1176 if (SOCK_IS_NONSTR(so)) { 1177 maxblk = so->so_proto_props.sopp_maxblk; 1178 } else { 1179 maxblk = (int)vp->v_stream->sd_maxblk; 1180 } 1181 break; 1182 case VREG: 1183 break; 1184 default: 1185 error = EINVAL; 1186 goto err; 1187 } 1188 1189 switch (opcode) { 1190 case SENDFILEV : 1191 break; 1192 #if defined(_SYSCALL32_IMPL) || defined(_ILP32) 1193 case SENDFILEV64 : 1194 return (sendvec64(fp, (struct ksendfilevec64 *)vec, sfvcnt, 1195 (size32_t *)xferred, fildes)); 1196 #endif 1197 default : 1198 error = ENOSYS; 1199 break; 1200 } 1201 1202 (void) VOP_RWLOCK(vp, V_WRITELOCK_TRUE, NULL); 1203 copy_vec = vec; 1204 1205 do { 1206 total_size = 0; 1207 copy_cnt = MIN(sfvcnt, SEND_MAX_CHUNK); 1208 #ifdef _SYSCALL32_IMPL 1209 /* 32-bit callers need to have their iovec expanded. */ 1210 if (get_udatamodel() == DATAMODEL_ILP32) { 1211 if (copyin(copy_vec, sfv32, 1212 copy_cnt * sizeof (ksendfilevec32_t))) { 1213 error = EFAULT; 1214 break; 1215 } 1216 1217 for (i = 0; i < copy_cnt; i++) { 1218 sfv[i].sfv_fd = sfv32[i].sfv_fd; 1219 sfv[i].sfv_off = 1220 (off_t)(uint32_t)sfv32[i].sfv_off; 1221 sfv[i].sfv_len = (size_t)sfv32[i].sfv_len; 1222 total_size += sfv[i].sfv_len; 1223 sfv[i].sfv_flag = sfv32[i].sfv_flag; 1224 /* 1225 * Individual elements of the vector must not 1226 * wrap or overflow, as later math is signed. 1227 * Equally total_size needs to be checked after 1228 * each vector is added in, to be sure that 1229 * rogue values haven't overflowed the counter. 1230 */ 1231 if (((ssize32_t)sfv[i].sfv_len < 0) || 1232 ((ssize32_t)total_size < 0)) { 1233 /* 1234 * Truncate the vector to send data 1235 * described by elements before the 1236 * error. 1237 */ 1238 copy_cnt = i; 1239 first_vector_error = EINVAL; 1240 /* total_size can't be trusted */ 1241 if ((ssize32_t)total_size < 0) 1242 error = EINVAL; 1243 break; 1244 } 1245 } 1246 /* Nothing to do, process errors */ 1247 if (copy_cnt == 0) 1248 break; 1249 1250 } else { 1251 #endif 1252 if (copyin(copy_vec, sfv, 1253 copy_cnt * sizeof (sendfilevec_t))) { 1254 error = EFAULT; 1255 break; 1256 } 1257 1258 for (i = 0; i < copy_cnt; i++) { 1259 total_size += sfv[i].sfv_len; 1260 /* 1261 * Individual elements of the vector must not 1262 * wrap or overflow, as later math is signed. 1263 * Equally total_size needs to be checked after 1264 * each vector is added in, to be sure that 1265 * rogue values haven't overflowed the counter. 1266 */ 1267 if (((ssize_t)sfv[i].sfv_len < 0) || 1268 (total_size < 0)) { 1269 /* 1270 * Truncate the vector to send data 1271 * described by elements before the 1272 * error. 1273 */ 1274 copy_cnt = i; 1275 first_vector_error = EINVAL; 1276 /* total_size can't be trusted */ 1277 if (total_size < 0) 1278 error = EINVAL; 1279 break; 1280 } 1281 } 1282 /* Nothing to do, process errors */ 1283 if (copy_cnt == 0) 1284 break; 1285 #ifdef _SYSCALL32_IMPL 1286 } 1287 #endif 1288 1289 /* 1290 * The task between deciding to use sendvec_small_chunk 1291 * and sendvec_chunk is dependant on multiple things: 1292 * 1293 * i) latency is important for smaller files. So if the 1294 * data is smaller than 'tcp_slow_start_initial' times 1295 * maxblk, then use sendvec_small_chunk which creates 1296 * maxblk size mblks and chains them together and sends 1297 * them to TCP in one shot. It also leaves 'wroff' size 1298 * space for the headers in each mblk. 1299 * 1300 * ii) for total size bigger than 'tcp_slow_start_initial' 1301 * time maxblk, its probably real file data which is 1302 * dominating. So its better to use sendvec_chunk because 1303 * performance goes to dog if we don't do pagesize reads. 1304 * sendvec_chunk will do pagesize reads and write them 1305 * in pagesize mblks to TCP. 1306 * 1307 * Side Notes: A write to file has not been optimized. 1308 * Future zero copy code will plugin into sendvec_chunk 1309 * only because doing zero copy for files smaller then 1310 * pagesize is useless. 1311 * 1312 * Note, if socket has NL7C enabled then call NL7C's 1313 * senfilev() function to consume the sfv[]. 1314 */ 1315 if (is_sock) { 1316 if (!SOCK_IS_NONSTR(so) && 1317 _SOTOTPI(so)->sti_nl7c_flags != 0) { 1318 error = nl7c_sendfilev(so, &fileoff, 1319 sfv, copy_cnt, &count); 1320 } else if ((total_size <= (4 * maxblk)) && 1321 error == 0) { 1322 error = sendvec_small_chunk(fp, 1323 &fileoff, sfv, copy_cnt, 1324 total_size, maxblk, &count); 1325 } else { 1326 error = sendvec_chunk(fp, &fileoff, 1327 sfv, copy_cnt, &count); 1328 } 1329 } else { 1330 ASSERT(vp->v_type == VREG); 1331 error = sendvec_chunk(fp, &fileoff, sfv, copy_cnt, 1332 &count); 1333 } 1334 1335 1336 #ifdef _SYSCALL32_IMPL 1337 if (get_udatamodel() == DATAMODEL_ILP32) { 1338 copy_vec = (const struct sendfilevec *) 1339 ((char *)copy_vec + 1340 (copy_cnt * sizeof (ksendfilevec32_t))); 1341 } else 1342 #endif 1343 copy_vec += copy_cnt; 1344 sfvcnt -= copy_cnt; 1345 1346 /* Process all vector members up to first error */ 1347 } while ((sfvcnt > 0) && first_vector_error == 0 && error == 0); 1348 1349 if (vp->v_type == VREG) 1350 fp->f_offset += count; 1351 1352 VOP_RWUNLOCK(vp, V_WRITELOCK_TRUE, NULL); 1353 1354 #ifdef _SYSCALL32_IMPL 1355 if (get_udatamodel() == DATAMODEL_ILP32) { 1356 ssize32_t count32 = (ssize32_t)count; 1357 if (copyout(&count32, xferred, sizeof (count32))) 1358 error = EFAULT; 1359 releasef(fildes); 1360 if (error != 0) 1361 return (set_errno(error)); 1362 if (first_vector_error != 0) 1363 return (set_errno(first_vector_error)); 1364 return (count32); 1365 } 1366 #endif 1367 if (copyout(&count, xferred, sizeof (count))) 1368 error = EFAULT; 1369 releasef(fildes); 1370 if (error != 0) 1371 return (set_errno(error)); 1372 if (first_vector_error != 0) 1373 return (set_errno(first_vector_error)); 1374 return (count); 1375 err: 1376 ASSERT(error != 0); 1377 releasef(fildes); 1378 return (set_errno(error)); 1379 } 1380