1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright 2015, Joyent, Inc. All rights reserved. 25 * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved. 26 * Copyright 2015 Nexenta Systems, Inc. All rights reserved. 27 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association. 28 * Copyright 2022 Garrett D'Amore 29 */ 30 31 #include <sys/types.h> 32 #include <sys/t_lock.h> 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/buf.h> 36 #include <sys/conf.h> 37 #include <sys/cred.h> 38 #include <sys/kmem.h> 39 #include <sys/sysmacros.h> 40 #include <sys/vfs.h> 41 #include <sys/vnode.h> 42 #include <sys/debug.h> 43 #include <sys/errno.h> 44 #include <sys/time.h> 45 #include <sys/file.h> 46 #include <sys/user.h> 47 #include <sys/stream.h> 48 #include <sys/strsubr.h> 49 #include <sys/strsun.h> 50 #include <sys/sunddi.h> 51 #include <sys/esunddi.h> 52 #include <sys/flock.h> 53 #include <sys/modctl.h> 54 #include <sys/cmn_err.h> 55 #include <sys/vmsystm.h> 56 #include <sys/policy.h> 57 #include <sys/limits.h> 58 59 #include <sys/socket.h> 60 #include <sys/socketvar.h> 61 62 #include <sys/isa_defs.h> 63 #include <sys/inttypes.h> 64 #include <sys/systm.h> 65 #include <sys/cpuvar.h> 66 #include <sys/filio.h> 67 #include <sys/sendfile.h> 68 #include <sys/ddi.h> 69 #include <vm/seg.h> 70 #include <vm/seg_map.h> 71 #include <vm/seg_kpm.h> 72 73 #include <fs/sockfs/sockcommon.h> 74 #include <fs/sockfs/sockfilter_impl.h> 75 #include <fs/sockfs/socktpi.h> 76 77 #ifdef SOCK_TEST 78 int do_useracc = 1; /* Controlled by setting SO_DEBUG to 4 */ 79 #else 80 #define do_useracc 1 81 #endif /* SOCK_TEST */ 82 83 extern int xnet_truncate_print; 84 85 /* 86 * Kernel component of socket creation. 87 * 88 * The socket library determines which version number to use. 89 * First the library calls this with a NULL devpath. If this fails 90 * to find a transport (using solookup) the library will look in /etc/netconfig 91 * for the appropriate transport. If one is found it will pass in the 92 * devpath for the kernel to use. 93 */ 94 int 95 so_socket(int family, int type_w_flags, int protocol, char *devpath, 96 int version) 97 { 98 struct sonode *so; 99 vnode_t *vp; 100 struct file *fp; 101 int fd; 102 int error; 103 int type; 104 105 type = type_w_flags & SOCK_TYPE_MASK; 106 type_w_flags &= ~SOCK_TYPE_MASK; 107 if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK)) 108 return (set_errno(EINVAL)); 109 110 if (devpath != NULL) { 111 char *buf; 112 size_t kdevpathlen = 0; 113 114 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP); 115 if ((error = copyinstr(devpath, buf, 116 MAXPATHLEN, &kdevpathlen)) != 0) { 117 kmem_free(buf, MAXPATHLEN); 118 return (set_errno(error)); 119 } 120 so = socket_create(family, type, protocol, buf, NULL, 121 SOCKET_SLEEP, version, CRED(), &error); 122 kmem_free(buf, MAXPATHLEN); 123 } else { 124 so = socket_create(family, type, protocol, NULL, NULL, 125 SOCKET_SLEEP, version, CRED(), &error); 126 } 127 if (so == NULL) 128 return (set_errno(error)); 129 130 /* Allocate a file descriptor for the socket */ 131 vp = SOTOV(so); 132 if (error = falloc(vp, FWRITE|FREAD, &fp, &fd)) { 133 (void) socket_close(so, 0, CRED()); 134 socket_destroy(so); 135 return (set_errno(error)); 136 } 137 138 /* 139 * Now fill in the entries that falloc reserved 140 */ 141 if (type_w_flags & SOCK_NDELAY) { 142 so->so_state |= SS_NDELAY; 143 fp->f_flag |= FNDELAY; 144 } 145 if (type_w_flags & SOCK_NONBLOCK) { 146 so->so_state |= SS_NONBLOCK; 147 fp->f_flag |= FNONBLOCK; 148 } 149 mutex_exit(&fp->f_tlock); 150 setf(fd, fp); 151 if ((type_w_flags & SOCK_CLOEXEC) != 0) { 152 f_setfd(fd, FD_CLOEXEC); 153 } 154 155 return (fd); 156 } 157 158 /* 159 * Map from a file descriptor to a socket node. 160 * Returns with the file descriptor held i.e. the caller has to 161 * use releasef when done with the file descriptor. 162 */ 163 struct sonode * 164 getsonode(int sock, int *errorp, file_t **fpp) 165 { 166 file_t *fp; 167 vnode_t *vp; 168 struct sonode *so; 169 170 if ((fp = getf(sock)) == NULL) { 171 *errorp = EBADF; 172 eprintline(*errorp); 173 return (NULL); 174 } 175 vp = fp->f_vnode; 176 /* Check if it is a socket */ 177 if (vp->v_type != VSOCK) { 178 releasef(sock); 179 *errorp = ENOTSOCK; 180 eprintline(*errorp); 181 return (NULL); 182 } 183 /* 184 * Use the stream head to find the real socket vnode. 185 * This is needed when namefs sits above sockfs. 186 */ 187 if (vp->v_stream) { 188 ASSERT(vp->v_stream->sd_vnode); 189 vp = vp->v_stream->sd_vnode; 190 191 so = VTOSO(vp); 192 if (so->so_version == SOV_STREAM) { 193 releasef(sock); 194 *errorp = ENOTSOCK; 195 eprintsoline(so, *errorp); 196 return (NULL); 197 } 198 } else { 199 so = VTOSO(vp); 200 } 201 if (fpp) 202 *fpp = fp; 203 return (so); 204 } 205 206 /* 207 * Allocate and copyin a sockaddr. 208 * Ensures NULL termination for AF_UNIX addresses by extending them 209 * with one NULL byte if need be. Verifies that the length is not 210 * excessive to prevent an application from consuming all of kernel 211 * memory. Returns NULL when an error occurred. 212 */ 213 static struct sockaddr * 214 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp, 215 int *errorp) 216 { 217 char *faddr; 218 size_t namelen = (size_t)*namelenp; 219 220 ASSERT(namelen != 0); 221 if (namelen > SO_MAXARGSIZE) { 222 *errorp = EINVAL; 223 eprintsoline(so, *errorp); 224 return (NULL); 225 } 226 227 faddr = (char *)kmem_alloc(namelen, KM_SLEEP); 228 if (copyin(name, faddr, namelen)) { 229 kmem_free(faddr, namelen); 230 *errorp = EFAULT; 231 eprintsoline(so, *errorp); 232 return (NULL); 233 } 234 235 /* 236 * Add space for NULL termination if needed. 237 * Do a quick check if the last byte is NUL. 238 */ 239 if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') { 240 /* Check if there is any NULL termination */ 241 size_t i; 242 int foundnull = 0; 243 244 for (i = sizeof (name->sa_family); i < namelen; i++) { 245 if (faddr[i] == '\0') { 246 foundnull = 1; 247 break; 248 } 249 } 250 if (!foundnull) { 251 /* Add extra byte for NUL padding */ 252 char *nfaddr; 253 254 nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP); 255 bcopy(faddr, nfaddr, namelen); 256 kmem_free(faddr, namelen); 257 258 /* NUL terminate */ 259 nfaddr[namelen] = '\0'; 260 namelen++; 261 ASSERT((socklen_t)namelen == namelen); 262 *namelenp = (socklen_t)namelen; 263 faddr = nfaddr; 264 } 265 } 266 return ((struct sockaddr *)faddr); 267 } 268 269 /* 270 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL. 271 */ 272 static int 273 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr, 274 socklen_t klen) 275 { 276 if (uaddr != NULL) { 277 if (ulen > klen) 278 ulen = klen; 279 280 if (ulen != 0) { 281 if (copyout(kaddr, uaddr, ulen)) 282 return (EFAULT); 283 } 284 } else 285 ulen = 0; 286 287 if (ulenp != NULL) { 288 if (copyout(&ulen, ulenp, sizeof (ulen))) 289 return (EFAULT); 290 } 291 return (0); 292 } 293 294 /* 295 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL. 296 * If klen is greater than ulen it still uses the non-truncated 297 * klen to update ulenp. 298 */ 299 static int 300 copyout_name(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr, 301 socklen_t klen) 302 { 303 if (uaddr != NULL) { 304 if (ulen >= klen) 305 ulen = klen; 306 else if (ulen != 0 && xnet_truncate_print) { 307 printf("sockfs: truncating copyout of address using " 308 "XNET semantics for pid = %d. Lengths %d, %d\n", 309 curproc->p_pid, klen, ulen); 310 } 311 312 if (ulen != 0) { 313 if (copyout(kaddr, uaddr, ulen)) 314 return (EFAULT); 315 } else 316 klen = 0; 317 } else 318 klen = 0; 319 320 if (ulenp != NULL) { 321 if (copyout(&klen, ulenp, sizeof (klen))) 322 return (EFAULT); 323 } 324 return (0); 325 } 326 327 /* 328 * The socketpair() code in libsocket creates two sockets (using 329 * the /etc/netconfig fallback if needed) before calling this routine 330 * to connect the two sockets together. 331 * 332 * For a SOCK_STREAM socketpair a listener is needed - in that case this 333 * routine will create a new file descriptor as part of accepting the 334 * connection. The library socketpair() will check if svs[2] has changed 335 * in which case it will close the changed fd. 336 * 337 * Note that this code could use the TPI feature of accepting the connection 338 * on the listening endpoint. However, that would require significant changes 339 * to soaccept. 340 */ 341 int 342 so_socketpair(int sv[2]) 343 { 344 int svs[2]; 345 struct sonode *so1, *so2; 346 int error; 347 int orig_flags; 348 struct sockaddr_ux *name; 349 size_t namelen; 350 sotpi_info_t *sti1; 351 sotpi_info_t *sti2; 352 353 dprint(1, ("so_socketpair(%p)\n", (void *)sv)); 354 355 error = useracc(sv, sizeof (svs), B_WRITE); 356 if (error && do_useracc) 357 return (set_errno(EFAULT)); 358 359 if (copyin(sv, svs, sizeof (svs))) 360 return (set_errno(EFAULT)); 361 362 if ((so1 = getsonode(svs[0], &error, NULL)) == NULL) 363 return (set_errno(error)); 364 365 if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) { 366 releasef(svs[0]); 367 return (set_errno(error)); 368 } 369 370 if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) { 371 error = EOPNOTSUPP; 372 goto done; 373 } 374 375 sti1 = SOTOTPI(so1); 376 sti2 = SOTOTPI(so2); 377 378 /* 379 * The code below makes assumptions about the "sockfs" implementation. 380 * So make sure that the correct implementation is really used. 381 */ 382 ASSERT(so1->so_ops == &sotpi_sonodeops); 383 ASSERT(so2->so_ops == &sotpi_sonodeops); 384 385 if (so1->so_type == SOCK_DGRAM) { 386 /* 387 * Bind both sockets and connect them with each other. 388 * Need to allocate name/namelen for soconnect. 389 */ 390 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED()); 391 if (error) { 392 eprintsoline(so1, error); 393 goto done; 394 } 395 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED()); 396 if (error) { 397 eprintsoline(so2, error); 398 goto done; 399 } 400 namelen = sizeof (struct sockaddr_ux); 401 name = kmem_alloc(namelen, KM_SLEEP); 402 name->sou_family = AF_UNIX; 403 name->sou_addr = sti2->sti_ux_laddr; 404 error = socket_connect(so1, 405 (struct sockaddr *)name, 406 (socklen_t)namelen, 407 0, _SOCONNECT_NOXLATE, CRED()); 408 if (error) { 409 kmem_free(name, namelen); 410 eprintsoline(so1, error); 411 goto done; 412 } 413 name->sou_addr = sti1->sti_ux_laddr; 414 error = socket_connect(so2, 415 (struct sockaddr *)name, 416 (socklen_t)namelen, 417 0, _SOCONNECT_NOXLATE, CRED()); 418 kmem_free(name, namelen); 419 if (error) { 420 eprintsoline(so2, error); 421 goto done; 422 } 423 releasef(svs[0]); 424 releasef(svs[1]); 425 } else { 426 /* 427 * Bind both sockets, with so1 being a listener. 428 * Connect so2 to so1 - nonblocking to avoid waiting for 429 * soaccept to complete. 430 * Accept a connection on so1. Pass out the new fd as sv[0]. 431 * The library will detect the changed fd and close 432 * the original one. 433 */ 434 struct sonode *nso; 435 struct vnode *nvp; 436 struct file *nfp; 437 int nfd; 438 439 /* 440 * We could simply call socket_listen() here (which would do the 441 * binding automatically) if the code didn't rely on passing 442 * _SOBIND_NOXLATE to the TPI implementation of socket_bind(). 443 */ 444 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC| 445 _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR, 446 CRED()); 447 if (error) { 448 eprintsoline(so1, error); 449 goto done; 450 } 451 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED()); 452 if (error) { 453 eprintsoline(so2, error); 454 goto done; 455 } 456 457 namelen = sizeof (struct sockaddr_ux); 458 name = kmem_alloc(namelen, KM_SLEEP); 459 name->sou_family = AF_UNIX; 460 name->sou_addr = sti1->sti_ux_laddr; 461 error = socket_connect(so2, 462 (struct sockaddr *)name, 463 (socklen_t)namelen, 464 FNONBLOCK, _SOCONNECT_NOXLATE, CRED()); 465 kmem_free(name, namelen); 466 if (error) { 467 if (error != EINPROGRESS) { 468 eprintsoline(so2, error); goto done; 469 } 470 } 471 472 error = socket_accept(so1, 0, CRED(), &nso); 473 if (error) { 474 eprintsoline(so1, error); 475 goto done; 476 } 477 478 /* wait for so2 being SS_CONNECTED ignoring signals */ 479 mutex_enter(&so2->so_lock); 480 error = sowaitconnected(so2, 0, 1); 481 mutex_exit(&so2->so_lock); 482 if (error != 0) { 483 (void) socket_close(nso, 0, CRED()); 484 socket_destroy(nso); 485 eprintsoline(so2, error); 486 goto done; 487 } 488 489 nvp = SOTOV(nso); 490 if (error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd)) { 491 (void) socket_close(nso, 0, CRED()); 492 socket_destroy(nso); 493 eprintsoline(nso, error); 494 goto done; 495 } 496 /* 497 * copy over FNONBLOCK and FNDELAY flags should they exist 498 */ 499 if (so1->so_state & SS_NONBLOCK) 500 nfp->f_flag |= FNONBLOCK; 501 if (so1->so_state & SS_NDELAY) 502 nfp->f_flag |= FNDELAY; 503 504 /* 505 * fill in the entries that falloc reserved 506 */ 507 mutex_exit(&nfp->f_tlock); 508 setf(nfd, nfp); 509 510 /* 511 * get the original flags before we release 512 */ 513 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0); 514 515 releasef(svs[0]); 516 releasef(svs[1]); 517 518 /* 519 * If FD_CLOEXEC was set on the filedescriptor we're 520 * swapping out, we should set it on the new one too. 521 */ 522 if (orig_flags & FD_CLOEXEC) { 523 f_setfd(nfd, FD_CLOEXEC); 524 } 525 526 /* 527 * The socketpair library routine will close the original 528 * svs[0] when this code passes out a different file 529 * descriptor. 530 */ 531 svs[0] = nfd; 532 533 if (copyout(svs, sv, sizeof (svs))) { 534 (void) closeandsetf(nfd, NULL); 535 eprintline(EFAULT); 536 return (set_errno(EFAULT)); 537 } 538 } 539 return (0); 540 541 done: 542 releasef(svs[0]); 543 releasef(svs[1]); 544 return (set_errno(error)); 545 } 546 547 int 548 bind(int sock, struct sockaddr *name, socklen_t namelen, int version) 549 { 550 struct sonode *so; 551 int error; 552 553 dprint(1, ("bind(%d, %p, %d)\n", 554 sock, (void *)name, namelen)); 555 556 if ((so = getsonode(sock, &error, NULL)) == NULL) 557 return (set_errno(error)); 558 559 /* Allocate and copyin name */ 560 /* 561 * X/Open test does not expect EFAULT with NULL name and non-zero 562 * namelen. 563 */ 564 if (name != NULL && namelen != 0) { 565 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 566 name = copyin_name(so, name, &namelen, &error); 567 if (name == NULL) { 568 releasef(sock); 569 return (set_errno(error)); 570 } 571 } else { 572 name = NULL; 573 namelen = 0; 574 } 575 576 switch (version) { 577 default: 578 error = socket_bind(so, name, namelen, 0, CRED()); 579 break; 580 case SOV_XPG4_2: 581 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED()); 582 break; 583 case SOV_SOCKBSD: 584 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED()); 585 break; 586 } 587 done: 588 releasef(sock); 589 if (name != NULL) 590 kmem_free(name, (size_t)namelen); 591 592 if (error) 593 return (set_errno(error)); 594 return (0); 595 } 596 597 /* ARGSUSED2 */ 598 int 599 listen(int sock, int backlog, int version) 600 { 601 struct sonode *so; 602 int error; 603 604 dprint(1, ("listen(%d, %d)\n", 605 sock, backlog)); 606 607 if ((so = getsonode(sock, &error, NULL)) == NULL) 608 return (set_errno(error)); 609 610 error = socket_listen(so, backlog, CRED()); 611 612 releasef(sock); 613 if (error) 614 return (set_errno(error)); 615 return (0); 616 } 617 618 /*ARGSUSED3*/ 619 int 620 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version, 621 int flags) 622 { 623 struct sonode *so; 624 file_t *fp; 625 int error; 626 socklen_t namelen; 627 struct sonode *nso; 628 struct vnode *nvp; 629 struct file *nfp; 630 int nfd; 631 int ssflags; 632 struct sockaddr *addrp; 633 socklen_t addrlen; 634 635 dprint(1, ("accept(%d, %p, %p)\n", 636 sock, (void *)name, (void *)namelenp)); 637 638 if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) { 639 return (set_errno(EINVAL)); 640 } 641 642 /* Translate SOCK_ flags to their SS_ variant */ 643 ssflags = 0; 644 if (flags & SOCK_NONBLOCK) 645 ssflags |= SS_NONBLOCK; 646 if (flags & SOCK_NDELAY) 647 ssflags |= SS_NDELAY; 648 649 if ((so = getsonode(sock, &error, &fp)) == NULL) 650 return (set_errno(error)); 651 652 if (name != NULL) { 653 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 654 if (copyin(namelenp, &namelen, sizeof (namelen))) { 655 releasef(sock); 656 return (set_errno(EFAULT)); 657 } 658 if (namelen != 0) { 659 error = useracc(name, (size_t)namelen, B_WRITE); 660 if (error && do_useracc) { 661 releasef(sock); 662 return (set_errno(EFAULT)); 663 } 664 } else 665 name = NULL; 666 } else { 667 namelen = 0; 668 } 669 670 /* 671 * Allocate the user fd before socket_accept() in order to 672 * catch EMFILE errors before calling socket_accept(). 673 */ 674 if ((nfd = ufalloc(0)) == -1) { 675 eprintsoline(so, EMFILE); 676 releasef(sock); 677 return (set_errno(EMFILE)); 678 } 679 error = socket_accept(so, fp->f_flag, CRED(), &nso); 680 if (error) { 681 setf(nfd, NULL); 682 releasef(sock); 683 return (set_errno(error)); 684 } 685 686 nvp = SOTOV(nso); 687 688 ASSERT(MUTEX_NOT_HELD(&nso->so_lock)); 689 if (namelen != 0) { 690 addrlen = so->so_max_addr_len; 691 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP); 692 693 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp, 694 &addrlen, B_TRUE, CRED())) == 0) { 695 error = copyout_name(name, namelen, namelenp, 696 addrp, addrlen); 697 } else { 698 ASSERT(error == EINVAL || error == ENOTCONN); 699 error = ECONNABORTED; 700 } 701 kmem_free(addrp, so->so_max_addr_len); 702 } 703 704 if (error) { 705 setf(nfd, NULL); 706 (void) socket_close(nso, 0, CRED()); 707 socket_destroy(nso); 708 releasef(sock); 709 return (set_errno(error)); 710 } 711 if (error = falloc(NULL, FWRITE|FREAD, &nfp, NULL)) { 712 setf(nfd, NULL); 713 (void) socket_close(nso, 0, CRED()); 714 socket_destroy(nso); 715 eprintsoline(so, error); 716 releasef(sock); 717 return (set_errno(error)); 718 } 719 /* 720 * fill in the entries that falloc reserved 721 */ 722 nfp->f_vnode = nvp; 723 mutex_exit(&nfp->f_tlock); 724 setf(nfd, nfp); 725 726 /* 727 * Act on SOCK_CLOEXEC from flags 728 */ 729 if (flags & SOCK_CLOEXEC) { 730 f_setfd(nfd, FD_CLOEXEC); 731 } 732 733 /* 734 * Copy FNDELAY and FNONBLOCK from listener to acceptor 735 * and from ssflags 736 */ 737 if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) { 738 uint_t oflag = nfp->f_flag; 739 int arg = 0; 740 741 if ((ssflags | so->so_state) & SS_NONBLOCK) 742 arg |= FNONBLOCK; 743 else if ((ssflags | so->so_state) & SS_NDELAY) 744 arg |= FNDELAY; 745 746 /* 747 * This code is a simplification of the F_SETFL code in fcntl() 748 * Ignore any errors from VOP_SETFL. 749 */ 750 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL)) 751 != 0) { 752 eprintsoline(so, error); 753 error = 0; 754 } else { 755 mutex_enter(&nfp->f_tlock); 756 nfp->f_flag &= ~FMASK | (FREAD|FWRITE); 757 nfp->f_flag |= arg; 758 mutex_exit(&nfp->f_tlock); 759 } 760 } 761 releasef(sock); 762 return (nfd); 763 } 764 765 int 766 connect(int sock, struct sockaddr *name, socklen_t namelen, int version) 767 { 768 struct sonode *so; 769 file_t *fp; 770 int error; 771 772 dprint(1, ("connect(%d, %p, %d)\n", 773 sock, (void *)name, namelen)); 774 775 if ((so = getsonode(sock, &error, &fp)) == NULL) 776 return (set_errno(error)); 777 778 /* Allocate and copyin name */ 779 if (namelen != 0) { 780 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 781 name = copyin_name(so, name, &namelen, &error); 782 if (name == NULL) { 783 releasef(sock); 784 return (set_errno(error)); 785 } 786 } else 787 name = NULL; 788 789 error = socket_connect(so, name, namelen, fp->f_flag, 790 (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED()); 791 releasef(sock); 792 if (name) 793 kmem_free(name, (size_t)namelen); 794 if (error) 795 return (set_errno(error)); 796 return (0); 797 } 798 799 /*ARGSUSED2*/ 800 int 801 shutdown(int sock, int how, int version) 802 { 803 struct sonode *so; 804 int error; 805 806 dprint(1, ("shutdown(%d, %d)\n", 807 sock, how)); 808 809 if ((so = getsonode(sock, &error, NULL)) == NULL) 810 return (set_errno(error)); 811 812 error = socket_shutdown(so, how, CRED()); 813 814 releasef(sock); 815 if (error) 816 return (set_errno(error)); 817 return (0); 818 } 819 820 /* 821 * Common receive routine. 822 */ 823 static ssize_t 824 recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags, 825 socklen_t *namelenp, socklen_t *controllenp, int *flagsp) 826 { 827 struct sonode *so; 828 file_t *fp; 829 void *name; 830 socklen_t namelen; 831 void *control; 832 socklen_t controllen, free_controllen; 833 ssize_t len; 834 int error; 835 836 if ((so = getsonode(sock, &error, &fp)) == NULL) 837 return (set_errno(error)); 838 839 len = uiop->uio_resid; 840 uiop->uio_fmode = fp->f_flag; 841 uiop->uio_extflg = UIO_COPY_CACHED; 842 843 name = msg->msg_name; 844 namelen = msg->msg_namelen; 845 control = msg->msg_control; 846 controllen = msg->msg_controllen; 847 848 msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL | 849 MSG_DONTWAIT | MSG_XPG4_2); 850 851 error = socket_recvmsg(so, msg, uiop, CRED()); 852 if (error) { 853 releasef(sock); 854 return (set_errno(error)); 855 } 856 lwp_stat_update(LWP_STAT_MSGRCV, 1); 857 releasef(sock); 858 859 free_controllen = msg->msg_controllen; 860 861 error = copyout_name(name, namelen, namelenp, 862 msg->msg_name, msg->msg_namelen); 863 if (error) 864 goto err; 865 866 if (flagsp != NULL) { 867 /* 868 * Clear internal flag. 869 */ 870 msg->msg_flags &= ~MSG_XPG4_2; 871 872 /* 873 * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only 874 * when controllen is zero and there is control data to 875 * copy out. 876 */ 877 if (controllen != 0 && 878 (msg->msg_controllen > controllen || control == NULL)) { 879 dprint(1, ("recvit: CTRUNC %d %d %p\n", 880 msg->msg_controllen, controllen, control)); 881 882 msg->msg_flags |= MSG_CTRUNC; 883 } 884 if (copyout(&msg->msg_flags, flagsp, 885 sizeof (msg->msg_flags))) { 886 error = EFAULT; 887 goto err; 888 } 889 } 890 891 if (controllen != 0) { 892 if (!(flags & MSG_XPG4_2)) { 893 /* 894 * Good old msg_accrights can only return a multiple 895 * of 4 bytes. 896 */ 897 controllen &= ~((int)sizeof (uint32_t) - 1); 898 } 899 900 if (msg->msg_controllen > controllen || control == NULL) { 901 /* 902 * If the truncated part contains file descriptors, 903 * then they must be closed in the kernel as they 904 * will not be included in the data returned to 905 * user space. Close them now so that the header size 906 * can be safely adjusted prior to copyout. In case of 907 * an error during copyout, the remaining file 908 * descriptors will be closed in the error handler 909 * below. 910 */ 911 so_closefds(msg->msg_control, msg->msg_controllen, 912 !(flags & MSG_XPG4_2), 913 control == NULL ? 0 : controllen); 914 915 /* 916 * In the case of a truncated control message, the last 917 * cmsg header that fits into the available buffer 918 * space must be adjusted to reflect the actual amount 919 * of associated data that will be returned. This only 920 * needs to be done for XPG4 messages as non-XPG4 921 * messages are not structured (they are just a 922 * buffer and a length - msg_accrights(len)). 923 */ 924 if (control != NULL && (flags & MSG_XPG4_2)) { 925 so_truncatecmsg(msg->msg_control, 926 msg->msg_controllen, controllen); 927 msg->msg_controllen = controllen; 928 } 929 } 930 931 error = copyout_arg(control, controllen, controllenp, 932 msg->msg_control, msg->msg_controllen); 933 934 if (error) 935 goto err; 936 937 } 938 if (msg->msg_namelen != 0) 939 kmem_free(msg->msg_name, (size_t)msg->msg_namelen); 940 if (free_controllen != 0) 941 kmem_free(msg->msg_control, (size_t)free_controllen); 942 return (len - uiop->uio_resid); 943 944 err: 945 /* 946 * If we fail and the control part contains file descriptors 947 * we have to close them. For a truncated control message, the 948 * descriptors which were cut off have already been closed and the 949 * length adjusted so that they will not be closed again. 950 */ 951 if (msg->msg_controllen != 0) 952 so_closefds(msg->msg_control, msg->msg_controllen, 953 !(flags & MSG_XPG4_2), 0); 954 if (msg->msg_namelen != 0) 955 kmem_free(msg->msg_name, (size_t)msg->msg_namelen); 956 if (free_controllen != 0) 957 kmem_free(msg->msg_control, (size_t)free_controllen); 958 return (set_errno(error)); 959 } 960 961 /* 962 * Native system call 963 */ 964 ssize_t 965 recv(int sock, void *buffer, size_t len, int flags) 966 { 967 struct nmsghdr lmsg; 968 struct uio auio; 969 struct iovec aiov[1]; 970 971 dprint(1, ("recv(%d, %p, %ld, %d)\n", 972 sock, buffer, len, flags)); 973 974 if ((ssize_t)len < 0) { 975 return (set_errno(EINVAL)); 976 } 977 978 aiov[0].iov_base = buffer; 979 aiov[0].iov_len = len; 980 auio.uio_loffset = 0; 981 auio.uio_iov = aiov; 982 auio.uio_iovcnt = 1; 983 auio.uio_resid = len; 984 auio.uio_segflg = UIO_USERSPACE; 985 auio.uio_limit = 0; 986 987 lmsg.msg_namelen = 0; 988 lmsg.msg_controllen = 0; 989 lmsg.msg_flags = 0; 990 return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL)); 991 } 992 993 ssize_t 994 recvfrom(int sock, void *buffer, size_t len, int flags, struct sockaddr *name, 995 socklen_t *namelenp) 996 { 997 struct nmsghdr lmsg; 998 struct uio auio; 999 struct iovec aiov[1]; 1000 1001 dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n", 1002 sock, buffer, len, flags, (void *)name, (void *)namelenp)); 1003 1004 if ((ssize_t)len < 0) { 1005 return (set_errno(EINVAL)); 1006 } 1007 1008 aiov[0].iov_base = buffer; 1009 aiov[0].iov_len = len; 1010 auio.uio_loffset = 0; 1011 auio.uio_iov = aiov; 1012 auio.uio_iovcnt = 1; 1013 auio.uio_resid = len; 1014 auio.uio_segflg = UIO_USERSPACE; 1015 auio.uio_limit = 0; 1016 1017 lmsg.msg_name = (char *)name; 1018 if (namelenp != NULL) { 1019 if (copyin(namelenp, &lmsg.msg_namelen, 1020 sizeof (lmsg.msg_namelen))) 1021 return (set_errno(EFAULT)); 1022 } else { 1023 lmsg.msg_namelen = 0; 1024 } 1025 lmsg.msg_controllen = 0; 1026 lmsg.msg_flags = 0; 1027 1028 return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL)); 1029 } 1030 1031 /* 1032 * Uses the MSG_XPG4_2 flag to determine if the caller is using 1033 * struct omsghdr or struct nmsghdr. 1034 */ 1035 ssize_t 1036 recvmsg(int sock, struct nmsghdr *msg, int flags) 1037 { 1038 STRUCT_DECL(nmsghdr, u_lmsg); 1039 STRUCT_HANDLE(nmsghdr, umsgptr); 1040 struct nmsghdr lmsg; 1041 struct uio auio; 1042 struct iovec buf[IOV_MAX_STACK], *aiov = buf; 1043 ssize_t iovsize = 0; 1044 int iovcnt; 1045 ssize_t len, rval; 1046 int i; 1047 int *flagsp; 1048 model_t model; 1049 1050 dprint(1, ("recvmsg(%d, %p, %d)\n", 1051 sock, (void *)msg, flags)); 1052 1053 model = get_udatamodel(); 1054 STRUCT_INIT(u_lmsg, model); 1055 STRUCT_SET_HANDLE(umsgptr, model, msg); 1056 1057 if (flags & MSG_XPG4_2) { 1058 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg))) 1059 return (set_errno(EFAULT)); 1060 flagsp = STRUCT_FADDR(umsgptr, msg_flags); 1061 } else { 1062 /* 1063 * Assumes that nmsghdr and omsghdr are identically shaped 1064 * except for the added msg_flags field. 1065 */ 1066 if (copyin(msg, STRUCT_BUF(u_lmsg), 1067 SIZEOF_STRUCT(omsghdr, model))) 1068 return (set_errno(EFAULT)); 1069 STRUCT_FSET(u_lmsg, msg_flags, 0); 1070 flagsp = NULL; 1071 } 1072 1073 /* 1074 * Code below us will kmem_alloc memory and hang it 1075 * off msg_control and msg_name fields. This forces 1076 * us to copy the structure to its native form. 1077 */ 1078 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name); 1079 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen); 1080 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov); 1081 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen); 1082 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control); 1083 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen); 1084 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags); 1085 1086 iovcnt = lmsg.msg_iovlen; 1087 1088 if (iovcnt <= 0 || iovcnt > IOV_MAX) { 1089 return (set_errno(EMSGSIZE)); 1090 } 1091 1092 if (iovcnt > IOV_MAX_STACK) { 1093 iovsize = iovcnt * sizeof (struct iovec); 1094 aiov = kmem_alloc(iovsize, KM_SLEEP); 1095 } 1096 1097 #ifdef _SYSCALL32_IMPL 1098 /* 1099 * 32-bit callers need to have their iovec expanded, while ensuring 1100 * that they can't move more than 2Gbytes of data in a single call. 1101 */ 1102 if (model == DATAMODEL_ILP32) { 1103 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32; 1104 ssize_t iov32size; 1105 ssize32_t count32; 1106 1107 iov32size = iovcnt * sizeof (struct iovec32); 1108 if (iovsize != 0) 1109 aiov32 = kmem_alloc(iov32size, KM_SLEEP); 1110 1111 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) { 1112 if (iovsize != 0) { 1113 kmem_free(aiov32, iov32size); 1114 kmem_free(aiov, iovsize); 1115 } 1116 1117 return (set_errno(EFAULT)); 1118 } 1119 1120 count32 = 0; 1121 for (i = 0; i < iovcnt; i++) { 1122 ssize32_t iovlen32; 1123 1124 iovlen32 = aiov32[i].iov_len; 1125 count32 += iovlen32; 1126 if (iovlen32 < 0 || count32 < 0) { 1127 if (iovsize != 0) { 1128 kmem_free(aiov32, iov32size); 1129 kmem_free(aiov, iovsize); 1130 } 1131 1132 return (set_errno(EINVAL)); 1133 } 1134 1135 aiov[i].iov_len = iovlen32; 1136 aiov[i].iov_base = 1137 (caddr_t)(uintptr_t)aiov32[i].iov_base; 1138 } 1139 1140 if (iovsize != 0) 1141 kmem_free(aiov32, iov32size); 1142 } else 1143 #endif /* _SYSCALL32_IMPL */ 1144 if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) { 1145 if (iovsize != 0) 1146 kmem_free(aiov, iovsize); 1147 1148 return (set_errno(EFAULT)); 1149 } 1150 len = 0; 1151 for (i = 0; i < iovcnt; i++) { 1152 ssize_t iovlen = aiov[i].iov_len; 1153 len += iovlen; 1154 if (iovlen < 0 || len < 0) { 1155 if (iovsize != 0) 1156 kmem_free(aiov, iovsize); 1157 1158 return (set_errno(EINVAL)); 1159 } 1160 } 1161 auio.uio_loffset = 0; 1162 auio.uio_iov = aiov; 1163 auio.uio_iovcnt = iovcnt; 1164 auio.uio_resid = len; 1165 auio.uio_segflg = UIO_USERSPACE; 1166 auio.uio_limit = 0; 1167 1168 if (lmsg.msg_control != NULL && 1169 (do_useracc == 0 || 1170 useracc(lmsg.msg_control, lmsg.msg_controllen, 1171 B_WRITE) != 0)) { 1172 if (iovsize != 0) 1173 kmem_free(aiov, iovsize); 1174 1175 return (set_errno(EFAULT)); 1176 } 1177 1178 rval = recvit(sock, &lmsg, &auio, flags, 1179 STRUCT_FADDR(umsgptr, msg_namelen), 1180 STRUCT_FADDR(umsgptr, msg_controllen), flagsp); 1181 1182 if (iovsize != 0) 1183 kmem_free(aiov, iovsize); 1184 1185 return (rval); 1186 } 1187 1188 /* 1189 * Common send function. 1190 */ 1191 static ssize_t 1192 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags) 1193 { 1194 struct sonode *so; 1195 file_t *fp; 1196 void *name; 1197 socklen_t namelen; 1198 void *control; 1199 socklen_t controllen; 1200 ssize_t len; 1201 int error; 1202 1203 if ((so = getsonode(sock, &error, &fp)) == NULL) 1204 return (set_errno(error)); 1205 1206 uiop->uio_fmode = fp->f_flag; 1207 1208 if (so->so_family == AF_UNIX) 1209 uiop->uio_extflg = UIO_COPY_CACHED; 1210 else 1211 uiop->uio_extflg = UIO_COPY_DEFAULT; 1212 1213 /* Allocate and copyin name and control */ 1214 name = msg->msg_name; 1215 namelen = msg->msg_namelen; 1216 if (name != NULL && namelen != 0) { 1217 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1218 name = copyin_name(so, 1219 (struct sockaddr *)name, 1220 &namelen, &error); 1221 if (name == NULL) 1222 goto done3; 1223 /* copyin_name null terminates addresses for AF_UNIX */ 1224 msg->msg_namelen = namelen; 1225 msg->msg_name = name; 1226 } else { 1227 msg->msg_name = name = NULL; 1228 msg->msg_namelen = namelen = 0; 1229 } 1230 1231 control = msg->msg_control; 1232 controllen = msg->msg_controllen; 1233 if ((control != NULL) && (controllen != 0)) { 1234 /* 1235 * Verify that the length is not excessive to prevent 1236 * an application from consuming all of kernel memory. 1237 */ 1238 if (controllen > SO_MAXARGSIZE) { 1239 error = EINVAL; 1240 goto done2; 1241 } 1242 control = kmem_alloc(controllen, KM_SLEEP); 1243 1244 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1245 if (copyin(msg->msg_control, control, controllen)) { 1246 error = EFAULT; 1247 goto done1; 1248 } 1249 msg->msg_control = control; 1250 } else { 1251 msg->msg_control = control = NULL; 1252 msg->msg_controllen = controllen = 0; 1253 } 1254 1255 len = uiop->uio_resid; 1256 msg->msg_flags = flags; 1257 1258 error = socket_sendmsg(so, msg, uiop, CRED()); 1259 done1: 1260 if (control != NULL) 1261 kmem_free(control, controllen); 1262 done2: 1263 if (name != NULL) 1264 kmem_free(name, namelen); 1265 done3: 1266 if (error != 0) { 1267 releasef(sock); 1268 return (set_errno(error)); 1269 } 1270 lwp_stat_update(LWP_STAT_MSGSND, 1); 1271 releasef(sock); 1272 return (len - uiop->uio_resid); 1273 } 1274 1275 /* 1276 * Native system call 1277 */ 1278 ssize_t 1279 send(int sock, void *buffer, size_t len, int flags) 1280 { 1281 struct nmsghdr lmsg; 1282 struct uio auio; 1283 struct iovec aiov[1]; 1284 1285 dprint(1, ("send(%d, %p, %ld, %d)\n", 1286 sock, buffer, len, flags)); 1287 1288 if ((ssize_t)len < 0) { 1289 return (set_errno(EINVAL)); 1290 } 1291 1292 aiov[0].iov_base = buffer; 1293 aiov[0].iov_len = len; 1294 auio.uio_loffset = 0; 1295 auio.uio_iov = aiov; 1296 auio.uio_iovcnt = 1; 1297 auio.uio_resid = len; 1298 auio.uio_segflg = UIO_USERSPACE; 1299 auio.uio_limit = 0; 1300 1301 lmsg.msg_name = NULL; 1302 lmsg.msg_control = NULL; 1303 if (!(flags & MSG_XPG4_2)) { 1304 /* 1305 * In order to be compatible with the libsocket/sockmod 1306 * implementation we set EOR for all send* calls. 1307 */ 1308 flags |= MSG_EOR; 1309 } 1310 return (sendit(sock, &lmsg, &auio, flags)); 1311 } 1312 1313 /* 1314 * Uses the MSG_XPG4_2 flag to determine if the caller is using 1315 * struct omsghdr or struct nmsghdr. 1316 */ 1317 ssize_t 1318 sendmsg(int sock, struct nmsghdr *msg, int flags) 1319 { 1320 struct nmsghdr lmsg; 1321 STRUCT_DECL(nmsghdr, u_lmsg); 1322 struct uio auio; 1323 struct iovec buf[IOV_MAX_STACK], *aiov = buf; 1324 ssize_t iovsize = 0; 1325 int iovcnt; 1326 ssize_t len, rval; 1327 int i; 1328 model_t model; 1329 1330 dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags)); 1331 1332 model = get_udatamodel(); 1333 STRUCT_INIT(u_lmsg, model); 1334 1335 if (flags & MSG_XPG4_2) { 1336 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg), 1337 STRUCT_SIZE(u_lmsg))) 1338 return (set_errno(EFAULT)); 1339 } else { 1340 /* 1341 * Assumes that nmsghdr and omsghdr are identically shaped 1342 * except for the added msg_flags field. 1343 */ 1344 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg), 1345 SIZEOF_STRUCT(omsghdr, model))) 1346 return (set_errno(EFAULT)); 1347 /* 1348 * In order to be compatible with the libsocket/sockmod 1349 * implementation we set EOR for all send* calls. 1350 */ 1351 flags |= MSG_EOR; 1352 } 1353 1354 /* 1355 * Code below us will kmem_alloc memory and hang it 1356 * off msg_control and msg_name fields. This forces 1357 * us to copy the structure to its native form. 1358 */ 1359 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name); 1360 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen); 1361 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov); 1362 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen); 1363 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control); 1364 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen); 1365 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags); 1366 1367 iovcnt = lmsg.msg_iovlen; 1368 1369 if (iovcnt <= 0 || iovcnt > IOV_MAX) { 1370 /* 1371 * Unless this is XPG 4.2 we allow iovcnt == 0 to 1372 * be compatible with SunOS 4.X and 4.4BSD. 1373 */ 1374 if (iovcnt != 0 || (flags & MSG_XPG4_2)) 1375 return (set_errno(EMSGSIZE)); 1376 } 1377 1378 if (iovcnt > IOV_MAX_STACK) { 1379 iovsize = iovcnt * sizeof (struct iovec); 1380 aiov = kmem_alloc(iovsize, KM_SLEEP); 1381 } 1382 1383 #ifdef _SYSCALL32_IMPL 1384 /* 1385 * 32-bit callers need to have their iovec expanded, while ensuring 1386 * that they can't move more than 2Gbytes of data in a single call. 1387 */ 1388 if (model == DATAMODEL_ILP32) { 1389 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32; 1390 ssize_t iov32size; 1391 ssize32_t count32; 1392 1393 iov32size = iovcnt * sizeof (struct iovec32); 1394 if (iovsize != 0) 1395 aiov32 = kmem_alloc(iov32size, KM_SLEEP); 1396 1397 if (iovcnt != 0 && 1398 copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) { 1399 if (iovsize != 0) { 1400 kmem_free(aiov32, iov32size); 1401 kmem_free(aiov, iovsize); 1402 } 1403 1404 return (set_errno(EFAULT)); 1405 } 1406 1407 count32 = 0; 1408 for (i = 0; i < iovcnt; i++) { 1409 ssize32_t iovlen32; 1410 1411 iovlen32 = aiov32[i].iov_len; 1412 count32 += iovlen32; 1413 if (iovlen32 < 0 || count32 < 0) { 1414 if (iovsize != 0) { 1415 kmem_free(aiov32, iov32size); 1416 kmem_free(aiov, iovsize); 1417 } 1418 1419 return (set_errno(EINVAL)); 1420 } 1421 1422 aiov[i].iov_len = iovlen32; 1423 aiov[i].iov_base = 1424 (caddr_t)(uintptr_t)aiov32[i].iov_base; 1425 } 1426 1427 if (iovsize != 0) 1428 kmem_free(aiov32, iov32size); 1429 } else 1430 #endif /* _SYSCALL32_IMPL */ 1431 if (iovcnt != 0 && 1432 copyin(lmsg.msg_iov, aiov, 1433 (unsigned)iovcnt * sizeof (struct iovec))) { 1434 if (iovsize != 0) 1435 kmem_free(aiov, iovsize); 1436 1437 return (set_errno(EFAULT)); 1438 } 1439 len = 0; 1440 for (i = 0; i < iovcnt; i++) { 1441 ssize_t iovlen = aiov[i].iov_len; 1442 len += iovlen; 1443 if (iovlen < 0 || len < 0) { 1444 if (iovsize != 0) 1445 kmem_free(aiov, iovsize); 1446 1447 return (set_errno(EINVAL)); 1448 } 1449 } 1450 auio.uio_loffset = 0; 1451 auio.uio_iov = aiov; 1452 auio.uio_iovcnt = iovcnt; 1453 auio.uio_resid = len; 1454 auio.uio_segflg = UIO_USERSPACE; 1455 auio.uio_limit = 0; 1456 1457 rval = sendit(sock, &lmsg, &auio, flags); 1458 1459 if (iovsize != 0) 1460 kmem_free(aiov, iovsize); 1461 1462 return (rval); 1463 } 1464 1465 ssize_t 1466 sendto(int sock, void *buffer, size_t len, int flags, 1467 struct sockaddr *name, socklen_t namelen) 1468 { 1469 struct nmsghdr lmsg; 1470 struct uio auio; 1471 struct iovec aiov[1]; 1472 1473 dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n", 1474 sock, buffer, len, flags, (void *)name, namelen)); 1475 1476 if ((ssize_t)len < 0) { 1477 return (set_errno(EINVAL)); 1478 } 1479 1480 aiov[0].iov_base = buffer; 1481 aiov[0].iov_len = len; 1482 auio.uio_loffset = 0; 1483 auio.uio_iov = aiov; 1484 auio.uio_iovcnt = 1; 1485 auio.uio_resid = len; 1486 auio.uio_segflg = UIO_USERSPACE; 1487 auio.uio_limit = 0; 1488 1489 lmsg.msg_name = (char *)name; 1490 lmsg.msg_namelen = namelen; 1491 lmsg.msg_control = NULL; 1492 if (!(flags & MSG_XPG4_2)) { 1493 /* 1494 * In order to be compatible with the libsocket/sockmod 1495 * implementation we set EOR for all send* calls. 1496 */ 1497 flags |= MSG_EOR; 1498 } 1499 return (sendit(sock, &lmsg, &auio, flags)); 1500 } 1501 1502 /*ARGSUSED3*/ 1503 int 1504 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version) 1505 { 1506 struct sonode *so; 1507 int error; 1508 socklen_t namelen; 1509 socklen_t sock_addrlen; 1510 struct sockaddr *sock_addrp; 1511 1512 dprint(1, ("getpeername(%d, %p, %p)\n", 1513 sock, (void *)name, (void *)namelenp)); 1514 1515 if ((so = getsonode(sock, &error, NULL)) == NULL) 1516 goto bad; 1517 1518 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1519 if (copyin(namelenp, &namelen, sizeof (namelen)) || 1520 (name == NULL && namelen != 0)) { 1521 error = EFAULT; 1522 goto rel_out; 1523 } 1524 sock_addrlen = so->so_max_addr_len; 1525 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP); 1526 1527 if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen, 1528 B_FALSE, CRED())) == 0) { 1529 ASSERT(sock_addrlen <= so->so_max_addr_len); 1530 error = copyout_name(name, namelen, namelenp, 1531 (void *)sock_addrp, sock_addrlen); 1532 } 1533 kmem_free(sock_addrp, so->so_max_addr_len); 1534 rel_out: 1535 releasef(sock); 1536 bad: return (error != 0 ? set_errno(error) : 0); 1537 } 1538 1539 /*ARGSUSED3*/ 1540 int 1541 getsockname(int sock, struct sockaddr *name, socklen_t *namelenp, int version) 1542 { 1543 struct sonode *so; 1544 int error; 1545 socklen_t namelen, sock_addrlen; 1546 struct sockaddr *sock_addrp; 1547 1548 dprint(1, ("getsockname(%d, %p, %p)\n", 1549 sock, (void *)name, (void *)namelenp)); 1550 1551 if ((so = getsonode(sock, &error, NULL)) == NULL) 1552 goto bad; 1553 1554 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1555 if (copyin(namelenp, &namelen, sizeof (namelen)) || 1556 (name == NULL && namelen != 0)) { 1557 error = EFAULT; 1558 goto rel_out; 1559 } 1560 1561 sock_addrlen = so->so_max_addr_len; 1562 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP); 1563 if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen, 1564 CRED())) == 0) { 1565 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1566 ASSERT(sock_addrlen <= so->so_max_addr_len); 1567 error = copyout_name(name, namelen, namelenp, 1568 (void *)sock_addrp, sock_addrlen); 1569 } 1570 kmem_free(sock_addrp, so->so_max_addr_len); 1571 rel_out: 1572 releasef(sock); 1573 bad: return (error != 0 ? set_errno(error) : 0); 1574 } 1575 1576 /*ARGSUSED5*/ 1577 int 1578 getsockopt(int sock, int level, int option_name, void *option_value, 1579 socklen_t *option_lenp, int version) 1580 { 1581 struct sonode *so; 1582 socklen_t optlen, optlen_res; 1583 void *optval; 1584 int error; 1585 1586 dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n", 1587 sock, level, option_name, option_value, (void *)option_lenp)); 1588 1589 if ((so = getsonode(sock, &error, NULL)) == NULL) 1590 return (set_errno(error)); 1591 1592 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1593 if (copyin(option_lenp, &optlen, sizeof (optlen))) { 1594 releasef(sock); 1595 return (set_errno(EFAULT)); 1596 } 1597 /* 1598 * Verify that the length is not excessive to prevent 1599 * an application from consuming all of kernel memory. 1600 */ 1601 if (optlen > SO_MAXARGSIZE) { 1602 error = EINVAL; 1603 releasef(sock); 1604 return (set_errno(error)); 1605 } 1606 optval = kmem_alloc(optlen, KM_SLEEP); 1607 optlen_res = optlen; 1608 error = socket_getsockopt(so, level, option_name, optval, 1609 &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2, 1610 CRED()); 1611 releasef(sock); 1612 if (error) { 1613 kmem_free(optval, optlen); 1614 return (set_errno(error)); 1615 } 1616 error = copyout_arg(option_value, optlen, option_lenp, 1617 optval, optlen_res); 1618 kmem_free(optval, optlen); 1619 if (error) 1620 return (set_errno(error)); 1621 return (0); 1622 } 1623 1624 /*ARGSUSED5*/ 1625 int 1626 setsockopt(int sock, int level, int option_name, void *option_value, 1627 socklen_t option_len, int version) 1628 { 1629 struct sonode *so; 1630 intptr_t buffer[2]; 1631 void *optval = NULL; 1632 int error; 1633 1634 dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n", 1635 sock, level, option_name, option_value, option_len)); 1636 1637 if ((so = getsonode(sock, &error, NULL)) == NULL) 1638 return (set_errno(error)); 1639 1640 if (option_value != NULL) { 1641 if (option_len != 0) { 1642 /* 1643 * Verify that the length is not excessive to prevent 1644 * an application from consuming all of kernel memory. 1645 */ 1646 if (option_len > SO_MAXARGSIZE) { 1647 error = EINVAL; 1648 goto done2; 1649 } 1650 optval = option_len <= sizeof (buffer) ? 1651 &buffer : kmem_alloc((size_t)option_len, KM_SLEEP); 1652 ASSERT(MUTEX_NOT_HELD(&so->so_lock)); 1653 if (copyin(option_value, optval, (size_t)option_len)) { 1654 error = EFAULT; 1655 goto done1; 1656 } 1657 } 1658 } else 1659 option_len = 0; 1660 1661 error = socket_setsockopt(so, level, option_name, optval, 1662 (t_uscalar_t)option_len, CRED()); 1663 done1: 1664 if (optval != buffer) 1665 kmem_free(optval, (size_t)option_len); 1666 done2: 1667 releasef(sock); 1668 if (error) 1669 return (set_errno(error)); 1670 return (0); 1671 } 1672 1673 static int 1674 sockconf_add_sock(int family, int type, int protocol, char *name) 1675 { 1676 int error = 0; 1677 char *kdevpath = NULL; 1678 char *kmodule = NULL; 1679 char *buf = NULL; 1680 size_t pathlen = 0; 1681 struct sockparams *sp; 1682 1683 if (name == NULL) 1684 return (EINVAL); 1685 /* 1686 * Copyin the name. 1687 * This also makes it possible to check for too long pathnames. 1688 * Compress the space needed for the name before passing it 1689 * to soconfig - soconfig will store the string until 1690 * the configuration is removed. 1691 */ 1692 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP); 1693 if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) { 1694 kmem_free(buf, MAXPATHLEN); 1695 return (error); 1696 } 1697 if (strncmp(buf, "/dev", strlen("/dev")) == 0) { 1698 /* For device */ 1699 kdevpath = kmem_alloc(pathlen, KM_SLEEP); 1700 bcopy(buf, kdevpath, pathlen); 1701 kdevpath[pathlen - 1] = '\0'; 1702 } else { 1703 /* For socket module */ 1704 kmodule = kmem_alloc(pathlen, KM_SLEEP); 1705 bcopy(buf, kmodule, pathlen); 1706 kmodule[pathlen - 1] = '\0'; 1707 pathlen = 0; 1708 } 1709 kmem_free(buf, MAXPATHLEN); 1710 1711 /* sockparams_create frees mod name and devpath upon failure */ 1712 sp = sockparams_create(family, type, protocol, kmodule, 1713 kdevpath, pathlen, 0, KM_SLEEP, &error); 1714 if (sp != NULL) { 1715 error = sockparams_add(sp); 1716 if (error != 0) 1717 sockparams_destroy(sp); 1718 } 1719 1720 return (error); 1721 } 1722 1723 static int 1724 sockconf_remove_sock(int family, int type, int protocol) 1725 { 1726 return (sockparams_delete(family, type, protocol)); 1727 } 1728 1729 static int 1730 sockconfig_remove_filter(const char *uname) 1731 { 1732 char kname[SOF_MAXNAMELEN]; 1733 size_t len; 1734 int error; 1735 sof_entry_t *ent; 1736 1737 if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0) 1738 return (error); 1739 1740 ent = sof_entry_remove_by_name(kname); 1741 if (ent == NULL) 1742 return (ENXIO); 1743 1744 mutex_enter(&ent->sofe_lock); 1745 ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED)); 1746 if (ent->sofe_refcnt == 0) { 1747 mutex_exit(&ent->sofe_lock); 1748 sof_entry_free(ent); 1749 } else { 1750 /* let the last socket free the filter */ 1751 ent->sofe_flags |= SOFEF_CONDEMED; 1752 mutex_exit(&ent->sofe_lock); 1753 } 1754 1755 return (0); 1756 } 1757 1758 static int 1759 sockconfig_add_filter(const char *uname, void *ufilpropp) 1760 { 1761 struct sockconfig_filter_props filprop; 1762 sof_entry_t *ent; 1763 int error; 1764 size_t tuplesz, len; 1765 char hintbuf[SOF_MAXNAMELEN]; 1766 1767 ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP); 1768 mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL); 1769 1770 if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN, 1771 &len)) != 0) { 1772 sof_entry_free(ent); 1773 return (error); 1774 } 1775 1776 if (get_udatamodel() == DATAMODEL_NATIVE) { 1777 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) { 1778 sof_entry_free(ent); 1779 return (EFAULT); 1780 } 1781 } 1782 #ifdef _SYSCALL32_IMPL 1783 else { 1784 struct sockconfig_filter_props32 filprop32; 1785 1786 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) { 1787 sof_entry_free(ent); 1788 return (EFAULT); 1789 } 1790 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname; 1791 filprop.sfp_autoattach = filprop32.sfp_autoattach; 1792 filprop.sfp_hint = filprop32.sfp_hint; 1793 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg; 1794 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt; 1795 filprop.sfp_socktuple = 1796 (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple; 1797 } 1798 #endif /* _SYSCALL32_IMPL */ 1799 1800 if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname, 1801 sizeof (ent->sofe_modname), &len)) != 0) { 1802 sof_entry_free(ent); 1803 return (error); 1804 } 1805 1806 /* 1807 * A filter must specify at least one socket tuple. 1808 */ 1809 if (filprop.sfp_socktuple_cnt == 0 || 1810 filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) { 1811 sof_entry_free(ent); 1812 return (EINVAL); 1813 } 1814 ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG; 1815 ent->sofe_hint = filprop.sfp_hint; 1816 1817 /* 1818 * Verify the hint, and copy in the hint argument, if necessary. 1819 */ 1820 switch (ent->sofe_hint) { 1821 case SOF_HINT_BEFORE: 1822 case SOF_HINT_AFTER: 1823 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf, 1824 sizeof (hintbuf), &len)) != 0) { 1825 sof_entry_free(ent); 1826 return (error); 1827 } 1828 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP); 1829 bcopy(hintbuf, ent->sofe_hintarg, len); 1830 /* FALLTHRU */ 1831 case SOF_HINT_TOP: 1832 case SOF_HINT_BOTTOM: 1833 /* hints cannot be used with programmatic filters */ 1834 if (ent->sofe_flags & SOFEF_PROG) { 1835 sof_entry_free(ent); 1836 return (EINVAL); 1837 } 1838 break; 1839 case SOF_HINT_NONE: 1840 break; 1841 default: 1842 /* bad hint value */ 1843 sof_entry_free(ent); 1844 return (EINVAL); 1845 } 1846 1847 ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt; 1848 tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt; 1849 ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP); 1850 1851 if (get_udatamodel() == DATAMODEL_NATIVE) { 1852 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple, 1853 tuplesz)) { 1854 sof_entry_free(ent); 1855 return (EFAULT); 1856 } 1857 } 1858 #ifdef _SYSCALL32_IMPL 1859 else { 1860 int i; 1861 caddr_t data = (caddr_t)filprop.sfp_socktuple; 1862 sof_socktuple_t *tup = ent->sofe_socktuple; 1863 sof_socktuple32_t tup32; 1864 1865 tup = ent->sofe_socktuple; 1866 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) { 1867 ASSERT(tup < ent->sofe_socktuple + tuplesz); 1868 1869 if (copyin(data, &tup32, sizeof (tup32)) != 0) { 1870 sof_entry_free(ent); 1871 return (EFAULT); 1872 } 1873 tup->sofst_family = tup32.sofst_family; 1874 tup->sofst_type = tup32.sofst_type; 1875 tup->sofst_protocol = tup32.sofst_protocol; 1876 1877 data += sizeof (tup32); 1878 } 1879 } 1880 #endif /* _SYSCALL32_IMPL */ 1881 1882 /* Sockets can start using the filter as soon as the filter is added */ 1883 if ((error = sof_entry_add(ent)) != 0) 1884 sof_entry_free(ent); 1885 1886 return (error); 1887 } 1888 1889 /* 1890 * Socket configuration system call. It is used to add and remove 1891 * socket types. 1892 */ 1893 int 1894 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4) 1895 { 1896 int error = 0; 1897 1898 if (secpolicy_net_config(CRED(), B_FALSE) != 0) 1899 return (set_errno(EPERM)); 1900 1901 switch (cmd) { 1902 case SOCKCONFIG_ADD_SOCK: 1903 error = sockconf_add_sock((int)(uintptr_t)arg1, 1904 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4); 1905 break; 1906 case SOCKCONFIG_REMOVE_SOCK: 1907 error = sockconf_remove_sock((int)(uintptr_t)arg1, 1908 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3); 1909 break; 1910 case SOCKCONFIG_ADD_FILTER: 1911 error = sockconfig_add_filter((const char *)arg1, arg2); 1912 break; 1913 case SOCKCONFIG_REMOVE_FILTER: 1914 error = sockconfig_remove_filter((const char *)arg1); 1915 break; 1916 case SOCKCONFIG_GET_SOCKTABLE: 1917 error = sockparams_copyout_socktable((int)(uintptr_t)arg1); 1918 break; 1919 default: 1920 #ifdef DEBUG 1921 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd); 1922 #endif 1923 error = EINVAL; 1924 break; 1925 } 1926 1927 if (error != 0) { 1928 eprintline(error); 1929 return (set_errno(error)); 1930 } 1931 return (0); 1932 } 1933 1934 1935 /* 1936 * Sendfile is implemented through two schemes, direct I/O or by 1937 * caching in the filesystem page cache. We cache the input file by 1938 * default and use direct I/O only if sendfile_max_size is set 1939 * appropriately as explained below. Note that this logic is consistent 1940 * with other filesystems where caching is turned on by default 1941 * unless explicitly turned off by using the DIRECTIO ioctl. 1942 * 1943 * We choose a slightly different scheme here. One can turn off 1944 * caching by setting sendfile_max_size to 0. One can also enable 1945 * caching of files <= sendfile_max_size by setting sendfile_max_size 1946 * to an appropriate value. By default sendfile_max_size is set to the 1947 * maximum value so that all files are cached. In future, we may provide 1948 * better interfaces for caching the file. 1949 * 1950 * Sendfile through Direct I/O (Zero copy) 1951 * -------------------------------------- 1952 * 1953 * As disks are normally slower than the network, we can't have a 1954 * single thread that reads the disk and writes to the network. We 1955 * need to have parallelism. This is done by having the sendfile 1956 * thread create another thread that reads from the filesystem 1957 * and queues it for network processing. In this scheme, the data 1958 * is never copied anywhere i.e it is zero copy unlike the other 1959 * scheme. 1960 * 1961 * We have a sendfile queue (snfq) where each sendfile 1962 * request (snf_req_t) is queued for processing by a thread. Number 1963 * of threads is dynamically allocated and they exit if they are idling 1964 * beyond a specified amount of time. When each request (snf_req_t) is 1965 * processed by a thread, it produces a number of mblk_t structures to 1966 * be consumed by the sendfile thread. snf_deque and snf_enque are 1967 * used for consuming and producing mblks. Size of the filesystem 1968 * read is determined by the tunable (sendfile_read_size). A single 1969 * mblk holds sendfile_read_size worth of data (except the last 1970 * read of the file) which is sent down as a whole to the network. 1971 * sendfile_read_size is set to 1 MB as this seems to be the optimal 1972 * value for the UFS filesystem backed by a striped storage array. 1973 * 1974 * Synchronisation between read (producer) and write (consumer) threads. 1975 * -------------------------------------------------------------------- 1976 * 1977 * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while 1978 * adding and deleting items in this list. Error can happen anytime 1979 * during read or write. There could be unprocessed mblks in the 1980 * sr_ib_XXX list when a read or write error occurs. Whenever error 1981 * is encountered, we need two things to happen : 1982 * 1983 * a) One of the threads need to clean the mblks. 1984 * b) When one thread encounters an error, the other should stop. 1985 * 1986 * For (a), we don't want to penalize the reader thread as it could do 1987 * some useful work processing other requests. For (b), the error can 1988 * be detected by examining sr_read_error or sr_write_error. 1989 * sr_lock protects sr_read_error and sr_write_error. If both reader and 1990 * writer encounters error, we need to report the write error back to 1991 * the application as that's what would have happened if the operations 1992 * were done sequentially. With this in mind, following should work : 1993 * 1994 * - Check for errors before read or write. 1995 * - If the reader encounters error, set the error in sr_read_error. 1996 * Check sr_write_error, if it is set, send cv_signal as it is 1997 * waiting for reader to complete. If it is not set, the writer 1998 * is either running sinking data to the network or blocked 1999 * because of flow control. For handling the latter case, we 2000 * always send a signal. In any case, it will examine sr_read_error 2001 * and return. sr_read_error is marked with SR_READ_DONE to tell 2002 * the writer that the reader is done in all the cases. 2003 * - If the writer encounters error, set the error in sr_write_error. 2004 * The reader thread is either blocked because of flow control or 2005 * running reading data from the disk. For the former, we need to 2006 * wakeup the thread. Again to keep it simple, we always wake up 2007 * the reader thread. Then, wait for the read thread to complete 2008 * if it is not done yet. Cleanup and return. 2009 * 2010 * High and low water marks for the read thread. 2011 * -------------------------------------------- 2012 * 2013 * If sendfile() is used to send data over a slow network, we need to 2014 * make sure that the read thread does not produce data at a faster 2015 * rate than the network. This can happen if the disk is faster than 2016 * the network. In such a case, we don't want to build a very large queue. 2017 * But we would still like to get all of the network throughput possible. 2018 * This implies that network should never block waiting for data. 2019 * As there are lot of disk throughput/network throughput combinations 2020 * possible, it is difficult to come up with an accurate number. 2021 * A typical 10K RPM disk has a max seek latency 17ms and rotational 2022 * latency of 3ms for reading a disk block. Thus, the total latency to 2023 * initiate a new read, transfer data from the disk and queue for 2024 * transmission would take about a max of 25ms. Todays max transfer rate 2025 * for network is 100MB/sec. If the thread is blocked because of flow 2026 * control, it would take 25ms to get new data ready for transmission. 2027 * We have to make sure that network is not idling, while we are initiating 2028 * new transfers. So, at 100MB/sec, to keep network busy we would need 2029 * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data. 2030 * We need to pick a high water mark so that the woken up thread would 2031 * do considerable work before blocking again to prevent thrashing. Currently, 2032 * we pick this to be 10 times that of the low water mark. 2033 * 2034 * Sendfile with segmap caching (One copy from page cache to mblks). 2035 * ---------------------------------------------------------------- 2036 * 2037 * We use the segmap cache for caching the file, if the size of file 2038 * is <= sendfile_max_size. In this case we don't use threads as VM 2039 * is reasonably fast enough to keep up with the network. If the underlying 2040 * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth 2041 * of data into segmap space, and use the virtual address from segmap 2042 * directly through desballoc() to avoid copy. Once the transport is done 2043 * with the data, the mapping will be released through segmap_release() 2044 * called by the call-back routine. 2045 * 2046 * If zero-copy is not allowed by the transport, we simply call VOP_READ() 2047 * to copy the data from the filesystem into our temporary network buffer. 2048 * 2049 * To disable caching, set sendfile_max_size to 0. 2050 */ 2051 2052 uint_t sendfile_read_size = 1024 * 1024; 2053 #define SENDFILE_REQ_LOWAT 3 * 1024 * 1024 2054 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT; 2055 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT; 2056 struct sendfile_stats sf_stats; 2057 struct sendfile_queue *snfq; 2058 clock_t snfq_timeout; 2059 off64_t sendfile_max_size; 2060 2061 static void snf_enque(snf_req_t *, mblk_t *); 2062 static mblk_t *snf_deque(snf_req_t *); 2063 2064 void 2065 sendfile_init(void) 2066 { 2067 snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP); 2068 2069 mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL); 2070 cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL); 2071 snfq->snfq_max_threads = max_ncpus; 2072 snfq_timeout = SNFQ_TIMEOUT; 2073 /* Cache all files by default. */ 2074 sendfile_max_size = MAXOFFSET_T; 2075 } 2076 2077 /* 2078 * Queues a mblk_t for network processing. 2079 */ 2080 static void 2081 snf_enque(snf_req_t *sr, mblk_t *mp) 2082 { 2083 mp->b_next = NULL; 2084 mutex_enter(&sr->sr_lock); 2085 if (sr->sr_mp_head == NULL) { 2086 sr->sr_mp_head = sr->sr_mp_tail = mp; 2087 cv_signal(&sr->sr_cv); 2088 } else { 2089 sr->sr_mp_tail->b_next = mp; 2090 sr->sr_mp_tail = mp; 2091 } 2092 sr->sr_qlen += MBLKL(mp); 2093 while ((sr->sr_qlen > sr->sr_hiwat) && 2094 (sr->sr_write_error == 0)) { 2095 sf_stats.ss_full_waits++; 2096 cv_wait(&sr->sr_cv, &sr->sr_lock); 2097 } 2098 mutex_exit(&sr->sr_lock); 2099 } 2100 2101 /* 2102 * De-queues a mblk_t for network processing. 2103 */ 2104 static mblk_t * 2105 snf_deque(snf_req_t *sr) 2106 { 2107 mblk_t *mp; 2108 2109 mutex_enter(&sr->sr_lock); 2110 /* 2111 * If we have encountered an error on read or read is 2112 * completed and no more mblks, return NULL. 2113 * We need to check for NULL sr_mp_head also as 2114 * the reads could have completed and there is 2115 * nothing more to come. 2116 */ 2117 if (((sr->sr_read_error & ~SR_READ_DONE) != 0) || 2118 ((sr->sr_read_error & SR_READ_DONE) && 2119 sr->sr_mp_head == NULL)) { 2120 mutex_exit(&sr->sr_lock); 2121 return (NULL); 2122 } 2123 /* 2124 * To start with neither SR_READ_DONE is marked nor 2125 * the error is set. When we wake up from cv_wait, 2126 * following are the possibilities : 2127 * 2128 * a) sr_read_error is zero and mblks are queued. 2129 * b) sr_read_error is set to SR_READ_DONE 2130 * and mblks are queued. 2131 * c) sr_read_error is set to SR_READ_DONE 2132 * and no mblks. 2133 * d) sr_read_error is set to some error other 2134 * than SR_READ_DONE. 2135 */ 2136 2137 while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) { 2138 sf_stats.ss_empty_waits++; 2139 cv_wait(&sr->sr_cv, &sr->sr_lock); 2140 } 2141 /* Handle (a) and (b) first - the normal case. */ 2142 if (((sr->sr_read_error & ~SR_READ_DONE) == 0) && 2143 (sr->sr_mp_head != NULL)) { 2144 mp = sr->sr_mp_head; 2145 sr->sr_mp_head = mp->b_next; 2146 sr->sr_qlen -= MBLKL(mp); 2147 if (sr->sr_qlen < sr->sr_lowat) 2148 cv_signal(&sr->sr_cv); 2149 mutex_exit(&sr->sr_lock); 2150 mp->b_next = NULL; 2151 return (mp); 2152 } 2153 /* Handle (c) and (d). */ 2154 mutex_exit(&sr->sr_lock); 2155 return (NULL); 2156 } 2157 2158 /* 2159 * Reads data from the filesystem and queues it for network processing. 2160 */ 2161 void 2162 snf_async_read(snf_req_t *sr) 2163 { 2164 size_t iosize; 2165 u_offset_t fileoff; 2166 u_offset_t size; 2167 int ret_size; 2168 int error; 2169 file_t *fp; 2170 mblk_t *mp; 2171 struct vnode *vp; 2172 int extra = 0; 2173 int maxblk = 0; 2174 int wroff = 0; 2175 struct sonode *so; 2176 2177 fp = sr->sr_fp; 2178 size = sr->sr_file_size; 2179 fileoff = sr->sr_file_off; 2180 2181 /* 2182 * Ignore the error for filesystems that doesn't support DIRECTIO. 2183 */ 2184 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0, 2185 kcred, NULL, NULL); 2186 2187 vp = sr->sr_vp; 2188 if (vp->v_type == VSOCK) { 2189 stdata_t *stp; 2190 2191 /* 2192 * Get the extra space to insert a header and a trailer. 2193 */ 2194 so = VTOSO(vp); 2195 stp = vp->v_stream; 2196 if (stp == NULL) { 2197 wroff = so->so_proto_props.sopp_wroff; 2198 maxblk = so->so_proto_props.sopp_maxblk; 2199 extra = wroff + so->so_proto_props.sopp_tail; 2200 } else { 2201 wroff = (int)(stp->sd_wroff); 2202 maxblk = (int)(stp->sd_maxblk); 2203 extra = wroff + (int)(stp->sd_tail); 2204 } 2205 } 2206 2207 while ((size != 0) && (sr->sr_write_error == 0)) { 2208 2209 iosize = (int)MIN(sr->sr_maxpsz, size); 2210 2211 /* 2212 * Socket filters can limit the mblk size, 2213 * so limit reads to maxblk if there are 2214 * filters present. 2215 */ 2216 if (vp->v_type == VSOCK && 2217 so->so_filter_active > 0 && maxblk != INFPSZ) 2218 iosize = (int)MIN(iosize, maxblk); 2219 2220 if (is_system_labeled()) { 2221 mp = allocb_cred(iosize + extra, CRED(), 2222 curproc->p_pid); 2223 } else { 2224 mp = allocb(iosize + extra, BPRI_MED); 2225 } 2226 if (mp == NULL) { 2227 error = EAGAIN; 2228 break; 2229 } 2230 2231 mp->b_rptr += wroff; 2232 2233 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize); 2234 2235 /* Error or Reached EOF ? */ 2236 if ((error != 0) || (ret_size == 0)) { 2237 freeb(mp); 2238 break; 2239 } 2240 mp->b_wptr = mp->b_rptr + ret_size; 2241 2242 snf_enque(sr, mp); 2243 size -= ret_size; 2244 fileoff += ret_size; 2245 } 2246 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0, 2247 kcred, NULL, NULL); 2248 mutex_enter(&sr->sr_lock); 2249 sr->sr_read_error = error; 2250 sr->sr_read_error |= SR_READ_DONE; 2251 cv_signal(&sr->sr_cv); 2252 mutex_exit(&sr->sr_lock); 2253 } 2254 2255 void 2256 snf_async_thread(void) 2257 { 2258 snf_req_t *sr; 2259 callb_cpr_t cprinfo; 2260 clock_t time_left = 1; 2261 2262 CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq"); 2263 2264 mutex_enter(&snfq->snfq_lock); 2265 for (;;) { 2266 /* 2267 * If we didn't find a entry, then block until woken up 2268 * again and then look through the queues again. 2269 */ 2270 while ((sr = snfq->snfq_req_head) == NULL) { 2271 CALLB_CPR_SAFE_BEGIN(&cprinfo); 2272 if (time_left <= 0) { 2273 snfq->snfq_svc_threads--; 2274 CALLB_CPR_EXIT(&cprinfo); 2275 thread_exit(); 2276 /* NOTREACHED */ 2277 } 2278 snfq->snfq_idle_cnt++; 2279 2280 time_left = cv_reltimedwait(&snfq->snfq_cv, 2281 &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK); 2282 snfq->snfq_idle_cnt--; 2283 2284 CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock); 2285 } 2286 snfq->snfq_req_head = sr->sr_next; 2287 snfq->snfq_req_cnt--; 2288 mutex_exit(&snfq->snfq_lock); 2289 snf_async_read(sr); 2290 mutex_enter(&snfq->snfq_lock); 2291 } 2292 } 2293 2294 2295 snf_req_t * 2296 create_thread(int operation, struct vnode *vp, file_t *fp, 2297 u_offset_t fileoff, u_offset_t size) 2298 { 2299 snf_req_t *sr; 2300 stdata_t *stp; 2301 2302 sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP); 2303 2304 sr->sr_vp = vp; 2305 sr->sr_fp = fp; 2306 stp = vp->v_stream; 2307 2308 /* 2309 * store sd_qn_maxpsz into sr_maxpsz while we have stream head. 2310 * stream might be closed before thread returns from snf_async_read. 2311 */ 2312 if (stp != NULL && stp->sd_qn_maxpsz > 0) { 2313 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz); 2314 } else { 2315 sr->sr_maxpsz = MAXBSIZE; 2316 } 2317 2318 sr->sr_operation = operation; 2319 sr->sr_file_off = fileoff; 2320 sr->sr_file_size = size; 2321 sr->sr_hiwat = sendfile_req_hiwat; 2322 sr->sr_lowat = sendfile_req_lowat; 2323 mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL); 2324 cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL); 2325 /* 2326 * See whether we need another thread for servicing this 2327 * request. If there are already enough requests queued 2328 * for the threads, create one if not exceeding 2329 * snfq_max_threads. 2330 */ 2331 mutex_enter(&snfq->snfq_lock); 2332 if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt && 2333 snfq->snfq_svc_threads < snfq->snfq_max_threads) { 2334 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0, 2335 TS_RUN, minclsyspri); 2336 snfq->snfq_svc_threads++; 2337 } 2338 if (snfq->snfq_req_head == NULL) { 2339 snfq->snfq_req_head = snfq->snfq_req_tail = sr; 2340 cv_signal(&snfq->snfq_cv); 2341 } else { 2342 snfq->snfq_req_tail->sr_next = sr; 2343 snfq->snfq_req_tail = sr; 2344 } 2345 snfq->snfq_req_cnt++; 2346 mutex_exit(&snfq->snfq_lock); 2347 return (sr); 2348 } 2349 2350 int 2351 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size, 2352 ssize_t *count) 2353 { 2354 snf_req_t *sr; 2355 mblk_t *mp; 2356 int iosize; 2357 int error = 0; 2358 short fflag; 2359 struct vnode *vp; 2360 int ksize; 2361 struct nmsghdr msg; 2362 2363 ksize = 0; 2364 *count = 0; 2365 bzero(&msg, sizeof (msg)); 2366 2367 vp = fp->f_vnode; 2368 fflag = fp->f_flag; 2369 if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL) 2370 return (EAGAIN); 2371 2372 /* 2373 * We check for read error in snf_deque. It has to check 2374 * for successful READ_DONE and return NULL, and we might 2375 * as well make an additional check there. 2376 */ 2377 while ((mp = snf_deque(sr)) != NULL) { 2378 2379 if (ISSIG(curthread, JUSTLOOKING)) { 2380 freeb(mp); 2381 error = EINTR; 2382 break; 2383 } 2384 iosize = MBLKL(mp); 2385 2386 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp); 2387 2388 if (error != 0) { 2389 if (mp != NULL) 2390 freeb(mp); 2391 break; 2392 } 2393 ksize += iosize; 2394 } 2395 *count = ksize; 2396 2397 mutex_enter(&sr->sr_lock); 2398 sr->sr_write_error = error; 2399 /* Look at the big comments on why we cv_signal here. */ 2400 cv_signal(&sr->sr_cv); 2401 2402 /* Wait for the reader to complete always. */ 2403 while (!(sr->sr_read_error & SR_READ_DONE)) { 2404 cv_wait(&sr->sr_cv, &sr->sr_lock); 2405 } 2406 /* If there is no write error, check for read error. */ 2407 if (error == 0) 2408 error = (sr->sr_read_error & ~SR_READ_DONE); 2409 2410 if (error != 0) { 2411 mblk_t *next_mp; 2412 2413 mp = sr->sr_mp_head; 2414 while (mp != NULL) { 2415 next_mp = mp->b_next; 2416 mp->b_next = NULL; 2417 freeb(mp); 2418 mp = next_mp; 2419 } 2420 } 2421 mutex_exit(&sr->sr_lock); 2422 kmem_free(sr, sizeof (snf_req_t)); 2423 return (error); 2424 } 2425 2426 /* Maximum no.of pages allocated by vpm for sendfile at a time */ 2427 #define SNF_VPMMAXPGS (VPMMAXPGS/2) 2428 2429 /* 2430 * Maximum no.of elements in the list returned by vpm, including 2431 * NULL for the last entry 2432 */ 2433 #define SNF_MAXVMAPS (SNF_VPMMAXPGS + 1) 2434 2435 typedef struct { 2436 unsigned int snfv_ref; 2437 frtn_t snfv_frtn; 2438 vnode_t *snfv_vp; 2439 struct vmap snfv_vml[SNF_MAXVMAPS]; 2440 } snf_vmap_desbinfo; 2441 2442 typedef struct { 2443 frtn_t snfi_frtn; 2444 caddr_t snfi_base; 2445 uint_t snfi_mapoff; 2446 size_t snfi_len; 2447 vnode_t *snfi_vp; 2448 } snf_smap_desbinfo; 2449 2450 /* 2451 * The callback function used for vpm mapped mblks called when the last ref of 2452 * the mblk is dropped which normally occurs when TCP receives the ack. But it 2453 * can be the driver too due to lazy reclaim. 2454 */ 2455 void 2456 snf_vmap_desbfree(snf_vmap_desbinfo *snfv) 2457 { 2458 ASSERT(snfv->snfv_ref != 0); 2459 if (atomic_dec_32_nv(&snfv->snfv_ref) == 0) { 2460 vpm_unmap_pages(snfv->snfv_vml, S_READ); 2461 VN_RELE(snfv->snfv_vp); 2462 kmem_free(snfv, sizeof (snf_vmap_desbinfo)); 2463 } 2464 } 2465 2466 /* 2467 * The callback function used for segmap'ped mblks called when the last ref of 2468 * the mblk is dropped which normally occurs when TCP receives the ack. But it 2469 * can be the driver too due to lazy reclaim. 2470 */ 2471 void 2472 snf_smap_desbfree(snf_smap_desbinfo *snfi) 2473 { 2474 if (! IS_KPM_ADDR(snfi->snfi_base)) { 2475 /* 2476 * We don't need to call segmap_fault(F_SOFTUNLOCK) for 2477 * segmap_kpm as long as the latter never falls back to 2478 * "use_segmap_range". (See segmap_getmapflt().) 2479 * 2480 * Using S_OTHER saves an redundant hat_setref() in 2481 * segmap_unlock() 2482 */ 2483 (void) segmap_fault(kas.a_hat, segkmap, 2484 (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base + 2485 snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len, 2486 F_SOFTUNLOCK, S_OTHER); 2487 } 2488 (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED); 2489 VN_RELE(snfi->snfi_vp); 2490 kmem_free(snfi, sizeof (*snfi)); 2491 } 2492 2493 /* 2494 * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk. 2495 * When segmap is used, the mblk contains a segmap slot of no more 2496 * than MAXBSIZE. 2497 * 2498 * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained 2499 * in each iteration and sent by socket_sendmblk until an error occurs or 2500 * the requested size has been transferred. An mblk is esballoca'ed from 2501 * each mapped page and a chain of these mblk is sent to the transport layer. 2502 * vpm will be called to unmap the pages when all mblks have been freed by 2503 * free_func. 2504 * 2505 * At the end of the whole sendfile() operation, we wait till the data from 2506 * the last mblk is ack'ed by the transport before returning so that the 2507 * caller of sendfile() can safely modify the file content. 2508 * 2509 * The caller of this function should make sure that total_size does not exceed 2510 * the actual file size of fvp. 2511 */ 2512 int 2513 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size, 2514 ssize_t *count, boolean_t nowait) 2515 { 2516 caddr_t base; 2517 int mapoff; 2518 vnode_t *vp; 2519 mblk_t *mp = NULL; 2520 int chain_size; 2521 int error; 2522 clock_t deadlk_wait; 2523 short fflag; 2524 int ksize; 2525 struct vattr va; 2526 boolean_t dowait = B_FALSE; 2527 struct nmsghdr msg; 2528 2529 vp = fp->f_vnode; 2530 fflag = fp->f_flag; 2531 ksize = 0; 2532 bzero(&msg, sizeof (msg)); 2533 2534 for (;;) { 2535 if (ISSIG(curthread, JUSTLOOKING)) { 2536 error = EINTR; 2537 break; 2538 } 2539 2540 if (vpm_enable) { 2541 snf_vmap_desbinfo *snfv; 2542 mblk_t *nmp; 2543 int mblk_size; 2544 int maxsize; 2545 int i; 2546 2547 mapoff = fileoff & PAGEOFFSET; 2548 maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size); 2549 2550 snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo), 2551 KM_SLEEP); 2552 2553 /* 2554 * Get vpm mappings for maxsize with read access. 2555 * If the pages aren't available yet, we get 2556 * DEADLK, so wait and try again a little later using 2557 * an increasing wait. We might be here a long time. 2558 * 2559 * If delay_sig returns EINTR, be sure to exit and 2560 * pass it up to the caller. 2561 */ 2562 deadlk_wait = 0; 2563 while ((error = vpm_map_pages(fvp, fileoff, 2564 (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml, 2565 SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) { 2566 deadlk_wait += (deadlk_wait < 5) ? 1 : 4; 2567 if ((error = delay_sig(deadlk_wait)) != 0) { 2568 break; 2569 } 2570 } 2571 if (error != 0) { 2572 kmem_free(snfv, sizeof (snf_vmap_desbinfo)); 2573 error = (error == EINTR) ? EINTR : EIO; 2574 goto out; 2575 } 2576 snfv->snfv_frtn.free_func = snf_vmap_desbfree; 2577 snfv->snfv_frtn.free_arg = (caddr_t)snfv; 2578 2579 /* Construct the mblk chain from the page mappings */ 2580 chain_size = 0; 2581 for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) && 2582 total_size > 0; i++) { 2583 ASSERT(chain_size < maxsize); 2584 mblk_size = MIN(snfv->snfv_vml[i].vs_len - 2585 mapoff, total_size); 2586 nmp = esballoca( 2587 (uchar_t *)snfv->snfv_vml[i].vs_addr + 2588 mapoff, mblk_size, BPRI_HI, 2589 &snfv->snfv_frtn); 2590 2591 /* 2592 * We return EAGAIN after unmapping the pages 2593 * if we cannot allocate the the head of the 2594 * chain. Otherwise, we continue sending the 2595 * mblks constructed so far. 2596 */ 2597 if (nmp == NULL) { 2598 if (i == 0) { 2599 vpm_unmap_pages(snfv->snfv_vml, 2600 S_READ); 2601 kmem_free(snfv, 2602 sizeof (snf_vmap_desbinfo)); 2603 error = EAGAIN; 2604 goto out; 2605 } 2606 break; 2607 } 2608 /* Mark this dblk with the zero-copy flag */ 2609 nmp->b_datap->db_struioflag |= STRUIO_ZC; 2610 nmp->b_wptr += mblk_size; 2611 chain_size += mblk_size; 2612 fileoff += mblk_size; 2613 total_size -= mblk_size; 2614 snfv->snfv_ref++; 2615 mapoff = 0; 2616 if (i > 0) 2617 linkb(mp, nmp); 2618 else 2619 mp = nmp; 2620 } 2621 VN_HOLD(fvp); 2622 snfv->snfv_vp = fvp; 2623 } else { 2624 /* vpm not supported. fallback to segmap */ 2625 snf_smap_desbinfo *snfi; 2626 2627 mapoff = fileoff & MAXBOFFSET; 2628 chain_size = MAXBSIZE - mapoff; 2629 if (chain_size > total_size) 2630 chain_size = total_size; 2631 /* 2632 * we don't forcefault because we'll call 2633 * segmap_fault(F_SOFTLOCK) next. 2634 * 2635 * S_READ will get the ref bit set (by either 2636 * segmap_getmapflt() or segmap_fault()) and page 2637 * shared locked. 2638 */ 2639 base = segmap_getmapflt(segkmap, fvp, fileoff, 2640 chain_size, segmap_kpm ? SM_FAULT : 0, S_READ); 2641 2642 snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP); 2643 snfi->snfi_len = (size_t)roundup(mapoff+chain_size, 2644 PAGESIZE)- (mapoff & PAGEMASK); 2645 /* 2646 * We must call segmap_fault() even for segmap_kpm 2647 * because that's how error gets returned. 2648 * (segmap_getmapflt() never fails but segmap_fault() 2649 * does.) 2650 * 2651 * If the pages aren't available yet, we get 2652 * DEADLK, so wait and try again a little later using 2653 * an increasing wait. We might be here a long time. 2654 * 2655 * If delay_sig returns EINTR, be sure to exit and 2656 * pass it up to the caller. 2657 */ 2658 deadlk_wait = 0; 2659 while ((error = FC_ERRNO(segmap_fault(kas.a_hat, 2660 segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base + 2661 mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK, 2662 S_READ))) == EDEADLK) { 2663 deadlk_wait += (deadlk_wait < 5) ? 1 : 4; 2664 if ((error = delay_sig(deadlk_wait)) != 0) { 2665 break; 2666 } 2667 } 2668 if (error != 0) { 2669 (void) segmap_release(segkmap, base, 0); 2670 kmem_free(snfi, sizeof (*snfi)); 2671 error = (error == EINTR) ? EINTR : EIO; 2672 goto out; 2673 } 2674 snfi->snfi_frtn.free_func = snf_smap_desbfree; 2675 snfi->snfi_frtn.free_arg = (caddr_t)snfi; 2676 snfi->snfi_base = base; 2677 snfi->snfi_mapoff = mapoff; 2678 mp = esballoca((uchar_t *)base + mapoff, chain_size, 2679 BPRI_HI, &snfi->snfi_frtn); 2680 2681 if (mp == NULL) { 2682 (void) segmap_fault(kas.a_hat, segkmap, 2683 (caddr_t)(uintptr_t)(((uintptr_t)base + 2684 mapoff) & PAGEMASK), snfi->snfi_len, 2685 F_SOFTUNLOCK, S_OTHER); 2686 (void) segmap_release(segkmap, base, 0); 2687 kmem_free(snfi, sizeof (*snfi)); 2688 freemsg(mp); 2689 error = EAGAIN; 2690 goto out; 2691 } 2692 VN_HOLD(fvp); 2693 snfi->snfi_vp = fvp; 2694 mp->b_wptr += chain_size; 2695 2696 /* Mark this dblk with the zero-copy flag */ 2697 mp->b_datap->db_struioflag |= STRUIO_ZC; 2698 fileoff += chain_size; 2699 total_size -= chain_size; 2700 } 2701 2702 if (total_size == 0 && !nowait) { 2703 ASSERT(!dowait); 2704 dowait = B_TRUE; 2705 mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY; 2706 } 2707 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2708 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp); 2709 if (error != 0) { 2710 /* 2711 * mp contains the mblks that were not sent by 2712 * socket_sendmblk. Use its size to update *count 2713 */ 2714 *count = ksize + (chain_size - msgdsize(mp)); 2715 if (mp != NULL) 2716 freemsg(mp); 2717 return (error); 2718 } 2719 ksize += chain_size; 2720 if (total_size == 0) 2721 goto done; 2722 2723 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2724 va.va_mask = AT_SIZE; 2725 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL); 2726 if (error) 2727 break; 2728 /* Read as much as possible. */ 2729 if (fileoff >= va.va_size) 2730 break; 2731 if (total_size + fileoff > va.va_size) 2732 total_size = va.va_size - fileoff; 2733 } 2734 out: 2735 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2736 done: 2737 *count = ksize; 2738 if (dowait) { 2739 stdata_t *stp; 2740 2741 stp = vp->v_stream; 2742 if (stp == NULL) { 2743 struct sonode *so; 2744 so = VTOSO(vp); 2745 error = so_zcopy_wait(so); 2746 } else { 2747 mutex_enter(&stp->sd_lock); 2748 while (!(stp->sd_flag & STZCNOTIFY)) { 2749 if (cv_wait_sig(&stp->sd_zcopy_wait, 2750 &stp->sd_lock) == 0) { 2751 error = EINTR; 2752 break; 2753 } 2754 } 2755 stp->sd_flag &= ~STZCNOTIFY; 2756 mutex_exit(&stp->sd_lock); 2757 } 2758 } 2759 return (error); 2760 } 2761 2762 int 2763 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size, 2764 uint_t maxpsz, ssize_t *count) 2765 { 2766 struct vnode *vp; 2767 mblk_t *mp; 2768 int iosize; 2769 int extra = 0; 2770 int error; 2771 short fflag; 2772 int ksize; 2773 int ioflag; 2774 struct uio auio; 2775 struct iovec aiov; 2776 struct vattr va; 2777 int maxblk = 0; 2778 int wroff = 0; 2779 struct sonode *so; 2780 struct nmsghdr msg; 2781 2782 vp = fp->f_vnode; 2783 if (vp->v_type == VSOCK) { 2784 stdata_t *stp; 2785 2786 /* 2787 * Get the extra space to insert a header and a trailer. 2788 */ 2789 so = VTOSO(vp); 2790 stp = vp->v_stream; 2791 if (stp == NULL) { 2792 wroff = so->so_proto_props.sopp_wroff; 2793 maxblk = so->so_proto_props.sopp_maxblk; 2794 extra = wroff + so->so_proto_props.sopp_tail; 2795 } else { 2796 wroff = (int)(stp->sd_wroff); 2797 maxblk = (int)(stp->sd_maxblk); 2798 extra = wroff + (int)(stp->sd_tail); 2799 } 2800 } 2801 bzero(&msg, sizeof (msg)); 2802 fflag = fp->f_flag; 2803 ksize = 0; 2804 auio.uio_iov = &aiov; 2805 auio.uio_iovcnt = 1; 2806 auio.uio_segflg = UIO_SYSSPACE; 2807 auio.uio_llimit = MAXOFFSET_T; 2808 auio.uio_fmode = fflag; 2809 auio.uio_extflg = UIO_COPY_CACHED; 2810 ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC); 2811 /* If read sync is not asked for, filter sync flags */ 2812 if ((ioflag & FRSYNC) == 0) 2813 ioflag &= ~(FSYNC|FDSYNC); 2814 for (;;) { 2815 if (ISSIG(curthread, JUSTLOOKING)) { 2816 error = EINTR; 2817 break; 2818 } 2819 iosize = (int)MIN(maxpsz, size); 2820 2821 /* 2822 * Socket filters can limit the mblk size, 2823 * so limit reads to maxblk if there are 2824 * filters present. 2825 */ 2826 if (vp->v_type == VSOCK && 2827 so->so_filter_active > 0 && maxblk != INFPSZ) 2828 iosize = (int)MIN(iosize, maxblk); 2829 2830 if (is_system_labeled()) { 2831 mp = allocb_cred(iosize + extra, CRED(), 2832 curproc->p_pid); 2833 } else { 2834 mp = allocb(iosize + extra, BPRI_MED); 2835 } 2836 if (mp == NULL) { 2837 error = EAGAIN; 2838 break; 2839 } 2840 2841 mp->b_rptr += wroff; 2842 2843 aiov.iov_base = (caddr_t)mp->b_rptr; 2844 aiov.iov_len = iosize; 2845 auio.uio_loffset = fileoff; 2846 auio.uio_resid = iosize; 2847 2848 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL); 2849 iosize -= auio.uio_resid; 2850 2851 if (error == EINTR && iosize != 0) 2852 error = 0; 2853 2854 if (error != 0 || iosize == 0) { 2855 freeb(mp); 2856 break; 2857 } 2858 mp->b_wptr = mp->b_rptr + iosize; 2859 2860 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2861 2862 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp); 2863 2864 if (error != 0) { 2865 *count = ksize; 2866 if (mp != NULL) 2867 freeb(mp); 2868 return (error); 2869 } 2870 ksize += iosize; 2871 size -= iosize; 2872 if (size == 0) 2873 goto done; 2874 2875 fileoff += iosize; 2876 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2877 va.va_mask = AT_SIZE; 2878 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL); 2879 if (error) 2880 break; 2881 /* Read as much as possible. */ 2882 if (fileoff >= va.va_size) 2883 size = 0; 2884 else if (size + fileoff > va.va_size) 2885 size = va.va_size - fileoff; 2886 } 2887 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2888 done: 2889 *count = ksize; 2890 return (error); 2891 } 2892 2893 #if defined(_SYSCALL32_IMPL) || defined(_ILP32) 2894 /* 2895 * Largefile support for 32 bit applications only. 2896 */ 2897 int 2898 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv, 2899 ssize32_t *count32) 2900 { 2901 ssize32_t sfv_len; 2902 u_offset_t sfv_off, va_size; 2903 struct vnode *vp, *fvp, *realvp; 2904 struct vattr va; 2905 stdata_t *stp; 2906 ssize_t count = 0; 2907 int error = 0; 2908 boolean_t dozcopy = B_FALSE; 2909 uint_t maxpsz; 2910 2911 sfv_len = (ssize32_t)sfv->sfv_len; 2912 if (sfv_len < 0) { 2913 error = EINVAL; 2914 goto out; 2915 } 2916 2917 if (sfv_len == 0) goto out; 2918 2919 sfv_off = (u_offset_t)sfv->sfv_off; 2920 2921 /* Same checks as in pread */ 2922 if (sfv_off > MAXOFFSET_T) { 2923 error = EINVAL; 2924 goto out; 2925 } 2926 if (sfv_off + sfv_len > MAXOFFSET_T) 2927 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off); 2928 2929 /* 2930 * There are no more checks on sfv_len. So, we cast it to 2931 * u_offset_t and share the snf_direct_io/snf_cache code between 2932 * 32 bit and 64 bit. 2933 * 2934 * TODO: should do nbl_need_check() like read()? 2935 */ 2936 if (sfv_len > sendfile_max_size) { 2937 sf_stats.ss_file_not_cached++; 2938 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len, 2939 &count); 2940 goto out; 2941 } 2942 fvp = rfp->f_vnode; 2943 if (VOP_REALVP(fvp, &realvp, NULL) == 0) 2944 fvp = realvp; 2945 /* 2946 * Grab the lock as a reader to prevent the file size 2947 * from changing underneath. 2948 */ 2949 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2950 va.va_mask = AT_SIZE; 2951 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL); 2952 va_size = va.va_size; 2953 if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) { 2954 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL); 2955 goto out; 2956 } 2957 /* Read as much as possible. */ 2958 if (sfv_off + sfv_len > va_size) 2959 sfv_len = va_size - sfv_off; 2960 2961 vp = fp->f_vnode; 2962 stp = vp->v_stream; 2963 /* 2964 * When the NOWAIT flag is not set, we enable zero-copy only if the 2965 * transfer size is large enough. This prevents performance loss 2966 * when the caller sends the file piece by piece. 2967 */ 2968 if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) || 2969 (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) && 2970 !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) { 2971 uint_t copyflag; 2972 copyflag = stp != NULL ? stp->sd_copyflag : 2973 VTOSO(vp)->so_proto_props.sopp_zcopyflag; 2974 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) { 2975 int on = 1; 2976 2977 if (socket_setsockopt(VTOSO(vp), SOL_SOCKET, 2978 SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0) 2979 dozcopy = B_TRUE; 2980 } else { 2981 dozcopy = copyflag & STZCVMSAFE; 2982 } 2983 } 2984 if (dozcopy) { 2985 sf_stats.ss_file_segmap++; 2986 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len, 2987 &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0)); 2988 } else { 2989 if (vp->v_type == VSOCK && stp == NULL) { 2990 sonode_t *so = VTOSO(vp); 2991 maxpsz = so->so_proto_props.sopp_maxpsz; 2992 } else if (stp != NULL) { 2993 maxpsz = stp->sd_qn_maxpsz; 2994 } else { 2995 maxpsz = maxphys; 2996 } 2997 2998 if (maxpsz == INFPSZ) 2999 maxpsz = maxphys; 3000 else 3001 maxpsz = roundup(maxpsz, MAXBSIZE); 3002 sf_stats.ss_file_cached++; 3003 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len, 3004 maxpsz, &count); 3005 } 3006 out: 3007 releasef(sfv->sfv_fd); 3008 *count32 = (ssize32_t)count; 3009 return (error); 3010 } 3011 #endif 3012 3013 #ifdef _SYSCALL32_IMPL 3014 /* 3015 * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a 3016 * ssize_t rather than ssize32_t; see the comments above read32 for details. 3017 */ 3018 3019 ssize_t 3020 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags) 3021 { 3022 return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags)); 3023 } 3024 3025 ssize_t 3026 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags, 3027 caddr32_t name, caddr32_t namelenp) 3028 { 3029 return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags, 3030 (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp)); 3031 } 3032 3033 ssize_t 3034 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags) 3035 { 3036 return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags)); 3037 } 3038 3039 ssize_t 3040 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags, 3041 caddr32_t name, socklen_t namelen) 3042 { 3043 return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags, 3044 (void *)(uintptr_t)name, namelen)); 3045 } 3046 #endif /* _SYSCALL32_IMPL */ 3047 3048 /* 3049 * Function wrappers (mostly around the sonode switch) for 3050 * backward compatibility. 3051 */ 3052 3053 int 3054 soaccept(struct sonode *so, int fflag, struct sonode **nsop) 3055 { 3056 return (socket_accept(so, fflag, CRED(), nsop)); 3057 } 3058 3059 int 3060 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen, 3061 int backlog, int flags) 3062 { 3063 int error; 3064 3065 error = socket_bind(so, name, namelen, flags, CRED()); 3066 if (error == 0 && backlog != 0) 3067 return (socket_listen(so, backlog, CRED())); 3068 3069 return (error); 3070 } 3071 3072 int 3073 solisten(struct sonode *so, int backlog) 3074 { 3075 return (socket_listen(so, backlog, CRED())); 3076 } 3077 3078 int 3079 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen, 3080 int fflag, int flags) 3081 { 3082 return (socket_connect(so, name, namelen, fflag, flags, CRED())); 3083 } 3084 3085 int 3086 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop) 3087 { 3088 return (socket_recvmsg(so, msg, uiop, CRED())); 3089 } 3090 3091 int 3092 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop) 3093 { 3094 return (socket_sendmsg(so, msg, uiop, CRED())); 3095 } 3096 3097 int 3098 soshutdown(struct sonode *so, int how) 3099 { 3100 return (socket_shutdown(so, how, CRED())); 3101 } 3102 3103 int 3104 sogetsockopt(struct sonode *so, int level, int option_name, void *optval, 3105 socklen_t *optlenp, int flags) 3106 { 3107 return (socket_getsockopt(so, level, option_name, optval, optlenp, 3108 flags, CRED())); 3109 } 3110 3111 int 3112 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval, 3113 t_uscalar_t optlen) 3114 { 3115 return (socket_setsockopt(so, level, option_name, optval, optlen, 3116 CRED())); 3117 } 3118 3119 /* 3120 * Because this is backward compatibility interface it only needs to be 3121 * able to handle the creation of TPI sockfs sockets. 3122 */ 3123 struct sonode * 3124 socreate(struct sockparams *sp, int family, int type, int protocol, int version, 3125 int *errorp) 3126 { 3127 struct sonode *so; 3128 3129 ASSERT(sp != NULL); 3130 3131 so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol, 3132 version, SOCKET_SLEEP, errorp, CRED()); 3133 if (so == NULL) { 3134 SOCKPARAMS_DEC_REF(sp); 3135 } else { 3136 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) { 3137 /* Cannot fail, only bumps so_count */ 3138 (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL); 3139 } else { 3140 socket_destroy(so); 3141 so = NULL; 3142 } 3143 } 3144 return (so); 3145 } 3146