1 /*- 2 * Copyright (c) 1982, 1986, 1989, 1990, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * sendfile(2) and related extensions: 6 * Copyright (c) 1998, David Greenman. All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_syscalls.c 8.4 (Berkeley) 2/21/94 33 */ 34 35 #include <sys/cdefs.h> 36 __FBSDID("$FreeBSD$"); 37 38 #include "opt_capsicum.h" 39 #include "opt_inet.h" 40 #include "opt_inet6.h" 41 #include "opt_compat.h" 42 #include "opt_ktrace.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/capsicum.h> 47 #include <sys/condvar.h> 48 #include <sys/kernel.h> 49 #include <sys/lock.h> 50 #include <sys/mutex.h> 51 #include <sys/sysproto.h> 52 #include <sys/malloc.h> 53 #include <sys/filedesc.h> 54 #include <sys/event.h> 55 #include <sys/proc.h> 56 #include <sys/fcntl.h> 57 #include <sys/file.h> 58 #include <sys/filio.h> 59 #include <sys/jail.h> 60 #include <sys/mman.h> 61 #include <sys/mount.h> 62 #include <sys/mbuf.h> 63 #include <sys/protosw.h> 64 #include <sys/rwlock.h> 65 #include <sys/sf_buf.h> 66 #include <sys/sysent.h> 67 #include <sys/socket.h> 68 #include <sys/socketvar.h> 69 #include <sys/signalvar.h> 70 #include <sys/syscallsubr.h> 71 #include <sys/sysctl.h> 72 #include <sys/uio.h> 73 #include <sys/vnode.h> 74 #ifdef KTRACE 75 #include <sys/ktrace.h> 76 #endif 77 #ifdef COMPAT_FREEBSD32 78 #include <compat/freebsd32/freebsd32_util.h> 79 #endif 80 81 #include <net/vnet.h> 82 83 #include <security/audit/audit.h> 84 #include <security/mac/mac_framework.h> 85 86 #include <vm/vm.h> 87 #include <vm/vm_param.h> 88 #include <vm/vm_object.h> 89 #include <vm/vm_page.h> 90 #include <vm/vm_pager.h> 91 #include <vm/vm_kern.h> 92 #include <vm/vm_extern.h> 93 #include <vm/uma.h> 94 95 /* 96 * Flags for accept1() and kern_accept4(), in addition to SOCK_CLOEXEC 97 * and SOCK_NONBLOCK. 98 */ 99 #define ACCEPT4_INHERIT 0x1 100 #define ACCEPT4_COMPAT 0x2 101 102 static int sendit(struct thread *td, int s, struct msghdr *mp, int flags); 103 static int recvit(struct thread *td, int s, struct msghdr *mp, void *namelenp); 104 105 static int accept1(struct thread *td, int s, struct sockaddr *uname, 106 socklen_t *anamelen, int flags); 107 static int do_sendfile(struct thread *td, struct sendfile_args *uap, 108 int compat); 109 static int getsockname1(struct thread *td, struct getsockname_args *uap, 110 int compat); 111 static int getpeername1(struct thread *td, struct getpeername_args *uap, 112 int compat); 113 114 counter_u64_t sfstat[sizeof(struct sfstat) / sizeof(uint64_t)]; 115 116 /* 117 * sendfile(2)-related variables and associated sysctls 118 */ 119 static SYSCTL_NODE(_kern_ipc, OID_AUTO, sendfile, CTLFLAG_RW, 0, 120 "sendfile(2) tunables"); 121 static int sfreadahead = 1; 122 SYSCTL_INT(_kern_ipc_sendfile, OID_AUTO, readahead, CTLFLAG_RW, 123 &sfreadahead, 0, "Number of sendfile(2) read-ahead MAXBSIZE blocks"); 124 125 static void 126 sfstat_init(const void *unused) 127 { 128 129 COUNTER_ARRAY_ALLOC(sfstat, sizeof(struct sfstat) / sizeof(uint64_t), 130 M_WAITOK); 131 } 132 SYSINIT(sfstat, SI_SUB_MBUF, SI_ORDER_FIRST, sfstat_init, NULL); 133 134 static int 135 sfstat_sysctl(SYSCTL_HANDLER_ARGS) 136 { 137 struct sfstat s; 138 139 COUNTER_ARRAY_COPY(sfstat, &s, sizeof(s) / sizeof(uint64_t)); 140 if (req->newptr) 141 COUNTER_ARRAY_ZERO(sfstat, sizeof(s) / sizeof(uint64_t)); 142 return (SYSCTL_OUT(req, &s, sizeof(s))); 143 } 144 SYSCTL_PROC(_kern_ipc, OID_AUTO, sfstat, CTLTYPE_OPAQUE | CTLFLAG_RW, 145 NULL, 0, sfstat_sysctl, "I", "sendfile statistics"); 146 147 /* 148 * Convert a user file descriptor to a kernel file entry and check if required 149 * capability rights are present. 150 * A reference on the file entry is held upon returning. 151 */ 152 int 153 getsock_cap(struct filedesc *fdp, int fd, cap_rights_t *rightsp, 154 struct file **fpp, u_int *fflagp) 155 { 156 struct file *fp; 157 int error; 158 159 error = fget_unlocked(fdp, fd, rightsp, 0, &fp, NULL); 160 if (error != 0) 161 return (error); 162 if (fp->f_type != DTYPE_SOCKET) { 163 fdrop(fp, curthread); 164 return (ENOTSOCK); 165 } 166 if (fflagp != NULL) 167 *fflagp = fp->f_flag; 168 *fpp = fp; 169 return (0); 170 } 171 172 /* 173 * System call interface to the socket abstraction. 174 */ 175 #if defined(COMPAT_43) 176 #define COMPAT_OLDSOCK 177 #endif 178 179 int 180 sys_socket(td, uap) 181 struct thread *td; 182 struct socket_args /* { 183 int domain; 184 int type; 185 int protocol; 186 } */ *uap; 187 { 188 struct socket *so; 189 struct file *fp; 190 int fd, error, type, oflag, fflag; 191 192 AUDIT_ARG_SOCKET(uap->domain, uap->type, uap->protocol); 193 194 type = uap->type; 195 oflag = 0; 196 fflag = 0; 197 if ((type & SOCK_CLOEXEC) != 0) { 198 type &= ~SOCK_CLOEXEC; 199 oflag |= O_CLOEXEC; 200 } 201 if ((type & SOCK_NONBLOCK) != 0) { 202 type &= ~SOCK_NONBLOCK; 203 fflag |= FNONBLOCK; 204 } 205 206 #ifdef MAC 207 error = mac_socket_check_create(td->td_ucred, uap->domain, type, 208 uap->protocol); 209 if (error != 0) 210 return (error); 211 #endif 212 error = falloc(td, &fp, &fd, oflag); 213 if (error != 0) 214 return (error); 215 /* An extra reference on `fp' has been held for us by falloc(). */ 216 error = socreate(uap->domain, &so, type, uap->protocol, 217 td->td_ucred, td); 218 if (error != 0) { 219 fdclose(td->td_proc->p_fd, fp, fd, td); 220 } else { 221 finit(fp, FREAD | FWRITE | fflag, DTYPE_SOCKET, so, &socketops); 222 if ((fflag & FNONBLOCK) != 0) 223 (void) fo_ioctl(fp, FIONBIO, &fflag, td->td_ucred, td); 224 td->td_retval[0] = fd; 225 } 226 fdrop(fp, td); 227 return (error); 228 } 229 230 /* ARGSUSED */ 231 int 232 sys_bind(td, uap) 233 struct thread *td; 234 struct bind_args /* { 235 int s; 236 caddr_t name; 237 int namelen; 238 } */ *uap; 239 { 240 struct sockaddr *sa; 241 int error; 242 243 error = getsockaddr(&sa, uap->name, uap->namelen); 244 if (error == 0) { 245 error = kern_bindat(td, AT_FDCWD, uap->s, sa); 246 free(sa, M_SONAME); 247 } 248 return (error); 249 } 250 251 int 252 kern_bindat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) 253 { 254 struct socket *so; 255 struct file *fp; 256 cap_rights_t rights; 257 int error; 258 259 AUDIT_ARG_FD(fd); 260 AUDIT_ARG_SOCKADDR(td, dirfd, sa); 261 error = getsock_cap(td->td_proc->p_fd, fd, 262 cap_rights_init(&rights, CAP_BIND), &fp, NULL); 263 if (error != 0) 264 return (error); 265 so = fp->f_data; 266 #ifdef KTRACE 267 if (KTRPOINT(td, KTR_STRUCT)) 268 ktrsockaddr(sa); 269 #endif 270 #ifdef MAC 271 error = mac_socket_check_bind(td->td_ucred, so, sa); 272 if (error == 0) { 273 #endif 274 if (dirfd == AT_FDCWD) 275 error = sobind(so, sa, td); 276 else 277 error = sobindat(dirfd, so, sa, td); 278 #ifdef MAC 279 } 280 #endif 281 fdrop(fp, td); 282 return (error); 283 } 284 285 /* ARGSUSED */ 286 int 287 sys_bindat(td, uap) 288 struct thread *td; 289 struct bindat_args /* { 290 int fd; 291 int s; 292 caddr_t name; 293 int namelen; 294 } */ *uap; 295 { 296 struct sockaddr *sa; 297 int error; 298 299 error = getsockaddr(&sa, uap->name, uap->namelen); 300 if (error == 0) { 301 error = kern_bindat(td, uap->fd, uap->s, sa); 302 free(sa, M_SONAME); 303 } 304 return (error); 305 } 306 307 /* ARGSUSED */ 308 int 309 sys_listen(td, uap) 310 struct thread *td; 311 struct listen_args /* { 312 int s; 313 int backlog; 314 } */ *uap; 315 { 316 struct socket *so; 317 struct file *fp; 318 cap_rights_t rights; 319 int error; 320 321 AUDIT_ARG_FD(uap->s); 322 error = getsock_cap(td->td_proc->p_fd, uap->s, 323 cap_rights_init(&rights, CAP_LISTEN), &fp, NULL); 324 if (error == 0) { 325 so = fp->f_data; 326 #ifdef MAC 327 error = mac_socket_check_listen(td->td_ucred, so); 328 if (error == 0) 329 #endif 330 error = solisten(so, uap->backlog, td); 331 fdrop(fp, td); 332 } 333 return(error); 334 } 335 336 /* 337 * accept1() 338 */ 339 static int 340 accept1(td, s, uname, anamelen, flags) 341 struct thread *td; 342 int s; 343 struct sockaddr *uname; 344 socklen_t *anamelen; 345 int flags; 346 { 347 struct sockaddr *name; 348 socklen_t namelen; 349 struct file *fp; 350 int error; 351 352 if (uname == NULL) 353 return (kern_accept4(td, s, NULL, NULL, flags, NULL)); 354 355 error = copyin(anamelen, &namelen, sizeof (namelen)); 356 if (error != 0) 357 return (error); 358 359 error = kern_accept4(td, s, &name, &namelen, flags, &fp); 360 361 if (error != 0) 362 return (error); 363 364 if (error == 0 && uname != NULL) { 365 #ifdef COMPAT_OLDSOCK 366 if (flags & ACCEPT4_COMPAT) 367 ((struct osockaddr *)name)->sa_family = 368 name->sa_family; 369 #endif 370 error = copyout(name, uname, namelen); 371 } 372 if (error == 0) 373 error = copyout(&namelen, anamelen, 374 sizeof(namelen)); 375 if (error != 0) 376 fdclose(td->td_proc->p_fd, fp, td->td_retval[0], td); 377 fdrop(fp, td); 378 free(name, M_SONAME); 379 return (error); 380 } 381 382 int 383 kern_accept(struct thread *td, int s, struct sockaddr **name, 384 socklen_t *namelen, struct file **fp) 385 { 386 return (kern_accept4(td, s, name, namelen, ACCEPT4_INHERIT, fp)); 387 } 388 389 int 390 kern_accept4(struct thread *td, int s, struct sockaddr **name, 391 socklen_t *namelen, int flags, struct file **fp) 392 { 393 struct filedesc *fdp; 394 struct file *headfp, *nfp = NULL; 395 struct sockaddr *sa = NULL; 396 struct socket *head, *so; 397 cap_rights_t rights; 398 u_int fflag; 399 pid_t pgid; 400 int error, fd, tmp; 401 402 if (name != NULL) 403 *name = NULL; 404 405 AUDIT_ARG_FD(s); 406 fdp = td->td_proc->p_fd; 407 error = getsock_cap(fdp, s, cap_rights_init(&rights, CAP_ACCEPT), 408 &headfp, &fflag); 409 if (error != 0) 410 return (error); 411 head = headfp->f_data; 412 if ((head->so_options & SO_ACCEPTCONN) == 0) { 413 error = EINVAL; 414 goto done; 415 } 416 #ifdef MAC 417 error = mac_socket_check_accept(td->td_ucred, head); 418 if (error != 0) 419 goto done; 420 #endif 421 error = falloc(td, &nfp, &fd, (flags & SOCK_CLOEXEC) ? O_CLOEXEC : 0); 422 if (error != 0) 423 goto done; 424 ACCEPT_LOCK(); 425 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->so_comp)) { 426 ACCEPT_UNLOCK(); 427 error = EWOULDBLOCK; 428 goto noconnection; 429 } 430 while (TAILQ_EMPTY(&head->so_comp) && head->so_error == 0) { 431 if (head->so_rcv.sb_state & SBS_CANTRCVMORE) { 432 head->so_error = ECONNABORTED; 433 break; 434 } 435 error = msleep(&head->so_timeo, &accept_mtx, PSOCK | PCATCH, 436 "accept", 0); 437 if (error != 0) { 438 ACCEPT_UNLOCK(); 439 goto noconnection; 440 } 441 } 442 if (head->so_error) { 443 error = head->so_error; 444 head->so_error = 0; 445 ACCEPT_UNLOCK(); 446 goto noconnection; 447 } 448 so = TAILQ_FIRST(&head->so_comp); 449 KASSERT(!(so->so_qstate & SQ_INCOMP), ("accept1: so SQ_INCOMP")); 450 KASSERT(so->so_qstate & SQ_COMP, ("accept1: so not SQ_COMP")); 451 452 /* 453 * Before changing the flags on the socket, we have to bump the 454 * reference count. Otherwise, if the protocol calls sofree(), 455 * the socket will be released due to a zero refcount. 456 */ 457 SOCK_LOCK(so); /* soref() and so_state update */ 458 soref(so); /* file descriptor reference */ 459 460 TAILQ_REMOVE(&head->so_comp, so, so_list); 461 head->so_qlen--; 462 if (flags & ACCEPT4_INHERIT) 463 so->so_state |= (head->so_state & SS_NBIO); 464 else 465 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 466 so->so_qstate &= ~SQ_COMP; 467 so->so_head = NULL; 468 469 SOCK_UNLOCK(so); 470 ACCEPT_UNLOCK(); 471 472 /* An extra reference on `nfp' has been held for us by falloc(). */ 473 td->td_retval[0] = fd; 474 475 /* connection has been removed from the listen queue */ 476 KNOTE_UNLOCKED(&head->so_rcv.sb_sel.si_note, 0); 477 478 if (flags & ACCEPT4_INHERIT) { 479 pgid = fgetown(&head->so_sigio); 480 if (pgid != 0) 481 fsetown(pgid, &so->so_sigio); 482 } else { 483 fflag &= ~(FNONBLOCK | FASYNC); 484 if (flags & SOCK_NONBLOCK) 485 fflag |= FNONBLOCK; 486 } 487 488 finit(nfp, fflag, DTYPE_SOCKET, so, &socketops); 489 /* Sync socket nonblocking/async state with file flags */ 490 tmp = fflag & FNONBLOCK; 491 (void) fo_ioctl(nfp, FIONBIO, &tmp, td->td_ucred, td); 492 tmp = fflag & FASYNC; 493 (void) fo_ioctl(nfp, FIOASYNC, &tmp, td->td_ucred, td); 494 sa = 0; 495 error = soaccept(so, &sa); 496 if (error != 0) 497 goto noconnection; 498 if (sa == NULL) { 499 if (name) 500 *namelen = 0; 501 goto done; 502 } 503 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, sa); 504 if (name) { 505 /* check sa_len before it is destroyed */ 506 if (*namelen > sa->sa_len) 507 *namelen = sa->sa_len; 508 #ifdef KTRACE 509 if (KTRPOINT(td, KTR_STRUCT)) 510 ktrsockaddr(sa); 511 #endif 512 *name = sa; 513 sa = NULL; 514 } 515 noconnection: 516 free(sa, M_SONAME); 517 518 /* 519 * close the new descriptor, assuming someone hasn't ripped it 520 * out from under us. 521 */ 522 if (error != 0) 523 fdclose(fdp, nfp, fd, td); 524 525 /* 526 * Release explicitly held references before returning. We return 527 * a reference on nfp to the caller on success if they request it. 528 */ 529 done: 530 if (fp != NULL) { 531 if (error == 0) { 532 *fp = nfp; 533 nfp = NULL; 534 } else 535 *fp = NULL; 536 } 537 if (nfp != NULL) 538 fdrop(nfp, td); 539 fdrop(headfp, td); 540 return (error); 541 } 542 543 int 544 sys_accept(td, uap) 545 struct thread *td; 546 struct accept_args *uap; 547 { 548 549 return (accept1(td, uap->s, uap->name, uap->anamelen, ACCEPT4_INHERIT)); 550 } 551 552 int 553 sys_accept4(td, uap) 554 struct thread *td; 555 struct accept4_args *uap; 556 { 557 558 if (uap->flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) 559 return (EINVAL); 560 561 return (accept1(td, uap->s, uap->name, uap->anamelen, uap->flags)); 562 } 563 564 #ifdef COMPAT_OLDSOCK 565 int 566 oaccept(td, uap) 567 struct thread *td; 568 struct accept_args *uap; 569 { 570 571 return (accept1(td, uap->s, uap->name, uap->anamelen, 572 ACCEPT4_INHERIT | ACCEPT4_COMPAT)); 573 } 574 #endif /* COMPAT_OLDSOCK */ 575 576 /* ARGSUSED */ 577 int 578 sys_connect(td, uap) 579 struct thread *td; 580 struct connect_args /* { 581 int s; 582 caddr_t name; 583 int namelen; 584 } */ *uap; 585 { 586 struct sockaddr *sa; 587 int error; 588 589 error = getsockaddr(&sa, uap->name, uap->namelen); 590 if (error == 0) { 591 error = kern_connectat(td, AT_FDCWD, uap->s, sa); 592 free(sa, M_SONAME); 593 } 594 return (error); 595 } 596 597 int 598 kern_connectat(struct thread *td, int dirfd, int fd, struct sockaddr *sa) 599 { 600 struct socket *so; 601 struct file *fp; 602 cap_rights_t rights; 603 int error, interrupted = 0; 604 605 AUDIT_ARG_FD(fd); 606 AUDIT_ARG_SOCKADDR(td, dirfd, sa); 607 error = getsock_cap(td->td_proc->p_fd, fd, 608 cap_rights_init(&rights, CAP_CONNECT), &fp, NULL); 609 if (error != 0) 610 return (error); 611 so = fp->f_data; 612 if (so->so_state & SS_ISCONNECTING) { 613 error = EALREADY; 614 goto done1; 615 } 616 #ifdef KTRACE 617 if (KTRPOINT(td, KTR_STRUCT)) 618 ktrsockaddr(sa); 619 #endif 620 #ifdef MAC 621 error = mac_socket_check_connect(td->td_ucred, so, sa); 622 if (error != 0) 623 goto bad; 624 #endif 625 if (dirfd == AT_FDCWD) 626 error = soconnect(so, sa, td); 627 else 628 error = soconnectat(dirfd, so, sa, td); 629 if (error != 0) 630 goto bad; 631 if ((so->so_state & SS_NBIO) && (so->so_state & SS_ISCONNECTING)) { 632 error = EINPROGRESS; 633 goto done1; 634 } 635 SOCK_LOCK(so); 636 while ((so->so_state & SS_ISCONNECTING) && so->so_error == 0) { 637 error = msleep(&so->so_timeo, SOCK_MTX(so), PSOCK | PCATCH, 638 "connec", 0); 639 if (error != 0) { 640 if (error == EINTR || error == ERESTART) 641 interrupted = 1; 642 break; 643 } 644 } 645 if (error == 0) { 646 error = so->so_error; 647 so->so_error = 0; 648 } 649 SOCK_UNLOCK(so); 650 bad: 651 if (!interrupted) 652 so->so_state &= ~SS_ISCONNECTING; 653 if (error == ERESTART) 654 error = EINTR; 655 done1: 656 fdrop(fp, td); 657 return (error); 658 } 659 660 /* ARGSUSED */ 661 int 662 sys_connectat(td, uap) 663 struct thread *td; 664 struct connectat_args /* { 665 int fd; 666 int s; 667 caddr_t name; 668 int namelen; 669 } */ *uap; 670 { 671 struct sockaddr *sa; 672 int error; 673 674 error = getsockaddr(&sa, uap->name, uap->namelen); 675 if (error == 0) { 676 error = kern_connectat(td, uap->fd, uap->s, sa); 677 free(sa, M_SONAME); 678 } 679 return (error); 680 } 681 682 int 683 kern_socketpair(struct thread *td, int domain, int type, int protocol, 684 int *rsv) 685 { 686 struct filedesc *fdp = td->td_proc->p_fd; 687 struct file *fp1, *fp2; 688 struct socket *so1, *so2; 689 int fd, error, oflag, fflag; 690 691 AUDIT_ARG_SOCKET(domain, type, protocol); 692 693 oflag = 0; 694 fflag = 0; 695 if ((type & SOCK_CLOEXEC) != 0) { 696 type &= ~SOCK_CLOEXEC; 697 oflag |= O_CLOEXEC; 698 } 699 if ((type & SOCK_NONBLOCK) != 0) { 700 type &= ~SOCK_NONBLOCK; 701 fflag |= FNONBLOCK; 702 } 703 #ifdef MAC 704 /* We might want to have a separate check for socket pairs. */ 705 error = mac_socket_check_create(td->td_ucred, domain, type, 706 protocol); 707 if (error != 0) 708 return (error); 709 #endif 710 error = socreate(domain, &so1, type, protocol, td->td_ucred, td); 711 if (error != 0) 712 return (error); 713 error = socreate(domain, &so2, type, protocol, td->td_ucred, td); 714 if (error != 0) 715 goto free1; 716 /* On success extra reference to `fp1' and 'fp2' is set by falloc. */ 717 error = falloc(td, &fp1, &fd, oflag); 718 if (error != 0) 719 goto free2; 720 rsv[0] = fd; 721 fp1->f_data = so1; /* so1 already has ref count */ 722 error = falloc(td, &fp2, &fd, oflag); 723 if (error != 0) 724 goto free3; 725 fp2->f_data = so2; /* so2 already has ref count */ 726 rsv[1] = fd; 727 error = soconnect2(so1, so2); 728 if (error != 0) 729 goto free4; 730 if (type == SOCK_DGRAM) { 731 /* 732 * Datagram socket connection is asymmetric. 733 */ 734 error = soconnect2(so2, so1); 735 if (error != 0) 736 goto free4; 737 } 738 finit(fp1, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp1->f_data, 739 &socketops); 740 finit(fp2, FREAD | FWRITE | fflag, DTYPE_SOCKET, fp2->f_data, 741 &socketops); 742 if ((fflag & FNONBLOCK) != 0) { 743 (void) fo_ioctl(fp1, FIONBIO, &fflag, td->td_ucred, td); 744 (void) fo_ioctl(fp2, FIONBIO, &fflag, td->td_ucred, td); 745 } 746 fdrop(fp1, td); 747 fdrop(fp2, td); 748 return (0); 749 free4: 750 fdclose(fdp, fp2, rsv[1], td); 751 fdrop(fp2, td); 752 free3: 753 fdclose(fdp, fp1, rsv[0], td); 754 fdrop(fp1, td); 755 free2: 756 if (so2 != NULL) 757 (void)soclose(so2); 758 free1: 759 if (so1 != NULL) 760 (void)soclose(so1); 761 return (error); 762 } 763 764 int 765 sys_socketpair(struct thread *td, struct socketpair_args *uap) 766 { 767 int error, sv[2]; 768 769 error = kern_socketpair(td, uap->domain, uap->type, 770 uap->protocol, sv); 771 if (error != 0) 772 return (error); 773 error = copyout(sv, uap->rsv, 2 * sizeof(int)); 774 if (error != 0) { 775 (void)kern_close(td, sv[0]); 776 (void)kern_close(td, sv[1]); 777 } 778 return (error); 779 } 780 781 static int 782 sendit(td, s, mp, flags) 783 struct thread *td; 784 int s; 785 struct msghdr *mp; 786 int flags; 787 { 788 struct mbuf *control; 789 struct sockaddr *to; 790 int error; 791 792 #ifdef CAPABILITY_MODE 793 if (IN_CAPABILITY_MODE(td) && (mp->msg_name != NULL)) 794 return (ECAPMODE); 795 #endif 796 797 if (mp->msg_name != NULL) { 798 error = getsockaddr(&to, mp->msg_name, mp->msg_namelen); 799 if (error != 0) { 800 to = NULL; 801 goto bad; 802 } 803 mp->msg_name = to; 804 } else { 805 to = NULL; 806 } 807 808 if (mp->msg_control) { 809 if (mp->msg_controllen < sizeof(struct cmsghdr) 810 #ifdef COMPAT_OLDSOCK 811 && mp->msg_flags != MSG_COMPAT 812 #endif 813 ) { 814 error = EINVAL; 815 goto bad; 816 } 817 error = sockargs(&control, mp->msg_control, 818 mp->msg_controllen, MT_CONTROL); 819 if (error != 0) 820 goto bad; 821 #ifdef COMPAT_OLDSOCK 822 if (mp->msg_flags == MSG_COMPAT) { 823 struct cmsghdr *cm; 824 825 M_PREPEND(control, sizeof(*cm), M_WAITOK); 826 cm = mtod(control, struct cmsghdr *); 827 cm->cmsg_len = control->m_len; 828 cm->cmsg_level = SOL_SOCKET; 829 cm->cmsg_type = SCM_RIGHTS; 830 } 831 #endif 832 } else { 833 control = NULL; 834 } 835 836 error = kern_sendit(td, s, mp, flags, control, UIO_USERSPACE); 837 838 bad: 839 free(to, M_SONAME); 840 return (error); 841 } 842 843 int 844 kern_sendit(td, s, mp, flags, control, segflg) 845 struct thread *td; 846 int s; 847 struct msghdr *mp; 848 int flags; 849 struct mbuf *control; 850 enum uio_seg segflg; 851 { 852 struct file *fp; 853 struct uio auio; 854 struct iovec *iov; 855 struct socket *so; 856 cap_rights_t rights; 857 #ifdef KTRACE 858 struct uio *ktruio = NULL; 859 #endif 860 ssize_t len; 861 int i, error; 862 863 AUDIT_ARG_FD(s); 864 cap_rights_init(&rights, CAP_SEND); 865 if (mp->msg_name != NULL) { 866 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, mp->msg_name); 867 cap_rights_set(&rights, CAP_CONNECT); 868 } 869 error = getsock_cap(td->td_proc->p_fd, s, &rights, &fp, NULL); 870 if (error != 0) 871 return (error); 872 so = (struct socket *)fp->f_data; 873 874 #ifdef KTRACE 875 if (mp->msg_name != NULL && KTRPOINT(td, KTR_STRUCT)) 876 ktrsockaddr(mp->msg_name); 877 #endif 878 #ifdef MAC 879 if (mp->msg_name != NULL) { 880 error = mac_socket_check_connect(td->td_ucred, so, 881 mp->msg_name); 882 if (error != 0) 883 goto bad; 884 } 885 error = mac_socket_check_send(td->td_ucred, so); 886 if (error != 0) 887 goto bad; 888 #endif 889 890 auio.uio_iov = mp->msg_iov; 891 auio.uio_iovcnt = mp->msg_iovlen; 892 auio.uio_segflg = segflg; 893 auio.uio_rw = UIO_WRITE; 894 auio.uio_td = td; 895 auio.uio_offset = 0; /* XXX */ 896 auio.uio_resid = 0; 897 iov = mp->msg_iov; 898 for (i = 0; i < mp->msg_iovlen; i++, iov++) { 899 if ((auio.uio_resid += iov->iov_len) < 0) { 900 error = EINVAL; 901 goto bad; 902 } 903 } 904 #ifdef KTRACE 905 if (KTRPOINT(td, KTR_GENIO)) 906 ktruio = cloneuio(&auio); 907 #endif 908 len = auio.uio_resid; 909 error = sosend(so, mp->msg_name, &auio, 0, control, flags, td); 910 if (error != 0) { 911 if (auio.uio_resid != len && (error == ERESTART || 912 error == EINTR || error == EWOULDBLOCK)) 913 error = 0; 914 /* Generation of SIGPIPE can be controlled per socket */ 915 if (error == EPIPE && !(so->so_options & SO_NOSIGPIPE) && 916 !(flags & MSG_NOSIGNAL)) { 917 PROC_LOCK(td->td_proc); 918 tdsignal(td, SIGPIPE); 919 PROC_UNLOCK(td->td_proc); 920 } 921 } 922 if (error == 0) 923 td->td_retval[0] = len - auio.uio_resid; 924 #ifdef KTRACE 925 if (ktruio != NULL) { 926 ktruio->uio_resid = td->td_retval[0]; 927 ktrgenio(s, UIO_WRITE, ktruio, error); 928 } 929 #endif 930 bad: 931 fdrop(fp, td); 932 return (error); 933 } 934 935 int 936 sys_sendto(td, uap) 937 struct thread *td; 938 struct sendto_args /* { 939 int s; 940 caddr_t buf; 941 size_t len; 942 int flags; 943 caddr_t to; 944 int tolen; 945 } */ *uap; 946 { 947 struct msghdr msg; 948 struct iovec aiov; 949 950 msg.msg_name = uap->to; 951 msg.msg_namelen = uap->tolen; 952 msg.msg_iov = &aiov; 953 msg.msg_iovlen = 1; 954 msg.msg_control = 0; 955 #ifdef COMPAT_OLDSOCK 956 msg.msg_flags = 0; 957 #endif 958 aiov.iov_base = uap->buf; 959 aiov.iov_len = uap->len; 960 return (sendit(td, uap->s, &msg, uap->flags)); 961 } 962 963 #ifdef COMPAT_OLDSOCK 964 int 965 osend(td, uap) 966 struct thread *td; 967 struct osend_args /* { 968 int s; 969 caddr_t buf; 970 int len; 971 int flags; 972 } */ *uap; 973 { 974 struct msghdr msg; 975 struct iovec aiov; 976 977 msg.msg_name = 0; 978 msg.msg_namelen = 0; 979 msg.msg_iov = &aiov; 980 msg.msg_iovlen = 1; 981 aiov.iov_base = uap->buf; 982 aiov.iov_len = uap->len; 983 msg.msg_control = 0; 984 msg.msg_flags = 0; 985 return (sendit(td, uap->s, &msg, uap->flags)); 986 } 987 988 int 989 osendmsg(td, uap) 990 struct thread *td; 991 struct osendmsg_args /* { 992 int s; 993 caddr_t msg; 994 int flags; 995 } */ *uap; 996 { 997 struct msghdr msg; 998 struct iovec *iov; 999 int error; 1000 1001 error = copyin(uap->msg, &msg, sizeof (struct omsghdr)); 1002 if (error != 0) 1003 return (error); 1004 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1005 if (error != 0) 1006 return (error); 1007 msg.msg_iov = iov; 1008 msg.msg_flags = MSG_COMPAT; 1009 error = sendit(td, uap->s, &msg, uap->flags); 1010 free(iov, M_IOV); 1011 return (error); 1012 } 1013 #endif 1014 1015 int 1016 sys_sendmsg(td, uap) 1017 struct thread *td; 1018 struct sendmsg_args /* { 1019 int s; 1020 caddr_t msg; 1021 int flags; 1022 } */ *uap; 1023 { 1024 struct msghdr msg; 1025 struct iovec *iov; 1026 int error; 1027 1028 error = copyin(uap->msg, &msg, sizeof (msg)); 1029 if (error != 0) 1030 return (error); 1031 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1032 if (error != 0) 1033 return (error); 1034 msg.msg_iov = iov; 1035 #ifdef COMPAT_OLDSOCK 1036 msg.msg_flags = 0; 1037 #endif 1038 error = sendit(td, uap->s, &msg, uap->flags); 1039 free(iov, M_IOV); 1040 return (error); 1041 } 1042 1043 int 1044 kern_recvit(td, s, mp, fromseg, controlp) 1045 struct thread *td; 1046 int s; 1047 struct msghdr *mp; 1048 enum uio_seg fromseg; 1049 struct mbuf **controlp; 1050 { 1051 struct uio auio; 1052 struct iovec *iov; 1053 struct mbuf *m, *control = NULL; 1054 caddr_t ctlbuf; 1055 struct file *fp; 1056 struct socket *so; 1057 struct sockaddr *fromsa = NULL; 1058 cap_rights_t rights; 1059 #ifdef KTRACE 1060 struct uio *ktruio = NULL; 1061 #endif 1062 ssize_t len; 1063 int error, i; 1064 1065 if (controlp != NULL) 1066 *controlp = NULL; 1067 1068 AUDIT_ARG_FD(s); 1069 error = getsock_cap(td->td_proc->p_fd, s, 1070 cap_rights_init(&rights, CAP_RECV), &fp, NULL); 1071 if (error != 0) 1072 return (error); 1073 so = fp->f_data; 1074 1075 #ifdef MAC 1076 error = mac_socket_check_receive(td->td_ucred, so); 1077 if (error != 0) { 1078 fdrop(fp, td); 1079 return (error); 1080 } 1081 #endif 1082 1083 auio.uio_iov = mp->msg_iov; 1084 auio.uio_iovcnt = mp->msg_iovlen; 1085 auio.uio_segflg = UIO_USERSPACE; 1086 auio.uio_rw = UIO_READ; 1087 auio.uio_td = td; 1088 auio.uio_offset = 0; /* XXX */ 1089 auio.uio_resid = 0; 1090 iov = mp->msg_iov; 1091 for (i = 0; i < mp->msg_iovlen; i++, iov++) { 1092 if ((auio.uio_resid += iov->iov_len) < 0) { 1093 fdrop(fp, td); 1094 return (EINVAL); 1095 } 1096 } 1097 #ifdef KTRACE 1098 if (KTRPOINT(td, KTR_GENIO)) 1099 ktruio = cloneuio(&auio); 1100 #endif 1101 len = auio.uio_resid; 1102 error = soreceive(so, &fromsa, &auio, NULL, 1103 (mp->msg_control || controlp) ? &control : NULL, 1104 &mp->msg_flags); 1105 if (error != 0) { 1106 if (auio.uio_resid != len && (error == ERESTART || 1107 error == EINTR || error == EWOULDBLOCK)) 1108 error = 0; 1109 } 1110 if (fromsa != NULL) 1111 AUDIT_ARG_SOCKADDR(td, AT_FDCWD, fromsa); 1112 #ifdef KTRACE 1113 if (ktruio != NULL) { 1114 ktruio->uio_resid = len - auio.uio_resid; 1115 ktrgenio(s, UIO_READ, ktruio, error); 1116 } 1117 #endif 1118 if (error != 0) 1119 goto out; 1120 td->td_retval[0] = len - auio.uio_resid; 1121 if (mp->msg_name) { 1122 len = mp->msg_namelen; 1123 if (len <= 0 || fromsa == NULL) 1124 len = 0; 1125 else { 1126 /* save sa_len before it is destroyed by MSG_COMPAT */ 1127 len = MIN(len, fromsa->sa_len); 1128 #ifdef COMPAT_OLDSOCK 1129 if (mp->msg_flags & MSG_COMPAT) 1130 ((struct osockaddr *)fromsa)->sa_family = 1131 fromsa->sa_family; 1132 #endif 1133 if (fromseg == UIO_USERSPACE) { 1134 error = copyout(fromsa, mp->msg_name, 1135 (unsigned)len); 1136 if (error != 0) 1137 goto out; 1138 } else 1139 bcopy(fromsa, mp->msg_name, len); 1140 } 1141 mp->msg_namelen = len; 1142 } 1143 if (mp->msg_control && controlp == NULL) { 1144 #ifdef COMPAT_OLDSOCK 1145 /* 1146 * We assume that old recvmsg calls won't receive access 1147 * rights and other control info, esp. as control info 1148 * is always optional and those options didn't exist in 4.3. 1149 * If we receive rights, trim the cmsghdr; anything else 1150 * is tossed. 1151 */ 1152 if (control && mp->msg_flags & MSG_COMPAT) { 1153 if (mtod(control, struct cmsghdr *)->cmsg_level != 1154 SOL_SOCKET || 1155 mtod(control, struct cmsghdr *)->cmsg_type != 1156 SCM_RIGHTS) { 1157 mp->msg_controllen = 0; 1158 goto out; 1159 } 1160 control->m_len -= sizeof (struct cmsghdr); 1161 control->m_data += sizeof (struct cmsghdr); 1162 } 1163 #endif 1164 len = mp->msg_controllen; 1165 m = control; 1166 mp->msg_controllen = 0; 1167 ctlbuf = mp->msg_control; 1168 1169 while (m && len > 0) { 1170 unsigned int tocopy; 1171 1172 if (len >= m->m_len) 1173 tocopy = m->m_len; 1174 else { 1175 mp->msg_flags |= MSG_CTRUNC; 1176 tocopy = len; 1177 } 1178 1179 if ((error = copyout(mtod(m, caddr_t), 1180 ctlbuf, tocopy)) != 0) 1181 goto out; 1182 1183 ctlbuf += tocopy; 1184 len -= tocopy; 1185 m = m->m_next; 1186 } 1187 mp->msg_controllen = ctlbuf - (caddr_t)mp->msg_control; 1188 } 1189 out: 1190 fdrop(fp, td); 1191 #ifdef KTRACE 1192 if (fromsa && KTRPOINT(td, KTR_STRUCT)) 1193 ktrsockaddr(fromsa); 1194 #endif 1195 free(fromsa, M_SONAME); 1196 1197 if (error == 0 && controlp != NULL) 1198 *controlp = control; 1199 else if (control) 1200 m_freem(control); 1201 1202 return (error); 1203 } 1204 1205 static int 1206 recvit(td, s, mp, namelenp) 1207 struct thread *td; 1208 int s; 1209 struct msghdr *mp; 1210 void *namelenp; 1211 { 1212 int error; 1213 1214 error = kern_recvit(td, s, mp, UIO_USERSPACE, NULL); 1215 if (error != 0) 1216 return (error); 1217 if (namelenp != NULL) { 1218 error = copyout(&mp->msg_namelen, namelenp, sizeof (socklen_t)); 1219 #ifdef COMPAT_OLDSOCK 1220 if (mp->msg_flags & MSG_COMPAT) 1221 error = 0; /* old recvfrom didn't check */ 1222 #endif 1223 } 1224 return (error); 1225 } 1226 1227 int 1228 sys_recvfrom(td, uap) 1229 struct thread *td; 1230 struct recvfrom_args /* { 1231 int s; 1232 caddr_t buf; 1233 size_t len; 1234 int flags; 1235 struct sockaddr * __restrict from; 1236 socklen_t * __restrict fromlenaddr; 1237 } */ *uap; 1238 { 1239 struct msghdr msg; 1240 struct iovec aiov; 1241 int error; 1242 1243 if (uap->fromlenaddr) { 1244 error = copyin(uap->fromlenaddr, 1245 &msg.msg_namelen, sizeof (msg.msg_namelen)); 1246 if (error != 0) 1247 goto done2; 1248 } else { 1249 msg.msg_namelen = 0; 1250 } 1251 msg.msg_name = uap->from; 1252 msg.msg_iov = &aiov; 1253 msg.msg_iovlen = 1; 1254 aiov.iov_base = uap->buf; 1255 aiov.iov_len = uap->len; 1256 msg.msg_control = 0; 1257 msg.msg_flags = uap->flags; 1258 error = recvit(td, uap->s, &msg, uap->fromlenaddr); 1259 done2: 1260 return (error); 1261 } 1262 1263 #ifdef COMPAT_OLDSOCK 1264 int 1265 orecvfrom(td, uap) 1266 struct thread *td; 1267 struct recvfrom_args *uap; 1268 { 1269 1270 uap->flags |= MSG_COMPAT; 1271 return (sys_recvfrom(td, uap)); 1272 } 1273 #endif 1274 1275 #ifdef COMPAT_OLDSOCK 1276 int 1277 orecv(td, uap) 1278 struct thread *td; 1279 struct orecv_args /* { 1280 int s; 1281 caddr_t buf; 1282 int len; 1283 int flags; 1284 } */ *uap; 1285 { 1286 struct msghdr msg; 1287 struct iovec aiov; 1288 1289 msg.msg_name = 0; 1290 msg.msg_namelen = 0; 1291 msg.msg_iov = &aiov; 1292 msg.msg_iovlen = 1; 1293 aiov.iov_base = uap->buf; 1294 aiov.iov_len = uap->len; 1295 msg.msg_control = 0; 1296 msg.msg_flags = uap->flags; 1297 return (recvit(td, uap->s, &msg, NULL)); 1298 } 1299 1300 /* 1301 * Old recvmsg. This code takes advantage of the fact that the old msghdr 1302 * overlays the new one, missing only the flags, and with the (old) access 1303 * rights where the control fields are now. 1304 */ 1305 int 1306 orecvmsg(td, uap) 1307 struct thread *td; 1308 struct orecvmsg_args /* { 1309 int s; 1310 struct omsghdr *msg; 1311 int flags; 1312 } */ *uap; 1313 { 1314 struct msghdr msg; 1315 struct iovec *iov; 1316 int error; 1317 1318 error = copyin(uap->msg, &msg, sizeof (struct omsghdr)); 1319 if (error != 0) 1320 return (error); 1321 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1322 if (error != 0) 1323 return (error); 1324 msg.msg_flags = uap->flags | MSG_COMPAT; 1325 msg.msg_iov = iov; 1326 error = recvit(td, uap->s, &msg, &uap->msg->msg_namelen); 1327 if (msg.msg_controllen && error == 0) 1328 error = copyout(&msg.msg_controllen, 1329 &uap->msg->msg_accrightslen, sizeof (int)); 1330 free(iov, M_IOV); 1331 return (error); 1332 } 1333 #endif 1334 1335 int 1336 sys_recvmsg(td, uap) 1337 struct thread *td; 1338 struct recvmsg_args /* { 1339 int s; 1340 struct msghdr *msg; 1341 int flags; 1342 } */ *uap; 1343 { 1344 struct msghdr msg; 1345 struct iovec *uiov, *iov; 1346 int error; 1347 1348 error = copyin(uap->msg, &msg, sizeof (msg)); 1349 if (error != 0) 1350 return (error); 1351 error = copyiniov(msg.msg_iov, msg.msg_iovlen, &iov, EMSGSIZE); 1352 if (error != 0) 1353 return (error); 1354 msg.msg_flags = uap->flags; 1355 #ifdef COMPAT_OLDSOCK 1356 msg.msg_flags &= ~MSG_COMPAT; 1357 #endif 1358 uiov = msg.msg_iov; 1359 msg.msg_iov = iov; 1360 error = recvit(td, uap->s, &msg, NULL); 1361 if (error == 0) { 1362 msg.msg_iov = uiov; 1363 error = copyout(&msg, uap->msg, sizeof(msg)); 1364 } 1365 free(iov, M_IOV); 1366 return (error); 1367 } 1368 1369 /* ARGSUSED */ 1370 int 1371 sys_shutdown(td, uap) 1372 struct thread *td; 1373 struct shutdown_args /* { 1374 int s; 1375 int how; 1376 } */ *uap; 1377 { 1378 struct socket *so; 1379 struct file *fp; 1380 cap_rights_t rights; 1381 int error; 1382 1383 AUDIT_ARG_FD(uap->s); 1384 error = getsock_cap(td->td_proc->p_fd, uap->s, 1385 cap_rights_init(&rights, CAP_SHUTDOWN), &fp, NULL); 1386 if (error == 0) { 1387 so = fp->f_data; 1388 error = soshutdown(so, uap->how); 1389 fdrop(fp, td); 1390 } 1391 return (error); 1392 } 1393 1394 /* ARGSUSED */ 1395 int 1396 sys_setsockopt(td, uap) 1397 struct thread *td; 1398 struct setsockopt_args /* { 1399 int s; 1400 int level; 1401 int name; 1402 caddr_t val; 1403 int valsize; 1404 } */ *uap; 1405 { 1406 1407 return (kern_setsockopt(td, uap->s, uap->level, uap->name, 1408 uap->val, UIO_USERSPACE, uap->valsize)); 1409 } 1410 1411 int 1412 kern_setsockopt(td, s, level, name, val, valseg, valsize) 1413 struct thread *td; 1414 int s; 1415 int level; 1416 int name; 1417 void *val; 1418 enum uio_seg valseg; 1419 socklen_t valsize; 1420 { 1421 struct socket *so; 1422 struct file *fp; 1423 struct sockopt sopt; 1424 cap_rights_t rights; 1425 int error; 1426 1427 if (val == NULL && valsize != 0) 1428 return (EFAULT); 1429 if ((int)valsize < 0) 1430 return (EINVAL); 1431 1432 sopt.sopt_dir = SOPT_SET; 1433 sopt.sopt_level = level; 1434 sopt.sopt_name = name; 1435 sopt.sopt_val = val; 1436 sopt.sopt_valsize = valsize; 1437 switch (valseg) { 1438 case UIO_USERSPACE: 1439 sopt.sopt_td = td; 1440 break; 1441 case UIO_SYSSPACE: 1442 sopt.sopt_td = NULL; 1443 break; 1444 default: 1445 panic("kern_setsockopt called with bad valseg"); 1446 } 1447 1448 AUDIT_ARG_FD(s); 1449 error = getsock_cap(td->td_proc->p_fd, s, 1450 cap_rights_init(&rights, CAP_SETSOCKOPT), &fp, NULL); 1451 if (error == 0) { 1452 so = fp->f_data; 1453 error = sosetopt(so, &sopt); 1454 fdrop(fp, td); 1455 } 1456 return(error); 1457 } 1458 1459 /* ARGSUSED */ 1460 int 1461 sys_getsockopt(td, uap) 1462 struct thread *td; 1463 struct getsockopt_args /* { 1464 int s; 1465 int level; 1466 int name; 1467 void * __restrict val; 1468 socklen_t * __restrict avalsize; 1469 } */ *uap; 1470 { 1471 socklen_t valsize; 1472 int error; 1473 1474 if (uap->val) { 1475 error = copyin(uap->avalsize, &valsize, sizeof (valsize)); 1476 if (error != 0) 1477 return (error); 1478 } 1479 1480 error = kern_getsockopt(td, uap->s, uap->level, uap->name, 1481 uap->val, UIO_USERSPACE, &valsize); 1482 1483 if (error == 0) 1484 error = copyout(&valsize, uap->avalsize, sizeof (valsize)); 1485 return (error); 1486 } 1487 1488 /* 1489 * Kernel version of getsockopt. 1490 * optval can be a userland or userspace. optlen is always a kernel pointer. 1491 */ 1492 int 1493 kern_getsockopt(td, s, level, name, val, valseg, valsize) 1494 struct thread *td; 1495 int s; 1496 int level; 1497 int name; 1498 void *val; 1499 enum uio_seg valseg; 1500 socklen_t *valsize; 1501 { 1502 struct socket *so; 1503 struct file *fp; 1504 struct sockopt sopt; 1505 cap_rights_t rights; 1506 int error; 1507 1508 if (val == NULL) 1509 *valsize = 0; 1510 if ((int)*valsize < 0) 1511 return (EINVAL); 1512 1513 sopt.sopt_dir = SOPT_GET; 1514 sopt.sopt_level = level; 1515 sopt.sopt_name = name; 1516 sopt.sopt_val = val; 1517 sopt.sopt_valsize = (size_t)*valsize; /* checked non-negative above */ 1518 switch (valseg) { 1519 case UIO_USERSPACE: 1520 sopt.sopt_td = td; 1521 break; 1522 case UIO_SYSSPACE: 1523 sopt.sopt_td = NULL; 1524 break; 1525 default: 1526 panic("kern_getsockopt called with bad valseg"); 1527 } 1528 1529 AUDIT_ARG_FD(s); 1530 error = getsock_cap(td->td_proc->p_fd, s, 1531 cap_rights_init(&rights, CAP_GETSOCKOPT), &fp, NULL); 1532 if (error == 0) { 1533 so = fp->f_data; 1534 error = sogetopt(so, &sopt); 1535 *valsize = sopt.sopt_valsize; 1536 fdrop(fp, td); 1537 } 1538 return (error); 1539 } 1540 1541 /* 1542 * getsockname1() - Get socket name. 1543 */ 1544 /* ARGSUSED */ 1545 static int 1546 getsockname1(td, uap, compat) 1547 struct thread *td; 1548 struct getsockname_args /* { 1549 int fdes; 1550 struct sockaddr * __restrict asa; 1551 socklen_t * __restrict alen; 1552 } */ *uap; 1553 int compat; 1554 { 1555 struct sockaddr *sa; 1556 socklen_t len; 1557 int error; 1558 1559 error = copyin(uap->alen, &len, sizeof(len)); 1560 if (error != 0) 1561 return (error); 1562 1563 error = kern_getsockname(td, uap->fdes, &sa, &len); 1564 if (error != 0) 1565 return (error); 1566 1567 if (len != 0) { 1568 #ifdef COMPAT_OLDSOCK 1569 if (compat) 1570 ((struct osockaddr *)sa)->sa_family = sa->sa_family; 1571 #endif 1572 error = copyout(sa, uap->asa, (u_int)len); 1573 } 1574 free(sa, M_SONAME); 1575 if (error == 0) 1576 error = copyout(&len, uap->alen, sizeof(len)); 1577 return (error); 1578 } 1579 1580 int 1581 kern_getsockname(struct thread *td, int fd, struct sockaddr **sa, 1582 socklen_t *alen) 1583 { 1584 struct socket *so; 1585 struct file *fp; 1586 cap_rights_t rights; 1587 socklen_t len; 1588 int error; 1589 1590 AUDIT_ARG_FD(fd); 1591 error = getsock_cap(td->td_proc->p_fd, fd, 1592 cap_rights_init(&rights, CAP_GETSOCKNAME), &fp, NULL); 1593 if (error != 0) 1594 return (error); 1595 so = fp->f_data; 1596 *sa = NULL; 1597 CURVNET_SET(so->so_vnet); 1598 error = (*so->so_proto->pr_usrreqs->pru_sockaddr)(so, sa); 1599 CURVNET_RESTORE(); 1600 if (error != 0) 1601 goto bad; 1602 if (*sa == NULL) 1603 len = 0; 1604 else 1605 len = MIN(*alen, (*sa)->sa_len); 1606 *alen = len; 1607 #ifdef KTRACE 1608 if (KTRPOINT(td, KTR_STRUCT)) 1609 ktrsockaddr(*sa); 1610 #endif 1611 bad: 1612 fdrop(fp, td); 1613 if (error != 0 && *sa != NULL) { 1614 free(*sa, M_SONAME); 1615 *sa = NULL; 1616 } 1617 return (error); 1618 } 1619 1620 int 1621 sys_getsockname(td, uap) 1622 struct thread *td; 1623 struct getsockname_args *uap; 1624 { 1625 1626 return (getsockname1(td, uap, 0)); 1627 } 1628 1629 #ifdef COMPAT_OLDSOCK 1630 int 1631 ogetsockname(td, uap) 1632 struct thread *td; 1633 struct getsockname_args *uap; 1634 { 1635 1636 return (getsockname1(td, uap, 1)); 1637 } 1638 #endif /* COMPAT_OLDSOCK */ 1639 1640 /* 1641 * getpeername1() - Get name of peer for connected socket. 1642 */ 1643 /* ARGSUSED */ 1644 static int 1645 getpeername1(td, uap, compat) 1646 struct thread *td; 1647 struct getpeername_args /* { 1648 int fdes; 1649 struct sockaddr * __restrict asa; 1650 socklen_t * __restrict alen; 1651 } */ *uap; 1652 int compat; 1653 { 1654 struct sockaddr *sa; 1655 socklen_t len; 1656 int error; 1657 1658 error = copyin(uap->alen, &len, sizeof (len)); 1659 if (error != 0) 1660 return (error); 1661 1662 error = kern_getpeername(td, uap->fdes, &sa, &len); 1663 if (error != 0) 1664 return (error); 1665 1666 if (len != 0) { 1667 #ifdef COMPAT_OLDSOCK 1668 if (compat) 1669 ((struct osockaddr *)sa)->sa_family = sa->sa_family; 1670 #endif 1671 error = copyout(sa, uap->asa, (u_int)len); 1672 } 1673 free(sa, M_SONAME); 1674 if (error == 0) 1675 error = copyout(&len, uap->alen, sizeof(len)); 1676 return (error); 1677 } 1678 1679 int 1680 kern_getpeername(struct thread *td, int fd, struct sockaddr **sa, 1681 socklen_t *alen) 1682 { 1683 struct socket *so; 1684 struct file *fp; 1685 cap_rights_t rights; 1686 socklen_t len; 1687 int error; 1688 1689 AUDIT_ARG_FD(fd); 1690 error = getsock_cap(td->td_proc->p_fd, fd, 1691 cap_rights_init(&rights, CAP_GETPEERNAME), &fp, NULL); 1692 if (error != 0) 1693 return (error); 1694 so = fp->f_data; 1695 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONFIRMING)) == 0) { 1696 error = ENOTCONN; 1697 goto done; 1698 } 1699 *sa = NULL; 1700 CURVNET_SET(so->so_vnet); 1701 error = (*so->so_proto->pr_usrreqs->pru_peeraddr)(so, sa); 1702 CURVNET_RESTORE(); 1703 if (error != 0) 1704 goto bad; 1705 if (*sa == NULL) 1706 len = 0; 1707 else 1708 len = MIN(*alen, (*sa)->sa_len); 1709 *alen = len; 1710 #ifdef KTRACE 1711 if (KTRPOINT(td, KTR_STRUCT)) 1712 ktrsockaddr(*sa); 1713 #endif 1714 bad: 1715 if (error != 0 && *sa != NULL) { 1716 free(*sa, M_SONAME); 1717 *sa = NULL; 1718 } 1719 done: 1720 fdrop(fp, td); 1721 return (error); 1722 } 1723 1724 int 1725 sys_getpeername(td, uap) 1726 struct thread *td; 1727 struct getpeername_args *uap; 1728 { 1729 1730 return (getpeername1(td, uap, 0)); 1731 } 1732 1733 #ifdef COMPAT_OLDSOCK 1734 int 1735 ogetpeername(td, uap) 1736 struct thread *td; 1737 struct ogetpeername_args *uap; 1738 { 1739 1740 /* XXX uap should have type `getpeername_args *' to begin with. */ 1741 return (getpeername1(td, (struct getpeername_args *)uap, 1)); 1742 } 1743 #endif /* COMPAT_OLDSOCK */ 1744 1745 int 1746 sockargs(mp, buf, buflen, type) 1747 struct mbuf **mp; 1748 caddr_t buf; 1749 int buflen, type; 1750 { 1751 struct sockaddr *sa; 1752 struct mbuf *m; 1753 int error; 1754 1755 if (buflen > MLEN) { 1756 #ifdef COMPAT_OLDSOCK 1757 if (type == MT_SONAME && buflen <= 112) 1758 buflen = MLEN; /* unix domain compat. hack */ 1759 else 1760 #endif 1761 if (buflen > MCLBYTES) 1762 return (EINVAL); 1763 } 1764 m = m_get2(buflen, M_WAITOK, type, 0); 1765 m->m_len = buflen; 1766 error = copyin(buf, mtod(m, caddr_t), (u_int)buflen); 1767 if (error != 0) 1768 (void) m_free(m); 1769 else { 1770 *mp = m; 1771 if (type == MT_SONAME) { 1772 sa = mtod(m, struct sockaddr *); 1773 1774 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN 1775 if (sa->sa_family == 0 && sa->sa_len < AF_MAX) 1776 sa->sa_family = sa->sa_len; 1777 #endif 1778 sa->sa_len = buflen; 1779 } 1780 } 1781 return (error); 1782 } 1783 1784 int 1785 getsockaddr(namp, uaddr, len) 1786 struct sockaddr **namp; 1787 caddr_t uaddr; 1788 size_t len; 1789 { 1790 struct sockaddr *sa; 1791 int error; 1792 1793 if (len > SOCK_MAXADDRLEN) 1794 return (ENAMETOOLONG); 1795 if (len < offsetof(struct sockaddr, sa_data[0])) 1796 return (EINVAL); 1797 sa = malloc(len, M_SONAME, M_WAITOK); 1798 error = copyin(uaddr, sa, len); 1799 if (error != 0) { 1800 free(sa, M_SONAME); 1801 } else { 1802 #if defined(COMPAT_OLDSOCK) && BYTE_ORDER != BIG_ENDIAN 1803 if (sa->sa_family == 0 && sa->sa_len < AF_MAX) 1804 sa->sa_family = sa->sa_len; 1805 #endif 1806 sa->sa_len = len; 1807 *namp = sa; 1808 } 1809 return (error); 1810 } 1811 1812 struct sendfile_sync { 1813 struct mtx mtx; 1814 struct cv cv; 1815 unsigned count; 1816 }; 1817 1818 /* 1819 * Add more references to a vm_page + sf_buf + sendfile_sync. 1820 */ 1821 void 1822 sf_ext_ref(void *arg1, void *arg2) 1823 { 1824 struct sf_buf *sf = arg1; 1825 struct sendfile_sync *sfs = arg2; 1826 vm_page_t pg = sf_buf_page(sf); 1827 1828 sf_buf_ref(sf); 1829 1830 vm_page_lock(pg); 1831 vm_page_wire(pg); 1832 vm_page_unlock(pg); 1833 1834 if (sfs != NULL) { 1835 mtx_lock(&sfs->mtx); 1836 KASSERT(sfs->count > 0, ("Sendfile sync botchup count == 0")); 1837 sfs->count++; 1838 mtx_unlock(&sfs->mtx); 1839 } 1840 } 1841 1842 /* 1843 * Detach mapped page and release resources back to the system. 1844 */ 1845 void 1846 sf_ext_free(void *arg1, void *arg2) 1847 { 1848 struct sf_buf *sf = arg1; 1849 struct sendfile_sync *sfs = arg2; 1850 vm_page_t pg = sf_buf_page(sf); 1851 1852 sf_buf_free(sf); 1853 1854 vm_page_lock(pg); 1855 vm_page_unwire(pg, PQ_INACTIVE); 1856 /* 1857 * Check for the object going away on us. This can 1858 * happen since we don't hold a reference to it. 1859 * If so, we're responsible for freeing the page. 1860 */ 1861 if (pg->wire_count == 0 && pg->object == NULL) 1862 vm_page_free(pg); 1863 vm_page_unlock(pg); 1864 1865 if (sfs != NULL) { 1866 mtx_lock(&sfs->mtx); 1867 KASSERT(sfs->count > 0, ("Sendfile sync botchup count == 0")); 1868 if (--sfs->count == 0) 1869 cv_signal(&sfs->cv); 1870 mtx_unlock(&sfs->mtx); 1871 } 1872 } 1873 1874 /* 1875 * sendfile(2) 1876 * 1877 * int sendfile(int fd, int s, off_t offset, size_t nbytes, 1878 * struct sf_hdtr *hdtr, off_t *sbytes, int flags) 1879 * 1880 * Send a file specified by 'fd' and starting at 'offset' to a socket 1881 * specified by 's'. Send only 'nbytes' of the file or until EOF if nbytes == 1882 * 0. Optionally add a header and/or trailer to the socket output. If 1883 * specified, write the total number of bytes sent into *sbytes. 1884 */ 1885 int 1886 sys_sendfile(struct thread *td, struct sendfile_args *uap) 1887 { 1888 1889 return (do_sendfile(td, uap, 0)); 1890 } 1891 1892 static int 1893 do_sendfile(struct thread *td, struct sendfile_args *uap, int compat) 1894 { 1895 struct sf_hdtr hdtr; 1896 struct uio *hdr_uio, *trl_uio; 1897 struct file *fp; 1898 cap_rights_t rights; 1899 off_t sbytes; 1900 int error; 1901 1902 /* 1903 * File offset must be positive. If it goes beyond EOF 1904 * we send only the header/trailer and no payload data. 1905 */ 1906 if (uap->offset < 0) 1907 return (EINVAL); 1908 1909 hdr_uio = trl_uio = NULL; 1910 1911 if (uap->hdtr != NULL) { 1912 error = copyin(uap->hdtr, &hdtr, sizeof(hdtr)); 1913 if (error != 0) 1914 goto out; 1915 if (hdtr.headers != NULL) { 1916 error = copyinuio(hdtr.headers, hdtr.hdr_cnt, 1917 &hdr_uio); 1918 if (error != 0) 1919 goto out; 1920 } 1921 if (hdtr.trailers != NULL) { 1922 error = copyinuio(hdtr.trailers, hdtr.trl_cnt, 1923 &trl_uio); 1924 if (error != 0) 1925 goto out; 1926 } 1927 } 1928 1929 AUDIT_ARG_FD(uap->fd); 1930 1931 /* 1932 * sendfile(2) can start at any offset within a file so we require 1933 * CAP_READ+CAP_SEEK = CAP_PREAD. 1934 */ 1935 if ((error = fget_read(td, uap->fd, 1936 cap_rights_init(&rights, CAP_PREAD), &fp)) != 0) { 1937 goto out; 1938 } 1939 1940 error = fo_sendfile(fp, uap->s, hdr_uio, trl_uio, uap->offset, 1941 uap->nbytes, &sbytes, uap->flags, compat ? SFK_COMPAT : 0, td); 1942 fdrop(fp, td); 1943 1944 if (uap->sbytes != NULL) 1945 copyout(&sbytes, uap->sbytes, sizeof(off_t)); 1946 1947 out: 1948 free(hdr_uio, M_IOV); 1949 free(trl_uio, M_IOV); 1950 return (error); 1951 } 1952 1953 #ifdef COMPAT_FREEBSD4 1954 int 1955 freebsd4_sendfile(struct thread *td, struct freebsd4_sendfile_args *uap) 1956 { 1957 struct sendfile_args args; 1958 1959 args.fd = uap->fd; 1960 args.s = uap->s; 1961 args.offset = uap->offset; 1962 args.nbytes = uap->nbytes; 1963 args.hdtr = uap->hdtr; 1964 args.sbytes = uap->sbytes; 1965 args.flags = uap->flags; 1966 1967 return (do_sendfile(td, &args, 1)); 1968 } 1969 #endif /* COMPAT_FREEBSD4 */ 1970 1971 static int 1972 sendfile_readpage(vm_object_t obj, struct vnode *vp, int nd, 1973 off_t off, int xfsize, int bsize, struct thread *td, vm_page_t *res) 1974 { 1975 vm_page_t m; 1976 vm_pindex_t pindex; 1977 ssize_t resid; 1978 int error, readahead, rv; 1979 1980 pindex = OFF_TO_IDX(off); 1981 VM_OBJECT_WLOCK(obj); 1982 m = vm_page_grab(obj, pindex, (vp != NULL ? VM_ALLOC_NOBUSY | 1983 VM_ALLOC_IGN_SBUSY : 0) | VM_ALLOC_WIRED | VM_ALLOC_NORMAL); 1984 1985 /* 1986 * Check if page is valid for what we need, otherwise initiate I/O. 1987 * 1988 * The non-zero nd argument prevents disk I/O, instead we 1989 * return the caller what he specified in nd. In particular, 1990 * if we already turned some pages into mbufs, nd == EAGAIN 1991 * and the main function send them the pages before we come 1992 * here again and block. 1993 */ 1994 if (m->valid != 0 && vm_page_is_valid(m, off & PAGE_MASK, xfsize)) { 1995 if (vp == NULL) 1996 vm_page_xunbusy(m); 1997 VM_OBJECT_WUNLOCK(obj); 1998 *res = m; 1999 return (0); 2000 } else if (nd != 0) { 2001 if (vp == NULL) 2002 vm_page_xunbusy(m); 2003 error = nd; 2004 goto free_page; 2005 } 2006 2007 /* 2008 * Get the page from backing store. 2009 */ 2010 error = 0; 2011 if (vp != NULL) { 2012 VM_OBJECT_WUNLOCK(obj); 2013 readahead = sfreadahead * MAXBSIZE; 2014 2015 /* 2016 * Use vn_rdwr() instead of the pager interface for 2017 * the vnode, to allow the read-ahead. 2018 * 2019 * XXXMAC: Because we don't have fp->f_cred here, we 2020 * pass in NOCRED. This is probably wrong, but is 2021 * consistent with our original implementation. 2022 */ 2023 error = vn_rdwr(UIO_READ, vp, NULL, readahead, trunc_page(off), 2024 UIO_NOCOPY, IO_NODELOCKED | IO_VMIO | ((readahead / 2025 bsize) << IO_SEQSHIFT), td->td_ucred, NOCRED, &resid, td); 2026 SFSTAT_INC(sf_iocnt); 2027 VM_OBJECT_WLOCK(obj); 2028 } else { 2029 if (vm_pager_has_page(obj, pindex, NULL, NULL)) { 2030 rv = vm_pager_get_pages(obj, &m, 1, 0); 2031 SFSTAT_INC(sf_iocnt); 2032 m = vm_page_lookup(obj, pindex); 2033 if (m == NULL) 2034 error = EIO; 2035 else if (rv != VM_PAGER_OK) { 2036 vm_page_lock(m); 2037 vm_page_free(m); 2038 vm_page_unlock(m); 2039 m = NULL; 2040 error = EIO; 2041 } 2042 } else { 2043 pmap_zero_page(m); 2044 m->valid = VM_PAGE_BITS_ALL; 2045 m->dirty = 0; 2046 } 2047 if (m != NULL) 2048 vm_page_xunbusy(m); 2049 } 2050 if (error == 0) { 2051 *res = m; 2052 } else if (m != NULL) { 2053 free_page: 2054 vm_page_lock(m); 2055 vm_page_unwire(m, PQ_INACTIVE); 2056 2057 /* 2058 * See if anyone else might know about this page. If 2059 * not and it is not valid, then free it. 2060 */ 2061 if (m->wire_count == 0 && m->valid == 0 && !vm_page_busied(m)) 2062 vm_page_free(m); 2063 vm_page_unlock(m); 2064 } 2065 KASSERT(error != 0 || (m->wire_count > 0 && 2066 vm_page_is_valid(m, off & PAGE_MASK, xfsize)), 2067 ("wrong page state m %p off %#jx xfsize %d", m, (uintmax_t)off, 2068 xfsize)); 2069 VM_OBJECT_WUNLOCK(obj); 2070 return (error); 2071 } 2072 2073 static int 2074 sendfile_getobj(struct thread *td, struct file *fp, vm_object_t *obj_res, 2075 struct vnode **vp_res, struct shmfd **shmfd_res, off_t *obj_size, 2076 int *bsize) 2077 { 2078 struct vattr va; 2079 vm_object_t obj; 2080 struct vnode *vp; 2081 struct shmfd *shmfd; 2082 int error; 2083 2084 vp = *vp_res = NULL; 2085 obj = NULL; 2086 shmfd = *shmfd_res = NULL; 2087 *bsize = 0; 2088 2089 /* 2090 * The file descriptor must be a regular file and have a 2091 * backing VM object. 2092 */ 2093 if (fp->f_type == DTYPE_VNODE) { 2094 vp = fp->f_vnode; 2095 vn_lock(vp, LK_SHARED | LK_RETRY); 2096 if (vp->v_type != VREG) { 2097 error = EINVAL; 2098 goto out; 2099 } 2100 *bsize = vp->v_mount->mnt_stat.f_iosize; 2101 error = VOP_GETATTR(vp, &va, td->td_ucred); 2102 if (error != 0) 2103 goto out; 2104 *obj_size = va.va_size; 2105 obj = vp->v_object; 2106 if (obj == NULL) { 2107 error = EINVAL; 2108 goto out; 2109 } 2110 } else if (fp->f_type == DTYPE_SHM) { 2111 shmfd = fp->f_data; 2112 obj = shmfd->shm_object; 2113 *obj_size = shmfd->shm_size; 2114 } else { 2115 error = EINVAL; 2116 goto out; 2117 } 2118 2119 VM_OBJECT_WLOCK(obj); 2120 if ((obj->flags & OBJ_DEAD) != 0) { 2121 VM_OBJECT_WUNLOCK(obj); 2122 error = EBADF; 2123 goto out; 2124 } 2125 2126 /* 2127 * Temporarily increase the backing VM object's reference 2128 * count so that a forced reclamation of its vnode does not 2129 * immediately destroy it. 2130 */ 2131 vm_object_reference_locked(obj); 2132 VM_OBJECT_WUNLOCK(obj); 2133 *obj_res = obj; 2134 *vp_res = vp; 2135 *shmfd_res = shmfd; 2136 2137 out: 2138 if (vp != NULL) 2139 VOP_UNLOCK(vp, 0); 2140 return (error); 2141 } 2142 2143 static int 2144 kern_sendfile_getsock(struct thread *td, int s, struct file **sock_fp, 2145 struct socket **so) 2146 { 2147 cap_rights_t rights; 2148 int error; 2149 2150 *sock_fp = NULL; 2151 *so = NULL; 2152 2153 /* 2154 * The socket must be a stream socket and connected. 2155 */ 2156 error = getsock_cap(td->td_proc->p_fd, s, cap_rights_init(&rights, 2157 CAP_SEND), sock_fp, NULL); 2158 if (error != 0) 2159 return (error); 2160 *so = (*sock_fp)->f_data; 2161 if ((*so)->so_type != SOCK_STREAM) 2162 return (EINVAL); 2163 if (((*so)->so_state & SS_ISCONNECTED) == 0) 2164 return (ENOTCONN); 2165 return (0); 2166 } 2167 2168 int 2169 vn_sendfile(struct file *fp, int sockfd, struct uio *hdr_uio, 2170 struct uio *trl_uio, off_t offset, size_t nbytes, off_t *sent, int flags, 2171 int kflags, struct thread *td) 2172 { 2173 struct file *sock_fp; 2174 struct vnode *vp; 2175 struct vm_object *obj; 2176 struct socket *so; 2177 struct mbuf *m; 2178 struct sf_buf *sf; 2179 struct vm_page *pg; 2180 struct shmfd *shmfd; 2181 struct sendfile_sync *sfs; 2182 struct vattr va; 2183 off_t off, xfsize, fsbytes, sbytes, rem, obj_size; 2184 int error, bsize, nd, hdrlen, mnw; 2185 2186 pg = NULL; 2187 obj = NULL; 2188 so = NULL; 2189 m = NULL; 2190 sfs = NULL; 2191 fsbytes = sbytes = 0; 2192 hdrlen = mnw = 0; 2193 rem = nbytes; 2194 obj_size = 0; 2195 2196 error = sendfile_getobj(td, fp, &obj, &vp, &shmfd, &obj_size, &bsize); 2197 if (error != 0) 2198 return (error); 2199 if (rem == 0) 2200 rem = obj_size; 2201 2202 error = kern_sendfile_getsock(td, sockfd, &sock_fp, &so); 2203 if (error != 0) 2204 goto out; 2205 2206 /* 2207 * Do not wait on memory allocations but return ENOMEM for 2208 * caller to retry later. 2209 * XXX: Experimental. 2210 */ 2211 if (flags & SF_MNOWAIT) 2212 mnw = 1; 2213 2214 if (flags & SF_SYNC) { 2215 sfs = malloc(sizeof *sfs, M_TEMP, M_WAITOK | M_ZERO); 2216 mtx_init(&sfs->mtx, "sendfile", NULL, MTX_DEF); 2217 cv_init(&sfs->cv, "sendfile"); 2218 } 2219 2220 #ifdef MAC 2221 error = mac_socket_check_send(td->td_ucred, so); 2222 if (error != 0) 2223 goto out; 2224 #endif 2225 2226 /* If headers are specified copy them into mbufs. */ 2227 if (hdr_uio != NULL) { 2228 hdr_uio->uio_td = td; 2229 hdr_uio->uio_rw = UIO_WRITE; 2230 if (hdr_uio->uio_resid > 0) { 2231 /* 2232 * In FBSD < 5.0 the nbytes to send also included 2233 * the header. If compat is specified subtract the 2234 * header size from nbytes. 2235 */ 2236 if (kflags & SFK_COMPAT) { 2237 if (nbytes > hdr_uio->uio_resid) 2238 nbytes -= hdr_uio->uio_resid; 2239 else 2240 nbytes = 0; 2241 } 2242 m = m_uiotombuf(hdr_uio, (mnw ? M_NOWAIT : M_WAITOK), 2243 0, 0, 0); 2244 if (m == NULL) { 2245 error = mnw ? EAGAIN : ENOBUFS; 2246 goto out; 2247 } 2248 hdrlen = m_length(m, NULL); 2249 } 2250 } 2251 2252 /* 2253 * Protect against multiple writers to the socket. 2254 * 2255 * XXXRW: Historically this has assumed non-interruptibility, so now 2256 * we implement that, but possibly shouldn't. 2257 */ 2258 (void)sblock(&so->so_snd, SBL_WAIT | SBL_NOINTR); 2259 2260 /* 2261 * Loop through the pages of the file, starting with the requested 2262 * offset. Get a file page (do I/O if necessary), map the file page 2263 * into an sf_buf, attach an mbuf header to the sf_buf, and queue 2264 * it on the socket. 2265 * This is done in two loops. The inner loop turns as many pages 2266 * as it can, up to available socket buffer space, without blocking 2267 * into mbufs to have it bulk delivered into the socket send buffer. 2268 * The outer loop checks the state and available space of the socket 2269 * and takes care of the overall progress. 2270 */ 2271 for (off = offset; ; ) { 2272 struct mbuf *mtail; 2273 int loopbytes; 2274 int space; 2275 int done; 2276 2277 if ((nbytes != 0 && nbytes == fsbytes) || 2278 (nbytes == 0 && obj_size == fsbytes)) 2279 break; 2280 2281 mtail = NULL; 2282 loopbytes = 0; 2283 space = 0; 2284 done = 0; 2285 2286 /* 2287 * Check the socket state for ongoing connection, 2288 * no errors and space in socket buffer. 2289 * If space is low allow for the remainder of the 2290 * file to be processed if it fits the socket buffer. 2291 * Otherwise block in waiting for sufficient space 2292 * to proceed, or if the socket is nonblocking, return 2293 * to userland with EAGAIN while reporting how far 2294 * we've come. 2295 * We wait until the socket buffer has significant free 2296 * space to do bulk sends. This makes good use of file 2297 * system read ahead and allows packet segmentation 2298 * offloading hardware to take over lots of work. If 2299 * we were not careful here we would send off only one 2300 * sfbuf at a time. 2301 */ 2302 SOCKBUF_LOCK(&so->so_snd); 2303 if (so->so_snd.sb_lowat < so->so_snd.sb_hiwat / 2) 2304 so->so_snd.sb_lowat = so->so_snd.sb_hiwat / 2; 2305 retry_space: 2306 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2307 error = EPIPE; 2308 SOCKBUF_UNLOCK(&so->so_snd); 2309 goto done; 2310 } else if (so->so_error) { 2311 error = so->so_error; 2312 so->so_error = 0; 2313 SOCKBUF_UNLOCK(&so->so_snd); 2314 goto done; 2315 } 2316 space = sbspace(&so->so_snd); 2317 if (space < rem && 2318 (space <= 0 || 2319 space < so->so_snd.sb_lowat)) { 2320 if (so->so_state & SS_NBIO) { 2321 SOCKBUF_UNLOCK(&so->so_snd); 2322 error = EAGAIN; 2323 goto done; 2324 } 2325 /* 2326 * sbwait drops the lock while sleeping. 2327 * When we loop back to retry_space the 2328 * state may have changed and we retest 2329 * for it. 2330 */ 2331 error = sbwait(&so->so_snd); 2332 /* 2333 * An error from sbwait usually indicates that we've 2334 * been interrupted by a signal. If we've sent anything 2335 * then return bytes sent, otherwise return the error. 2336 */ 2337 if (error != 0) { 2338 SOCKBUF_UNLOCK(&so->so_snd); 2339 goto done; 2340 } 2341 goto retry_space; 2342 } 2343 SOCKBUF_UNLOCK(&so->so_snd); 2344 2345 /* 2346 * Reduce space in the socket buffer by the size of 2347 * the header mbuf chain. 2348 * hdrlen is set to 0 after the first loop. 2349 */ 2350 space -= hdrlen; 2351 2352 if (vp != NULL) { 2353 error = vn_lock(vp, LK_SHARED); 2354 if (error != 0) 2355 goto done; 2356 error = VOP_GETATTR(vp, &va, td->td_ucred); 2357 if (error != 0 || off >= va.va_size) { 2358 VOP_UNLOCK(vp, 0); 2359 goto done; 2360 } 2361 obj_size = va.va_size; 2362 } 2363 2364 /* 2365 * Loop and construct maximum sized mbuf chain to be bulk 2366 * dumped into socket buffer. 2367 */ 2368 while (space > loopbytes) { 2369 vm_offset_t pgoff; 2370 struct mbuf *m0; 2371 2372 /* 2373 * Calculate the amount to transfer. 2374 * Not to exceed a page, the EOF, 2375 * or the passed in nbytes. 2376 */ 2377 pgoff = (vm_offset_t)(off & PAGE_MASK); 2378 rem = obj_size - offset; 2379 if (nbytes != 0) 2380 rem = omin(rem, nbytes); 2381 rem -= fsbytes + loopbytes; 2382 xfsize = omin(PAGE_SIZE - pgoff, rem); 2383 xfsize = omin(space - loopbytes, xfsize); 2384 if (xfsize <= 0) { 2385 done = 1; /* all data sent */ 2386 break; 2387 } 2388 2389 /* 2390 * Attempt to look up the page. Allocate 2391 * if not found or wait and loop if busy. 2392 */ 2393 if (m != NULL) 2394 nd = EAGAIN; /* send what we already got */ 2395 else if ((flags & SF_NODISKIO) != 0) 2396 nd = EBUSY; 2397 else 2398 nd = 0; 2399 error = sendfile_readpage(obj, vp, nd, off, 2400 xfsize, bsize, td, &pg); 2401 if (error != 0) { 2402 if (error == EAGAIN) 2403 error = 0; /* not a real error */ 2404 break; 2405 } 2406 2407 /* 2408 * Get a sendfile buf. When allocating the 2409 * first buffer for mbuf chain, we usually 2410 * wait as long as necessary, but this wait 2411 * can be interrupted. For consequent 2412 * buffers, do not sleep, since several 2413 * threads might exhaust the buffers and then 2414 * deadlock. 2415 */ 2416 sf = sf_buf_alloc(pg, (mnw || m != NULL) ? SFB_NOWAIT : 2417 SFB_CATCH); 2418 if (sf == NULL) { 2419 SFSTAT_INC(sf_allocfail); 2420 vm_page_lock(pg); 2421 vm_page_unwire(pg, PQ_INACTIVE); 2422 KASSERT(pg->object != NULL, 2423 ("%s: object disappeared", __func__)); 2424 vm_page_unlock(pg); 2425 if (m == NULL) 2426 error = (mnw ? EAGAIN : EINTR); 2427 break; 2428 } 2429 2430 /* 2431 * Get an mbuf and set it up as having 2432 * external storage. 2433 */ 2434 m0 = m_get((mnw ? M_NOWAIT : M_WAITOK), MT_DATA); 2435 if (m0 == NULL) { 2436 error = (mnw ? EAGAIN : ENOBUFS); 2437 sf_ext_free(sf, NULL); 2438 break; 2439 } 2440 /* 2441 * Attach EXT_SFBUF external storage. 2442 */ 2443 m0->m_ext.ext_buf = (caddr_t )sf_buf_kva(sf); 2444 m0->m_ext.ext_size = PAGE_SIZE; 2445 m0->m_ext.ext_arg1 = sf; 2446 m0->m_ext.ext_arg2 = sfs; 2447 m0->m_ext.ext_type = EXT_SFBUF; 2448 m0->m_ext.ext_flags = 0; 2449 m0->m_flags |= (M_EXT|M_RDONLY); 2450 m0->m_data = (char *)sf_buf_kva(sf) + pgoff; 2451 m0->m_len = xfsize; 2452 2453 /* Append to mbuf chain. */ 2454 if (mtail != NULL) 2455 mtail->m_next = m0; 2456 else if (m != NULL) 2457 m_last(m)->m_next = m0; 2458 else 2459 m = m0; 2460 mtail = m0; 2461 2462 /* Keep track of bits processed. */ 2463 loopbytes += xfsize; 2464 off += xfsize; 2465 2466 if (sfs != NULL) { 2467 mtx_lock(&sfs->mtx); 2468 sfs->count++; 2469 mtx_unlock(&sfs->mtx); 2470 } 2471 } 2472 2473 if (vp != NULL) 2474 VOP_UNLOCK(vp, 0); 2475 2476 /* Add the buffer chain to the socket buffer. */ 2477 if (m != NULL) { 2478 int mlen, err; 2479 2480 mlen = m_length(m, NULL); 2481 SOCKBUF_LOCK(&so->so_snd); 2482 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2483 error = EPIPE; 2484 SOCKBUF_UNLOCK(&so->so_snd); 2485 goto done; 2486 } 2487 SOCKBUF_UNLOCK(&so->so_snd); 2488 CURVNET_SET(so->so_vnet); 2489 /* Avoid error aliasing. */ 2490 err = (*so->so_proto->pr_usrreqs->pru_send) 2491 (so, 0, m, NULL, NULL, td); 2492 CURVNET_RESTORE(); 2493 if (err == 0) { 2494 /* 2495 * We need two counters to get the 2496 * file offset and nbytes to send 2497 * right: 2498 * - sbytes contains the total amount 2499 * of bytes sent, including headers. 2500 * - fsbytes contains the total amount 2501 * of bytes sent from the file. 2502 */ 2503 sbytes += mlen; 2504 fsbytes += mlen; 2505 if (hdrlen) { 2506 fsbytes -= hdrlen; 2507 hdrlen = 0; 2508 } 2509 } else if (error == 0) 2510 error = err; 2511 m = NULL; /* pru_send always consumes */ 2512 } 2513 2514 /* Quit outer loop on error or when we're done. */ 2515 if (done) 2516 break; 2517 if (error != 0) 2518 goto done; 2519 } 2520 2521 /* 2522 * Send trailers. Wimp out and use writev(2). 2523 */ 2524 if (trl_uio != NULL) { 2525 sbunlock(&so->so_snd); 2526 error = kern_writev(td, sockfd, trl_uio); 2527 if (error == 0) 2528 sbytes += td->td_retval[0]; 2529 goto out; 2530 } 2531 2532 done: 2533 sbunlock(&so->so_snd); 2534 out: 2535 /* 2536 * If there was no error we have to clear td->td_retval[0] 2537 * because it may have been set by writev. 2538 */ 2539 if (error == 0) { 2540 td->td_retval[0] = 0; 2541 } 2542 if (sent != NULL) { 2543 (*sent) = sbytes; 2544 } 2545 if (obj != NULL) 2546 vm_object_deallocate(obj); 2547 if (so) 2548 fdrop(sock_fp, td); 2549 if (m) 2550 m_freem(m); 2551 2552 if (sfs != NULL) { 2553 mtx_lock(&sfs->mtx); 2554 if (sfs->count != 0) 2555 cv_wait(&sfs->cv, &sfs->mtx); 2556 KASSERT(sfs->count == 0, ("sendfile sync still busy")); 2557 cv_destroy(&sfs->cv); 2558 mtx_destroy(&sfs->mtx); 2559 free(sfs, M_TEMP); 2560 } 2561 2562 if (error == ERESTART) 2563 error = EINTR; 2564 2565 return (error); 2566 } 2567