1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. All rights reserved. 4 * Copyright (c) 2004 The FreeBSD Foundation 5 * Copyright (c) 2004-2006 Robert N. M. Watson 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 4. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 32 */ 33 34 /* 35 * Comments on the socket life cycle: 36 * 37 * soalloc() sets of socket layer state for a socket, called only by 38 * socreate() and sonewconn(). Socket layer private. 39 * 40 * sodealloc() tears down socket layer state for a socket, called only by 41 * sofree() and sonewconn(). Socket layer private. 42 * 43 * pru_attach() associates protocol layer state with an allocated socket; 44 * called only once, may fail, aborting socket allocation. This is called 45 * from socreate() and sonewconn(). Socket layer private. 46 * 47 * pru_detach() disassociates protocol layer state from an attached socket, 48 * and will be called exactly once for sockets in which pru_attach() has 49 * been successfully called. If pru_attach() returned an error, 50 * pru_detach() will not be called. Socket layer private. 51 * 52 * pru_abort() and pru_close() notify the protocol layer that the last 53 * consumer of a socket is starting to tear down the socket, and that the 54 * protocol should terminate the connection. Historically, pru_abort() also 55 * detached protocol state from the socket state, but this is no longer the 56 * case. 57 * 58 * socreate() creates a socket and attaches protocol state. This is a public 59 * interface that may be used by socket layer consumers to create new 60 * sockets. 61 * 62 * sonewconn() creates a socket and attaches protocol state. This is a 63 * public interface that may be used by protocols to create new sockets when 64 * a new connection is received and will be available for accept() on a 65 * listen socket. 66 * 67 * soclose() destroys a socket after possibly waiting for it to disconnect. 68 * This is a public interface that socket consumers should use to close and 69 * release a socket when done with it. 70 * 71 * soabort() destroys a socket without waiting for it to disconnect (used 72 * only for incoming connections that are already partially or fully 73 * connected). This is used internally by the socket layer when clearing 74 * listen socket queues (due to overflow or close on the listen socket), but 75 * is also a public interface protocols may use to abort connections in 76 * their incomplete listen queues should they no longer be required. Sockets 77 * placed in completed connection listen queues should not be aborted for 78 * reasons described in the comment above the soclose() implementation. This 79 * is not a general purpose close routine, and except in the specific 80 * circumstances described here, should not be used. 81 * 82 * sofree() will free a socket and its protocol state if all references on 83 * the socket have been released, and is the public interface to attempt to 84 * free a socket when a reference is removed. This is a socket layer private 85 * interface. 86 * 87 * NOTE: In addition to socreate() and soclose(), which provide a single 88 * socket reference to the consumer to be managed as required, there are two 89 * calls to explicitly manage socket references, soref(), and sorele(). 90 * Currently, these are generally required only when transitioning a socket 91 * from a listen queue to a file descriptor, in order to prevent garbage 92 * collection of the socket at an untimely moment. For a number of reasons, 93 * these interfaces are not preferred, and should be avoided. 94 */ 95 96 #include <sys/cdefs.h> 97 __FBSDID("$FreeBSD$"); 98 99 #include "opt_inet.h" 100 #include "opt_mac.h" 101 #include "opt_zero.h" 102 #include "opt_compat.h" 103 104 #include <sys/param.h> 105 #include <sys/systm.h> 106 #include <sys/fcntl.h> 107 #include <sys/limits.h> 108 #include <sys/lock.h> 109 #include <sys/mac.h> 110 #include <sys/malloc.h> 111 #include <sys/mbuf.h> 112 #include <sys/mutex.h> 113 #include <sys/domain.h> 114 #include <sys/file.h> /* for struct knote */ 115 #include <sys/kernel.h> 116 #include <sys/event.h> 117 #include <sys/eventhandler.h> 118 #include <sys/poll.h> 119 #include <sys/proc.h> 120 #include <sys/protosw.h> 121 #include <sys/socket.h> 122 #include <sys/socketvar.h> 123 #include <sys/resourcevar.h> 124 #include <sys/signalvar.h> 125 #include <sys/sysctl.h> 126 #include <sys/uio.h> 127 #include <sys/jail.h> 128 129 #include <security/mac/mac_framework.h> 130 131 #include <vm/uma.h> 132 133 #ifdef COMPAT_IA32 134 #include <sys/mount.h> 135 #include <compat/freebsd32/freebsd32.h> 136 137 extern struct sysentvec ia32_freebsd_sysvec; 138 #endif 139 140 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 141 int flags); 142 143 static void filt_sordetach(struct knote *kn); 144 static int filt_soread(struct knote *kn, long hint); 145 static void filt_sowdetach(struct knote *kn); 146 static int filt_sowrite(struct knote *kn, long hint); 147 static int filt_solisten(struct knote *kn, long hint); 148 149 static struct filterops solisten_filtops = 150 { 1, NULL, filt_sordetach, filt_solisten }; 151 static struct filterops soread_filtops = 152 { 1, NULL, filt_sordetach, filt_soread }; 153 static struct filterops sowrite_filtops = 154 { 1, NULL, filt_sowdetach, filt_sowrite }; 155 156 uma_zone_t socket_zone; 157 so_gen_t so_gencnt; /* generation count for sockets */ 158 159 int maxsockets; 160 161 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 162 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 163 164 static int somaxconn = SOMAXCONN; 165 static int somaxconn_sysctl(SYSCTL_HANDLER_ARGS); 166 /* XXX: we dont have SYSCTL_USHORT */ 167 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLTYPE_UINT | CTLFLAG_RW, 168 0, sizeof(int), somaxconn_sysctl, "I", "Maximum pending socket connection " 169 "queue size"); 170 static int numopensockets; 171 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 172 &numopensockets, 0, "Number of open sockets"); 173 #ifdef ZERO_COPY_SOCKETS 174 /* These aren't static because they're used in other files. */ 175 int so_zero_copy_send = 1; 176 int so_zero_copy_receive = 1; 177 SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0, 178 "Zero copy controls"); 179 SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW, 180 &so_zero_copy_receive, 0, "Enable zero copy receive"); 181 SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW, 182 &so_zero_copy_send, 0, "Enable zero copy send"); 183 #endif /* ZERO_COPY_SOCKETS */ 184 185 /* 186 * accept_mtx locks down per-socket fields relating to accept queues. See 187 * socketvar.h for an annotation of the protected fields of struct socket. 188 */ 189 struct mtx accept_mtx; 190 MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 191 192 /* 193 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 194 * so_gencnt field. 195 */ 196 static struct mtx so_global_mtx; 197 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 198 199 /* 200 * General IPC sysctl name space, used by sockets and a variety of other IPC 201 * types. 202 */ 203 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC"); 204 205 /* 206 * Sysctl to get and set the maximum global sockets limit. Notify protocols 207 * of the change so that they can update their dependent limits as required. 208 */ 209 static int 210 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 211 { 212 int error, newmaxsockets; 213 214 newmaxsockets = maxsockets; 215 error = sysctl_handle_int(oidp, &newmaxsockets, sizeof(int), req); 216 if (error == 0 && req->newptr) { 217 if (newmaxsockets > maxsockets) { 218 maxsockets = newmaxsockets; 219 if (maxsockets > ((maxfiles / 4) * 3)) { 220 maxfiles = (maxsockets * 5) / 4; 221 maxfilesperproc = (maxfiles * 9) / 10; 222 } 223 EVENTHANDLER_INVOKE(maxsockets_change); 224 } else 225 error = EINVAL; 226 } 227 return (error); 228 } 229 230 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, CTLTYPE_INT|CTLFLAG_RW, 231 &maxsockets, 0, sysctl_maxsockets, "IU", 232 "Maximum number of sockets avaliable"); 233 234 /* 235 * Initialise maxsockets. 236 */ 237 static void init_maxsockets(void *ignored) 238 { 239 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 240 maxsockets = imax(maxsockets, imax(maxfiles, nmbclusters)); 241 } 242 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 243 244 /* 245 * Socket operation routines. These routines are called by the routines in 246 * sys_socket.c or from a system process, and implement the semantics of 247 * socket operations by switching out to the protocol specific routines. 248 */ 249 250 /* 251 * Get a socket structure from our zone, and initialize it. Note that it 252 * would probably be better to allocate socket and PCB at the same time, but 253 * I'm not convinced that all the protocols can be easily modified to do 254 * this. 255 * 256 * soalloc() returns a socket with a ref count of 0. 257 */ 258 static struct socket * 259 soalloc(int mflags) 260 { 261 struct socket *so; 262 263 so = uma_zalloc(socket_zone, mflags | M_ZERO); 264 if (so == NULL) 265 return (NULL); 266 #ifdef MAC 267 if (mac_init_socket(so, mflags) != 0) { 268 uma_zfree(socket_zone, so); 269 return (NULL); 270 } 271 #endif 272 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 273 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 274 TAILQ_INIT(&so->so_aiojobq); 275 mtx_lock(&so_global_mtx); 276 so->so_gencnt = ++so_gencnt; 277 ++numopensockets; 278 mtx_unlock(&so_global_mtx); 279 return (so); 280 } 281 282 /* 283 * Free the storage associated with a socket at the socket layer, tear down 284 * locks, labels, etc. All protocol state is assumed already to have been 285 * torn down (and possibly never set up) by the caller. 286 */ 287 static void 288 sodealloc(struct socket *so) 289 { 290 291 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 292 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 293 294 mtx_lock(&so_global_mtx); 295 so->so_gencnt = ++so_gencnt; 296 --numopensockets; /* Could be below, but faster here. */ 297 mtx_unlock(&so_global_mtx); 298 if (so->so_rcv.sb_hiwat) 299 (void)chgsbsize(so->so_cred->cr_uidinfo, 300 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 301 if (so->so_snd.sb_hiwat) 302 (void)chgsbsize(so->so_cred->cr_uidinfo, 303 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 304 #ifdef INET 305 /* remove acccept filter if one is present. */ 306 if (so->so_accf != NULL) 307 do_setopt_accept_filter(so, NULL); 308 #endif 309 #ifdef MAC 310 mac_destroy_socket(so); 311 #endif 312 crfree(so->so_cred); 313 SOCKBUF_LOCK_DESTROY(&so->so_snd); 314 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 315 uma_zfree(socket_zone, so); 316 } 317 318 /* 319 * socreate returns a socket with a ref count of 1. The socket should be 320 * closed with soclose(). 321 */ 322 int 323 socreate(dom, aso, type, proto, cred, td) 324 int dom; 325 struct socket **aso; 326 int type; 327 int proto; 328 struct ucred *cred; 329 struct thread *td; 330 { 331 struct protosw *prp; 332 struct socket *so; 333 int error; 334 335 if (proto) 336 prp = pffindproto(dom, proto, type); 337 else 338 prp = pffindtype(dom, type); 339 340 if (prp == NULL || prp->pr_usrreqs->pru_attach == NULL || 341 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 342 return (EPROTONOSUPPORT); 343 344 if (jailed(cred) && jail_socket_unixiproute_only && 345 prp->pr_domain->dom_family != PF_LOCAL && 346 prp->pr_domain->dom_family != PF_INET && 347 prp->pr_domain->dom_family != PF_ROUTE) { 348 return (EPROTONOSUPPORT); 349 } 350 351 if (prp->pr_type != type) 352 return (EPROTOTYPE); 353 so = soalloc(M_WAITOK); 354 if (so == NULL) 355 return (ENOBUFS); 356 357 TAILQ_INIT(&so->so_incomp); 358 TAILQ_INIT(&so->so_comp); 359 so->so_type = type; 360 so->so_cred = crhold(cred); 361 so->so_proto = prp; 362 #ifdef MAC 363 mac_create_socket(cred, so); 364 #endif 365 knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv), 366 NULL, NULL, NULL); 367 knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd), 368 NULL, NULL, NULL); 369 so->so_count = 1; 370 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 371 if (error) { 372 KASSERT(so->so_count == 1, ("socreate: so_count %d", 373 so->so_count)); 374 so->so_count = 0; 375 sodealloc(so); 376 return (error); 377 } 378 *aso = so; 379 return (0); 380 } 381 382 #ifdef REGRESSION 383 static int regression_sonewconn_earlytest = 1; 384 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 385 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 386 #endif 387 388 /* 389 * When an attempt at a new connection is noted on a socket which accepts 390 * connections, sonewconn is called. If the connection is possible (subject 391 * to space constraints, etc.) then we allocate a new structure, propoerly 392 * linked into the data structure of the original socket, and return this. 393 * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED. 394 * 395 * Note: the ref count on the socket is 0 on return. 396 */ 397 struct socket * 398 sonewconn(head, connstatus) 399 register struct socket *head; 400 int connstatus; 401 { 402 register struct socket *so; 403 int over; 404 405 ACCEPT_LOCK(); 406 over = (head->so_qlen > 3 * head->so_qlimit / 2); 407 ACCEPT_UNLOCK(); 408 #ifdef REGRESSION 409 if (regression_sonewconn_earlytest && over) 410 #else 411 if (over) 412 #endif 413 return (NULL); 414 so = soalloc(M_NOWAIT); 415 if (so == NULL) 416 return (NULL); 417 if ((head->so_options & SO_ACCEPTFILTER) != 0) 418 connstatus = 0; 419 so->so_head = head; 420 so->so_type = head->so_type; 421 so->so_options = head->so_options &~ SO_ACCEPTCONN; 422 so->so_linger = head->so_linger; 423 so->so_state = head->so_state | SS_NOFDREF; 424 so->so_proto = head->so_proto; 425 so->so_cred = crhold(head->so_cred); 426 #ifdef MAC 427 SOCK_LOCK(head); 428 mac_create_socket_from_socket(head, so); 429 SOCK_UNLOCK(head); 430 #endif 431 knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv), 432 NULL, NULL, NULL); 433 knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd), 434 NULL, NULL, NULL); 435 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) || 436 (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 437 sodealloc(so); 438 return (NULL); 439 } 440 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 441 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 442 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 443 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 444 so->so_state |= connstatus; 445 ACCEPT_LOCK(); 446 if (connstatus) { 447 TAILQ_INSERT_TAIL(&head->so_comp, so, so_list); 448 so->so_qstate |= SQ_COMP; 449 head->so_qlen++; 450 } else { 451 /* 452 * Keep removing sockets from the head until there's room for 453 * us to insert on the tail. In pre-locking revisions, this 454 * was a simple if(), but as we could be racing with other 455 * threads and soabort() requires dropping locks, we must 456 * loop waiting for the condition to be true. 457 */ 458 while (head->so_incqlen > head->so_qlimit) { 459 struct socket *sp; 460 sp = TAILQ_FIRST(&head->so_incomp); 461 TAILQ_REMOVE(&head->so_incomp, sp, so_list); 462 head->so_incqlen--; 463 sp->so_qstate &= ~SQ_INCOMP; 464 sp->so_head = NULL; 465 ACCEPT_UNLOCK(); 466 soabort(sp); 467 ACCEPT_LOCK(); 468 } 469 TAILQ_INSERT_TAIL(&head->so_incomp, so, so_list); 470 so->so_qstate |= SQ_INCOMP; 471 head->so_incqlen++; 472 } 473 ACCEPT_UNLOCK(); 474 if (connstatus) { 475 sorwakeup(head); 476 wakeup_one(&head->so_timeo); 477 } 478 return (so); 479 } 480 481 int 482 sobind(so, nam, td) 483 struct socket *so; 484 struct sockaddr *nam; 485 struct thread *td; 486 { 487 488 return ((*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td)); 489 } 490 491 /* 492 * solisten() transitions a socket from a non-listening state to a listening 493 * state, but can also be used to update the listen queue depth on an 494 * existing listen socket. The protocol will call back into the sockets 495 * layer using solisten_proto_check() and solisten_proto() to check and set 496 * socket-layer listen state. Call backs are used so that the protocol can 497 * acquire both protocol and socket layer locks in whatever order is required 498 * by the protocol. 499 * 500 * Protocol implementors are advised to hold the socket lock across the 501 * socket-layer test and set to avoid races at the socket layer. 502 */ 503 int 504 solisten(so, backlog, td) 505 struct socket *so; 506 int backlog; 507 struct thread *td; 508 { 509 510 return ((*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td)); 511 } 512 513 int 514 solisten_proto_check(so) 515 struct socket *so; 516 { 517 518 SOCK_LOCK_ASSERT(so); 519 520 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 521 SS_ISDISCONNECTING)) 522 return (EINVAL); 523 return (0); 524 } 525 526 void 527 solisten_proto(so, backlog) 528 struct socket *so; 529 int backlog; 530 { 531 532 SOCK_LOCK_ASSERT(so); 533 534 if (backlog < 0 || backlog > somaxconn) 535 backlog = somaxconn; 536 so->so_qlimit = backlog; 537 so->so_options |= SO_ACCEPTCONN; 538 } 539 540 /* 541 * Attempt to free a socket. This should really be sotryfree(). 542 * 543 * sofree() will succeed if: 544 * 545 * - There are no outstanding file descriptor references or related consumers 546 * (so_count == 0). 547 * 548 * - The socket has been closed by user space, if ever open (SS_NOFDREF). 549 * 550 * - The protocol does not have an outstanding strong reference on the socket 551 * (SS_PROTOREF). 552 * 553 * - The socket is not in a completed connection queue, so a process has been 554 * notified that it is present. If it is removed, the user process may 555 * block in accept() despite select() saying the socket was ready. 556 * 557 * Otherwise, it will quietly abort so that a future call to sofree(), when 558 * conditions are right, can succeed. 559 */ 560 void 561 sofree(so) 562 struct socket *so; 563 { 564 struct protosw *pr = so->so_proto; 565 struct socket *head; 566 567 ACCEPT_LOCK_ASSERT(); 568 SOCK_LOCK_ASSERT(so); 569 570 if ((so->so_state & SS_NOFDREF) == 0 || so->so_count != 0 || 571 (so->so_state & SS_PROTOREF) || (so->so_qstate & SQ_COMP)) { 572 SOCK_UNLOCK(so); 573 ACCEPT_UNLOCK(); 574 return; 575 } 576 577 head = so->so_head; 578 if (head != NULL) { 579 KASSERT((so->so_qstate & SQ_COMP) != 0 || 580 (so->so_qstate & SQ_INCOMP) != 0, 581 ("sofree: so_head != NULL, but neither SQ_COMP nor " 582 "SQ_INCOMP")); 583 KASSERT((so->so_qstate & SQ_COMP) == 0 || 584 (so->so_qstate & SQ_INCOMP) == 0, 585 ("sofree: so->so_qstate is SQ_COMP and also SQ_INCOMP")); 586 TAILQ_REMOVE(&head->so_incomp, so, so_list); 587 head->so_incqlen--; 588 so->so_qstate &= ~SQ_INCOMP; 589 so->so_head = NULL; 590 } 591 KASSERT((so->so_qstate & SQ_COMP) == 0 && 592 (so->so_qstate & SQ_INCOMP) == 0, 593 ("sofree: so_head == NULL, but still SQ_COMP(%d) or SQ_INCOMP(%d)", 594 so->so_qstate & SQ_COMP, so->so_qstate & SQ_INCOMP)); 595 if (so->so_options & SO_ACCEPTCONN) { 596 KASSERT((TAILQ_EMPTY(&so->so_comp)), ("sofree: so_comp populated")); 597 KASSERT((TAILQ_EMPTY(&so->so_incomp)), ("sofree: so_comp populated")); 598 } 599 SOCK_UNLOCK(so); 600 ACCEPT_UNLOCK(); 601 602 /* 603 * From this point on, we assume that no other references to this 604 * socket exist anywhere else in the stack. Therefore, no locks need 605 * to be acquired or held. 606 * 607 * We used to do a lot of socket buffer and socket locking here, as 608 * well as invoke sorflush() and perform wakeups. The direct call to 609 * dom_dispose() and sbrelease_internal() are an inlining of what was 610 * necessary from sorflush(). 611 * 612 * Notice that the socket buffer and kqueue state are torn down 613 * before calling pru_detach. This means that protocols shold not 614 * assume they can perform socket wakeups, etc, in their detach 615 * code. 616 */ 617 KASSERT((so->so_snd.sb_flags & SB_LOCK) == 0, ("sofree: snd sblock")); 618 KASSERT((so->so_rcv.sb_flags & SB_LOCK) == 0, ("sofree: rcv sblock")); 619 sbdestroy(&so->so_snd, so); 620 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 621 (*pr->pr_domain->dom_dispose)(so->so_rcv.sb_mb); 622 sbdestroy(&so->so_rcv, so); 623 if (pr->pr_usrreqs->pru_detach != NULL) 624 (*pr->pr_usrreqs->pru_detach)(so); 625 knlist_destroy(&so->so_rcv.sb_sel.si_note); 626 knlist_destroy(&so->so_snd.sb_sel.si_note); 627 sodealloc(so); 628 } 629 630 /* 631 * Close a socket on last file table reference removal. Initiate disconnect 632 * if connected. Free socket when disconnect complete. 633 * 634 * This function will sorele() the socket. Note that soclose() may be called 635 * prior to the ref count reaching zero. The actual socket structure will 636 * not be freed until the ref count reaches zero. 637 */ 638 int 639 soclose(so) 640 struct socket *so; 641 { 642 int error = 0; 643 644 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 645 646 funsetown(&so->so_sigio); 647 if (so->so_state & SS_ISCONNECTED) { 648 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 649 error = sodisconnect(so); 650 if (error) 651 goto drop; 652 } 653 if (so->so_options & SO_LINGER) { 654 if ((so->so_state & SS_ISDISCONNECTING) && 655 (so->so_state & SS_NBIO)) 656 goto drop; 657 while (so->so_state & SS_ISCONNECTED) { 658 error = tsleep(&so->so_timeo, 659 PSOCK | PCATCH, "soclos", so->so_linger * hz); 660 if (error) 661 break; 662 } 663 } 664 } 665 666 drop: 667 if (so->so_proto->pr_usrreqs->pru_close != NULL) 668 (*so->so_proto->pr_usrreqs->pru_close)(so); 669 if (so->so_options & SO_ACCEPTCONN) { 670 struct socket *sp; 671 ACCEPT_LOCK(); 672 while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { 673 TAILQ_REMOVE(&so->so_incomp, sp, so_list); 674 so->so_incqlen--; 675 sp->so_qstate &= ~SQ_INCOMP; 676 sp->so_head = NULL; 677 ACCEPT_UNLOCK(); 678 soabort(sp); 679 ACCEPT_LOCK(); 680 } 681 while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { 682 TAILQ_REMOVE(&so->so_comp, sp, so_list); 683 so->so_qlen--; 684 sp->so_qstate &= ~SQ_COMP; 685 sp->so_head = NULL; 686 ACCEPT_UNLOCK(); 687 soabort(sp); 688 ACCEPT_LOCK(); 689 } 690 ACCEPT_UNLOCK(); 691 } 692 ACCEPT_LOCK(); 693 SOCK_LOCK(so); 694 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 695 so->so_state |= SS_NOFDREF; 696 sorele(so); 697 return (error); 698 } 699 700 /* 701 * soabort() is used to abruptly tear down a connection, such as when a 702 * resource limit is reached (listen queue depth exceeded), or if a listen 703 * socket is closed while there are sockets waiting to be accepted. 704 * 705 * This interface is tricky, because it is called on an unreferenced socket, 706 * and must be called only by a thread that has actually removed the socket 707 * from the listen queue it was on, or races with other threads are risked. 708 * 709 * This interface will call into the protocol code, so must not be called 710 * with any socket locks held. Protocols do call it while holding their own 711 * recursible protocol mutexes, but this is something that should be subject 712 * to review in the future. 713 */ 714 void 715 soabort(so) 716 struct socket *so; 717 { 718 719 /* 720 * In as much as is possible, assert that no references to this 721 * socket are held. This is not quite the same as asserting that the 722 * current thread is responsible for arranging for no references, but 723 * is as close as we can get for now. 724 */ 725 KASSERT(so->so_count == 0, ("soabort: so_count")); 726 KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF")); 727 KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF")); 728 KASSERT((so->so_state & SQ_COMP) == 0, ("soabort: SQ_COMP")); 729 KASSERT((so->so_state & SQ_INCOMP) == 0, ("soabort: SQ_INCOMP")); 730 731 if (so->so_proto->pr_usrreqs->pru_abort != NULL) 732 (*so->so_proto->pr_usrreqs->pru_abort)(so); 733 ACCEPT_LOCK(); 734 SOCK_LOCK(so); 735 sofree(so); 736 } 737 738 int 739 soaccept(so, nam) 740 struct socket *so; 741 struct sockaddr **nam; 742 { 743 int error; 744 745 SOCK_LOCK(so); 746 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 747 so->so_state &= ~SS_NOFDREF; 748 SOCK_UNLOCK(so); 749 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 750 return (error); 751 } 752 753 int 754 soconnect(so, nam, td) 755 struct socket *so; 756 struct sockaddr *nam; 757 struct thread *td; 758 { 759 int error; 760 761 if (so->so_options & SO_ACCEPTCONN) 762 return (EOPNOTSUPP); 763 /* 764 * If protocol is connection-based, can only connect once. 765 * Otherwise, if connected, try to disconnect first. This allows 766 * user to disconnect by connecting to, e.g., a null address. 767 */ 768 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 769 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 770 (error = sodisconnect(so)))) { 771 error = EISCONN; 772 } else { 773 /* 774 * Prevent accumulated error from previous connection from 775 * biting us. 776 */ 777 so->so_error = 0; 778 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td); 779 } 780 781 return (error); 782 } 783 784 int 785 soconnect2(so1, so2) 786 struct socket *so1; 787 struct socket *so2; 788 { 789 790 return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2)); 791 } 792 793 int 794 sodisconnect(so) 795 struct socket *so; 796 { 797 int error; 798 799 if ((so->so_state & SS_ISCONNECTED) == 0) 800 return (ENOTCONN); 801 if (so->so_state & SS_ISDISCONNECTING) 802 return (EALREADY); 803 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 804 return (error); 805 } 806 807 #ifdef ZERO_COPY_SOCKETS 808 struct so_zerocopy_stats{ 809 int size_ok; 810 int align_ok; 811 int found_ifp; 812 }; 813 struct so_zerocopy_stats so_zerocp_stats = {0,0,0}; 814 #include <netinet/in.h> 815 #include <net/route.h> 816 #include <netinet/in_pcb.h> 817 #include <vm/vm.h> 818 #include <vm/vm_page.h> 819 #include <vm/vm_object.h> 820 821 /* 822 * sosend_copyin() is only used if zero copy sockets are enabled. Otherwise 823 * sosend_dgram() and sosend_generic() use m_uiotombuf(). 824 * 825 * sosend_copyin() accepts a uio and prepares an mbuf chain holding part or 826 * all of the data referenced by the uio. If desired, it uses zero-copy. 827 * *space will be updated to reflect data copied in. 828 * 829 * NB: If atomic I/O is requested, the caller must already have checked that 830 * space can hold resid bytes. 831 * 832 * NB: In the event of an error, the caller may need to free the partial 833 * chain pointed to by *mpp. The contents of both *uio and *space may be 834 * modified even in the case of an error. 835 */ 836 static int 837 sosend_copyin(struct uio *uio, struct mbuf **retmp, int atomic, long *space, 838 int flags) 839 { 840 struct mbuf *m, **mp, *top; 841 long len, resid; 842 int error; 843 #ifdef ZERO_COPY_SOCKETS 844 int cow_send; 845 #endif 846 847 *retmp = top = NULL; 848 mp = ⊤ 849 len = 0; 850 resid = uio->uio_resid; 851 error = 0; 852 do { 853 #ifdef ZERO_COPY_SOCKETS 854 cow_send = 0; 855 #endif /* ZERO_COPY_SOCKETS */ 856 if (resid >= MINCLSIZE) { 857 #ifdef ZERO_COPY_SOCKETS 858 if (top == NULL) { 859 MGETHDR(m, M_TRYWAIT, MT_DATA); 860 if (m == NULL) { 861 error = ENOBUFS; 862 goto out; 863 } 864 m->m_pkthdr.len = 0; 865 m->m_pkthdr.rcvif = NULL; 866 } else { 867 MGET(m, M_TRYWAIT, MT_DATA); 868 if (m == NULL) { 869 error = ENOBUFS; 870 goto out; 871 } 872 } 873 if (so_zero_copy_send && 874 resid>=PAGE_SIZE && 875 *space>=PAGE_SIZE && 876 uio->uio_iov->iov_len>=PAGE_SIZE) { 877 so_zerocp_stats.size_ok++; 878 so_zerocp_stats.align_ok++; 879 cow_send = socow_setup(m, uio); 880 len = cow_send; 881 } 882 if (!cow_send) { 883 MCLGET(m, M_TRYWAIT); 884 if ((m->m_flags & M_EXT) == 0) { 885 m_free(m); 886 m = NULL; 887 } else { 888 len = min(min(MCLBYTES, resid), 889 *space); 890 } 891 } 892 #else /* ZERO_COPY_SOCKETS */ 893 if (top == NULL) { 894 m = m_getcl(M_TRYWAIT, MT_DATA, M_PKTHDR); 895 m->m_pkthdr.len = 0; 896 m->m_pkthdr.rcvif = NULL; 897 } else 898 m = m_getcl(M_TRYWAIT, MT_DATA, 0); 899 len = min(min(MCLBYTES, resid), *space); 900 #endif /* ZERO_COPY_SOCKETS */ 901 } else { 902 if (top == NULL) { 903 m = m_gethdr(M_TRYWAIT, MT_DATA); 904 m->m_pkthdr.len = 0; 905 m->m_pkthdr.rcvif = NULL; 906 907 len = min(min(MHLEN, resid), *space); 908 /* 909 * For datagram protocols, leave room 910 * for protocol headers in first mbuf. 911 */ 912 if (atomic && m && len < MHLEN) 913 MH_ALIGN(m, len); 914 } else { 915 m = m_get(M_TRYWAIT, MT_DATA); 916 len = min(min(MLEN, resid), *space); 917 } 918 } 919 if (m == NULL) { 920 error = ENOBUFS; 921 goto out; 922 } 923 924 *space -= len; 925 #ifdef ZERO_COPY_SOCKETS 926 if (cow_send) 927 error = 0; 928 else 929 #endif /* ZERO_COPY_SOCKETS */ 930 error = uiomove(mtod(m, void *), (int)len, uio); 931 resid = uio->uio_resid; 932 m->m_len = len; 933 *mp = m; 934 top->m_pkthdr.len += len; 935 if (error) 936 goto out; 937 mp = &m->m_next; 938 if (resid <= 0) { 939 if (flags & MSG_EOR) 940 top->m_flags |= M_EOR; 941 break; 942 } 943 } while (*space > 0 && atomic); 944 out: 945 *retmp = top; 946 return (error); 947 } 948 #endif /*ZERO_COPY_SOCKETS*/ 949 950 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK) 951 952 int 953 sosend_dgram(so, addr, uio, top, control, flags, td) 954 struct socket *so; 955 struct sockaddr *addr; 956 struct uio *uio; 957 struct mbuf *top; 958 struct mbuf *control; 959 int flags; 960 struct thread *td; 961 { 962 long space, resid; 963 int clen = 0, error, dontroute; 964 #ifdef ZERO_COPY_SOCKETS 965 int atomic = sosendallatonce(so) || top; 966 #endif 967 968 KASSERT(so->so_type == SOCK_DGRAM, ("sodgram_send: !SOCK_DGRAM")); 969 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 970 ("sodgram_send: !PR_ATOMIC")); 971 972 if (uio != NULL) 973 resid = uio->uio_resid; 974 else 975 resid = top->m_pkthdr.len; 976 /* 977 * In theory resid should be unsigned. However, space must be 978 * signed, as it might be less than 0 if we over-committed, and we 979 * must use a signed comparison of space and resid. On the other 980 * hand, a negative resid causes us to loop sending 0-length 981 * segments to the protocol. 982 * 983 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 984 * type sockets since that's an error. 985 */ 986 if (resid < 0) { 987 error = EINVAL; 988 goto out; 989 } 990 991 dontroute = 992 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 993 if (td != NULL) 994 td->td_proc->p_stats->p_ru.ru_msgsnd++; 995 if (control != NULL) 996 clen = control->m_len; 997 998 SOCKBUF_LOCK(&so->so_snd); 999 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1000 SOCKBUF_UNLOCK(&so->so_snd); 1001 error = EPIPE; 1002 goto out; 1003 } 1004 if (so->so_error) { 1005 error = so->so_error; 1006 so->so_error = 0; 1007 SOCKBUF_UNLOCK(&so->so_snd); 1008 goto out; 1009 } 1010 if ((so->so_state & SS_ISCONNECTED) == 0) { 1011 /* 1012 * `sendto' and `sendmsg' is allowed on a connection-based 1013 * socket if it supports implied connect. Return ENOTCONN if 1014 * not connected and no address is supplied. 1015 */ 1016 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1017 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1018 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1019 !(resid == 0 && clen != 0)) { 1020 SOCKBUF_UNLOCK(&so->so_snd); 1021 error = ENOTCONN; 1022 goto out; 1023 } 1024 } else if (addr == NULL) { 1025 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1026 error = ENOTCONN; 1027 else 1028 error = EDESTADDRREQ; 1029 SOCKBUF_UNLOCK(&so->so_snd); 1030 goto out; 1031 } 1032 } 1033 1034 /* 1035 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 1036 * problem and need fixing. 1037 */ 1038 space = sbspace(&so->so_snd); 1039 if (flags & MSG_OOB) 1040 space += 1024; 1041 space -= clen; 1042 SOCKBUF_UNLOCK(&so->so_snd); 1043 if (resid > space) { 1044 error = EMSGSIZE; 1045 goto out; 1046 } 1047 if (uio == NULL) { 1048 resid = 0; 1049 if (flags & MSG_EOR) 1050 top->m_flags |= M_EOR; 1051 } else { 1052 #ifdef ZERO_COPY_SOCKETS 1053 error = sosend_copyin(uio, &top, atomic, &space, flags); 1054 if (error) 1055 goto out; 1056 #else 1057 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 1058 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 1059 if (top == NULL) { 1060 error = EFAULT; /* only possible error */ 1061 goto out; 1062 } 1063 space -= resid - uio->uio_resid; 1064 #endif 1065 resid = uio->uio_resid; 1066 } 1067 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 1068 /* 1069 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 1070 * than with. 1071 */ 1072 if (dontroute) { 1073 SOCK_LOCK(so); 1074 so->so_options |= SO_DONTROUTE; 1075 SOCK_UNLOCK(so); 1076 } 1077 /* 1078 * XXX all the SBS_CANTSENDMORE checks previously done could be out 1079 * of date. We could have recieved a reset packet in an interrupt or 1080 * maybe we slept while doing page faults in uiomove() etc. We could 1081 * probably recheck again inside the locking protection here, but 1082 * there are probably other places that this also happens. We must 1083 * rethink this. 1084 */ 1085 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1086 (flags & MSG_OOB) ? PRUS_OOB : 1087 /* 1088 * If the user set MSG_EOF, the protocol understands this flag and 1089 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 1090 */ 1091 ((flags & MSG_EOF) && 1092 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1093 (resid <= 0)) ? 1094 PRUS_EOF : 1095 /* If there is more to send set PRUS_MORETOCOME */ 1096 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1097 top, addr, control, td); 1098 if (dontroute) { 1099 SOCK_LOCK(so); 1100 so->so_options &= ~SO_DONTROUTE; 1101 SOCK_UNLOCK(so); 1102 } 1103 clen = 0; 1104 control = NULL; 1105 top = NULL; 1106 out: 1107 if (top != NULL) 1108 m_freem(top); 1109 if (control != NULL) 1110 m_freem(control); 1111 return (error); 1112 } 1113 1114 /* 1115 * Send on a socket. If send must go all at once and message is larger than 1116 * send buffering, then hard error. Lock against other senders. If must go 1117 * all at once and not enough room now, then inform user that this would 1118 * block and do nothing. Otherwise, if nonblocking, send as much as 1119 * possible. The data to be sent is described by "uio" if nonzero, otherwise 1120 * by the mbuf chain "top" (which must be null if uio is not). Data provided 1121 * in mbuf chain must be small enough to send all at once. 1122 * 1123 * Returns nonzero on error, timeout or signal; callers must check for short 1124 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 1125 * on return. 1126 */ 1127 #define snderr(errno) { error = (errno); goto release; } 1128 int 1129 sosend_generic(so, addr, uio, top, control, flags, td) 1130 struct socket *so; 1131 struct sockaddr *addr; 1132 struct uio *uio; 1133 struct mbuf *top; 1134 struct mbuf *control; 1135 int flags; 1136 struct thread *td; 1137 { 1138 long space, resid; 1139 int clen = 0, error, dontroute; 1140 int atomic = sosendallatonce(so) || top; 1141 1142 if (uio != NULL) 1143 resid = uio->uio_resid; 1144 else 1145 resid = top->m_pkthdr.len; 1146 /* 1147 * In theory resid should be unsigned. However, space must be 1148 * signed, as it might be less than 0 if we over-committed, and we 1149 * must use a signed comparison of space and resid. On the other 1150 * hand, a negative resid causes us to loop sending 0-length 1151 * segments to the protocol. 1152 * 1153 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 1154 * type sockets since that's an error. 1155 */ 1156 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 1157 error = EINVAL; 1158 goto out; 1159 } 1160 1161 dontroute = 1162 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1163 (so->so_proto->pr_flags & PR_ATOMIC); 1164 if (td != NULL) 1165 td->td_proc->p_stats->p_ru.ru_msgsnd++; 1166 if (control != NULL) 1167 clen = control->m_len; 1168 1169 SOCKBUF_LOCK(&so->so_snd); 1170 restart: 1171 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1172 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1173 if (error) 1174 goto out_locked; 1175 do { 1176 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1177 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 1178 snderr(EPIPE); 1179 if (so->so_error) { 1180 error = so->so_error; 1181 so->so_error = 0; 1182 goto release; 1183 } 1184 if ((so->so_state & SS_ISCONNECTED) == 0) { 1185 /* 1186 * `sendto' and `sendmsg' is allowed on a connection- 1187 * based socket if it supports implied connect. 1188 * Return ENOTCONN if not connected and no address is 1189 * supplied. 1190 */ 1191 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1192 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1193 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1194 !(resid == 0 && clen != 0)) 1195 snderr(ENOTCONN); 1196 } else if (addr == NULL) 1197 snderr(so->so_proto->pr_flags & PR_CONNREQUIRED ? 1198 ENOTCONN : EDESTADDRREQ); 1199 } 1200 space = sbspace(&so->so_snd); 1201 if (flags & MSG_OOB) 1202 space += 1024; 1203 if ((atomic && resid > so->so_snd.sb_hiwat) || 1204 clen > so->so_snd.sb_hiwat) 1205 snderr(EMSGSIZE); 1206 if (space < resid + clen && 1207 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1208 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) 1209 snderr(EWOULDBLOCK); 1210 sbunlock(&so->so_snd); 1211 error = sbwait(&so->so_snd); 1212 if (error) 1213 goto out_locked; 1214 goto restart; 1215 } 1216 SOCKBUF_UNLOCK(&so->so_snd); 1217 space -= clen; 1218 do { 1219 if (uio == NULL) { 1220 resid = 0; 1221 if (flags & MSG_EOR) 1222 top->m_flags |= M_EOR; 1223 } else { 1224 #ifdef ZERO_COPY_SOCKETS 1225 error = sosend_copyin(uio, &top, atomic, 1226 &space, flags); 1227 if (error != 0) { 1228 SOCKBUF_LOCK(&so->so_snd); 1229 goto release; 1230 } 1231 #else 1232 top = m_uiotombuf(uio, M_WAITOK, space, 1233 (atomic ? max_hdr : 0), 1234 (atomic ? M_PKTHDR : 0) | 1235 ((flags & MSG_EOR) ? M_EOR : 0)); 1236 if (top == NULL) { 1237 SOCKBUF_LOCK(&so->so_snd); 1238 error = EFAULT; /* only possible error */ 1239 goto release; 1240 } 1241 space -= resid - uio->uio_resid; 1242 #endif 1243 resid = uio->uio_resid; 1244 } 1245 if (dontroute) { 1246 SOCK_LOCK(so); 1247 so->so_options |= SO_DONTROUTE; 1248 SOCK_UNLOCK(so); 1249 } 1250 /* 1251 * XXX all the SBS_CANTSENDMORE checks previously 1252 * done could be out of date. We could have recieved 1253 * a reset packet in an interrupt or maybe we slept 1254 * while doing page faults in uiomove() etc. We 1255 * could probably recheck again inside the locking 1256 * protection here, but there are probably other 1257 * places that this also happens. We must rethink 1258 * this. 1259 */ 1260 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1261 (flags & MSG_OOB) ? PRUS_OOB : 1262 /* 1263 * If the user set MSG_EOF, the protocol understands 1264 * this flag and nothing left to send then use 1265 * PRU_SEND_EOF instead of PRU_SEND. 1266 */ 1267 ((flags & MSG_EOF) && 1268 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1269 (resid <= 0)) ? 1270 PRUS_EOF : 1271 /* If there is more to send set PRUS_MORETOCOME. */ 1272 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1273 top, addr, control, td); 1274 if (dontroute) { 1275 SOCK_LOCK(so); 1276 so->so_options &= ~SO_DONTROUTE; 1277 SOCK_UNLOCK(so); 1278 } 1279 clen = 0; 1280 control = NULL; 1281 top = NULL; 1282 if (error) { 1283 SOCKBUF_LOCK(&so->so_snd); 1284 goto release; 1285 } 1286 } while (resid && space > 0); 1287 SOCKBUF_LOCK(&so->so_snd); 1288 } while (resid); 1289 1290 release: 1291 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1292 sbunlock(&so->so_snd); 1293 out_locked: 1294 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1295 SOCKBUF_UNLOCK(&so->so_snd); 1296 out: 1297 if (top != NULL) 1298 m_freem(top); 1299 if (control != NULL) 1300 m_freem(control); 1301 return (error); 1302 } 1303 #undef snderr 1304 1305 int 1306 sosend(so, addr, uio, top, control, flags, td) 1307 struct socket *so; 1308 struct sockaddr *addr; 1309 struct uio *uio; 1310 struct mbuf *top; 1311 struct mbuf *control; 1312 int flags; 1313 struct thread *td; 1314 { 1315 1316 /* XXXRW: Temporary debugging. */ 1317 KASSERT(so->so_proto->pr_usrreqs->pru_sosend != sosend, 1318 ("sosend: protocol calls sosend")); 1319 1320 return (so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, top, 1321 control, flags, td)); 1322 } 1323 1324 /* 1325 * The part of soreceive() that implements reading non-inline out-of-band 1326 * data from a socket. For more complete comments, see soreceive(), from 1327 * which this code originated. 1328 * 1329 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1330 * unable to return an mbuf chain to the caller. 1331 */ 1332 static int 1333 soreceive_rcvoob(so, uio, flags) 1334 struct socket *so; 1335 struct uio *uio; 1336 int flags; 1337 { 1338 struct protosw *pr = so->so_proto; 1339 struct mbuf *m; 1340 int error; 1341 1342 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1343 1344 m = m_get(M_TRYWAIT, MT_DATA); 1345 if (m == NULL) 1346 return (ENOBUFS); 1347 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1348 if (error) 1349 goto bad; 1350 do { 1351 #ifdef ZERO_COPY_SOCKETS 1352 if (so_zero_copy_receive) { 1353 int disposable; 1354 1355 if ((m->m_flags & M_EXT) 1356 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1357 disposable = 1; 1358 else 1359 disposable = 0; 1360 1361 error = uiomoveco(mtod(m, void *), 1362 min(uio->uio_resid, m->m_len), 1363 uio, disposable); 1364 } else 1365 #endif /* ZERO_COPY_SOCKETS */ 1366 error = uiomove(mtod(m, void *), 1367 (int) min(uio->uio_resid, m->m_len), uio); 1368 m = m_free(m); 1369 } while (uio->uio_resid && error == 0 && m); 1370 bad: 1371 if (m != NULL) 1372 m_freem(m); 1373 return (error); 1374 } 1375 1376 /* 1377 * Following replacement or removal of the first mbuf on the first mbuf chain 1378 * of a socket buffer, push necessary state changes back into the socket 1379 * buffer so that other consumers see the values consistently. 'nextrecord' 1380 * is the callers locally stored value of the original value of 1381 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1382 * NOTE: 'nextrecord' may be NULL. 1383 */ 1384 static __inline void 1385 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1386 { 1387 1388 SOCKBUF_LOCK_ASSERT(sb); 1389 /* 1390 * First, update for the new value of nextrecord. If necessary, make 1391 * it the first record. 1392 */ 1393 if (sb->sb_mb != NULL) 1394 sb->sb_mb->m_nextpkt = nextrecord; 1395 else 1396 sb->sb_mb = nextrecord; 1397 1398 /* 1399 * Now update any dependent socket buffer fields to reflect the new 1400 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1401 * addition of a second clause that takes care of the case where 1402 * sb_mb has been updated, but remains the last record. 1403 */ 1404 if (sb->sb_mb == NULL) { 1405 sb->sb_mbtail = NULL; 1406 sb->sb_lastrecord = NULL; 1407 } else if (sb->sb_mb->m_nextpkt == NULL) 1408 sb->sb_lastrecord = sb->sb_mb; 1409 } 1410 1411 1412 /* 1413 * Implement receive operations on a socket. We depend on the way that 1414 * records are added to the sockbuf by sbappend. In particular, each record 1415 * (mbufs linked through m_next) must begin with an address if the protocol 1416 * so specifies, followed by an optional mbuf or mbufs containing ancillary 1417 * data, and then zero or more mbufs of data. In order to allow parallelism 1418 * between network receive and copying to user space, as well as avoid 1419 * sleeping with a mutex held, we release the socket buffer mutex during the 1420 * user space copy. Although the sockbuf is locked, new data may still be 1421 * appended, and thus we must maintain consistency of the sockbuf during that 1422 * time. 1423 * 1424 * The caller may receive the data as a single mbuf chain by supplying an 1425 * mbuf **mp0 for use in returning the chain. The uio is then used only for 1426 * the count in uio_resid. 1427 */ 1428 int 1429 soreceive_generic(so, psa, uio, mp0, controlp, flagsp) 1430 struct socket *so; 1431 struct sockaddr **psa; 1432 struct uio *uio; 1433 struct mbuf **mp0; 1434 struct mbuf **controlp; 1435 int *flagsp; 1436 { 1437 struct mbuf *m, **mp; 1438 int flags, len, error, offset; 1439 struct protosw *pr = so->so_proto; 1440 struct mbuf *nextrecord; 1441 int moff, type = 0; 1442 int orig_resid = uio->uio_resid; 1443 1444 mp = mp0; 1445 if (psa != NULL) 1446 *psa = NULL; 1447 if (controlp != NULL) 1448 *controlp = NULL; 1449 if (flagsp != NULL) 1450 flags = *flagsp &~ MSG_EOR; 1451 else 1452 flags = 0; 1453 if (flags & MSG_OOB) 1454 return (soreceive_rcvoob(so, uio, flags)); 1455 if (mp != NULL) 1456 *mp = NULL; 1457 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1458 && uio->uio_resid) 1459 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1460 1461 SOCKBUF_LOCK(&so->so_rcv); 1462 restart: 1463 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1464 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1465 if (error) 1466 goto out; 1467 1468 m = so->so_rcv.sb_mb; 1469 /* 1470 * If we have less data than requested, block awaiting more (subject 1471 * to any timeout) if: 1472 * 1. the current count is less than the low water mark, or 1473 * 2. MSG_WAITALL is set, and it is possible to do the entire 1474 * receive operation at once if we block (resid <= hiwat). 1475 * 3. MSG_DONTWAIT is not set 1476 * If MSG_WAITALL is set but resid is larger than the receive buffer, 1477 * we have to do the receive in sections, and thus risk returning a 1478 * short count if a timeout or signal occurs after we start. 1479 */ 1480 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1481 so->so_rcv.sb_cc < uio->uio_resid) && 1482 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat || 1483 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) && 1484 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1485 KASSERT(m != NULL || !so->so_rcv.sb_cc, 1486 ("receive: m == %p so->so_rcv.sb_cc == %u", 1487 m, so->so_rcv.sb_cc)); 1488 if (so->so_error) { 1489 if (m != NULL) 1490 goto dontblock; 1491 error = so->so_error; 1492 if ((flags & MSG_PEEK) == 0) 1493 so->so_error = 0; 1494 goto release; 1495 } 1496 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1497 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1498 if (m) 1499 goto dontblock; 1500 else 1501 goto release; 1502 } 1503 for (; m != NULL; m = m->m_next) 1504 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1505 m = so->so_rcv.sb_mb; 1506 goto dontblock; 1507 } 1508 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1509 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1510 error = ENOTCONN; 1511 goto release; 1512 } 1513 if (uio->uio_resid == 0) 1514 goto release; 1515 if ((so->so_state & SS_NBIO) || 1516 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1517 error = EWOULDBLOCK; 1518 goto release; 1519 } 1520 SBLASTRECORDCHK(&so->so_rcv); 1521 SBLASTMBUFCHK(&so->so_rcv); 1522 sbunlock(&so->so_rcv); 1523 error = sbwait(&so->so_rcv); 1524 if (error) 1525 goto out; 1526 goto restart; 1527 } 1528 dontblock: 1529 /* 1530 * From this point onward, we maintain 'nextrecord' as a cache of the 1531 * pointer to the next record in the socket buffer. We must keep the 1532 * various socket buffer pointers and local stack versions of the 1533 * pointers in sync, pushing out modifications before dropping the 1534 * socket buffer mutex, and re-reading them when picking it up. 1535 * 1536 * Otherwise, we will race with the network stack appending new data 1537 * or records onto the socket buffer by using inconsistent/stale 1538 * versions of the field, possibly resulting in socket buffer 1539 * corruption. 1540 * 1541 * By holding the high-level sblock(), we prevent simultaneous 1542 * readers from pulling off the front of the socket buffer. 1543 */ 1544 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1545 if (uio->uio_td) 1546 uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++; 1547 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1548 SBLASTRECORDCHK(&so->so_rcv); 1549 SBLASTMBUFCHK(&so->so_rcv); 1550 nextrecord = m->m_nextpkt; 1551 if (pr->pr_flags & PR_ADDR) { 1552 KASSERT(m->m_type == MT_SONAME, 1553 ("m->m_type == %d", m->m_type)); 1554 orig_resid = 0; 1555 if (psa != NULL) 1556 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1557 M_NOWAIT); 1558 if (flags & MSG_PEEK) { 1559 m = m->m_next; 1560 } else { 1561 sbfree(&so->so_rcv, m); 1562 so->so_rcv.sb_mb = m_free(m); 1563 m = so->so_rcv.sb_mb; 1564 sockbuf_pushsync(&so->so_rcv, nextrecord); 1565 } 1566 } 1567 1568 /* 1569 * Process one or more MT_CONTROL mbufs present before any data mbufs 1570 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1571 * just copy the data; if !MSG_PEEK, we call into the protocol to 1572 * perform externalization (or freeing if controlp == NULL). 1573 */ 1574 if (m != NULL && m->m_type == MT_CONTROL) { 1575 struct mbuf *cm = NULL, *cmn; 1576 struct mbuf **cme = &cm; 1577 1578 do { 1579 if (flags & MSG_PEEK) { 1580 if (controlp != NULL) { 1581 *controlp = m_copy(m, 0, m->m_len); 1582 controlp = &(*controlp)->m_next; 1583 } 1584 m = m->m_next; 1585 } else { 1586 sbfree(&so->so_rcv, m); 1587 so->so_rcv.sb_mb = m->m_next; 1588 m->m_next = NULL; 1589 *cme = m; 1590 cme = &(*cme)->m_next; 1591 m = so->so_rcv.sb_mb; 1592 } 1593 } while (m != NULL && m->m_type == MT_CONTROL); 1594 if ((flags & MSG_PEEK) == 0) 1595 sockbuf_pushsync(&so->so_rcv, nextrecord); 1596 while (cm != NULL) { 1597 cmn = cm->m_next; 1598 cm->m_next = NULL; 1599 if (pr->pr_domain->dom_externalize != NULL) { 1600 SOCKBUF_UNLOCK(&so->so_rcv); 1601 error = (*pr->pr_domain->dom_externalize) 1602 (cm, controlp); 1603 SOCKBUF_LOCK(&so->so_rcv); 1604 } else if (controlp != NULL) 1605 *controlp = cm; 1606 else 1607 m_freem(cm); 1608 if (controlp != NULL) { 1609 orig_resid = 0; 1610 while (*controlp != NULL) 1611 controlp = &(*controlp)->m_next; 1612 } 1613 cm = cmn; 1614 } 1615 if (m != NULL) 1616 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1617 else 1618 nextrecord = so->so_rcv.sb_mb; 1619 orig_resid = 0; 1620 } 1621 if (m != NULL) { 1622 if ((flags & MSG_PEEK) == 0) { 1623 KASSERT(m->m_nextpkt == nextrecord, 1624 ("soreceive: post-control, nextrecord !sync")); 1625 if (nextrecord == NULL) { 1626 KASSERT(so->so_rcv.sb_mb == m, 1627 ("soreceive: post-control, sb_mb!=m")); 1628 KASSERT(so->so_rcv.sb_lastrecord == m, 1629 ("soreceive: post-control, lastrecord!=m")); 1630 } 1631 } 1632 type = m->m_type; 1633 if (type == MT_OOBDATA) 1634 flags |= MSG_OOB; 1635 } else { 1636 if ((flags & MSG_PEEK) == 0) { 1637 KASSERT(so->so_rcv.sb_mb == nextrecord, 1638 ("soreceive: sb_mb != nextrecord")); 1639 if (so->so_rcv.sb_mb == NULL) { 1640 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1641 ("soreceive: sb_lastercord != NULL")); 1642 } 1643 } 1644 } 1645 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1646 SBLASTRECORDCHK(&so->so_rcv); 1647 SBLASTMBUFCHK(&so->so_rcv); 1648 1649 /* 1650 * Now continue to read any data mbufs off of the head of the socket 1651 * buffer until the read request is satisfied. Note that 'type' is 1652 * used to store the type of any mbuf reads that have happened so far 1653 * such that soreceive() can stop reading if the type changes, which 1654 * causes soreceive() to return only one of regular data and inline 1655 * out-of-band data in a single socket receive operation. 1656 */ 1657 moff = 0; 1658 offset = 0; 1659 while (m != NULL && uio->uio_resid > 0 && error == 0) { 1660 /* 1661 * If the type of mbuf has changed since the last mbuf 1662 * examined ('type'), end the receive operation. 1663 */ 1664 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1665 if (m->m_type == MT_OOBDATA) { 1666 if (type != MT_OOBDATA) 1667 break; 1668 } else if (type == MT_OOBDATA) 1669 break; 1670 else 1671 KASSERT(m->m_type == MT_DATA, 1672 ("m->m_type == %d", m->m_type)); 1673 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1674 len = uio->uio_resid; 1675 if (so->so_oobmark && len > so->so_oobmark - offset) 1676 len = so->so_oobmark - offset; 1677 if (len > m->m_len - moff) 1678 len = m->m_len - moff; 1679 /* 1680 * If mp is set, just pass back the mbufs. Otherwise copy 1681 * them out via the uio, then free. Sockbuf must be 1682 * consistent here (points to current mbuf, it points to next 1683 * record) when we drop priority; we must note any additions 1684 * to the sockbuf when we block interrupts again. 1685 */ 1686 if (mp == NULL) { 1687 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1688 SBLASTRECORDCHK(&so->so_rcv); 1689 SBLASTMBUFCHK(&so->so_rcv); 1690 SOCKBUF_UNLOCK(&so->so_rcv); 1691 #ifdef ZERO_COPY_SOCKETS 1692 if (so_zero_copy_receive) { 1693 int disposable; 1694 1695 if ((m->m_flags & M_EXT) 1696 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1697 disposable = 1; 1698 else 1699 disposable = 0; 1700 1701 error = uiomoveco(mtod(m, char *) + moff, 1702 (int)len, uio, 1703 disposable); 1704 } else 1705 #endif /* ZERO_COPY_SOCKETS */ 1706 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1707 SOCKBUF_LOCK(&so->so_rcv); 1708 if (error) { 1709 /* 1710 * The MT_SONAME mbuf has already been removed 1711 * from the record, so it is necessary to 1712 * remove the data mbufs, if any, to preserve 1713 * the invariant in the case of PR_ADDR that 1714 * requires MT_SONAME mbufs at the head of 1715 * each record. 1716 */ 1717 if (m && pr->pr_flags & PR_ATOMIC 1718 && ((flags & MSG_PEEK) == 0)) { 1719 (void)sbdroprecord_locked(&so->so_rcv); 1720 } 1721 goto release; 1722 } 1723 } else 1724 uio->uio_resid -= len; 1725 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1726 if (len == m->m_len - moff) { 1727 if (m->m_flags & M_EOR) 1728 flags |= MSG_EOR; 1729 if (flags & MSG_PEEK) { 1730 m = m->m_next; 1731 moff = 0; 1732 } else { 1733 nextrecord = m->m_nextpkt; 1734 sbfree(&so->so_rcv, m); 1735 if (mp != NULL) { 1736 *mp = m; 1737 mp = &m->m_next; 1738 so->so_rcv.sb_mb = m = m->m_next; 1739 *mp = NULL; 1740 } else { 1741 so->so_rcv.sb_mb = m_free(m); 1742 m = so->so_rcv.sb_mb; 1743 } 1744 sockbuf_pushsync(&so->so_rcv, nextrecord); 1745 SBLASTRECORDCHK(&so->so_rcv); 1746 SBLASTMBUFCHK(&so->so_rcv); 1747 } 1748 } else { 1749 if (flags & MSG_PEEK) 1750 moff += len; 1751 else { 1752 if (mp != NULL) { 1753 int copy_flag; 1754 1755 if (flags & MSG_DONTWAIT) 1756 copy_flag = M_DONTWAIT; 1757 else 1758 copy_flag = M_TRYWAIT; 1759 if (copy_flag == M_TRYWAIT) 1760 SOCKBUF_UNLOCK(&so->so_rcv); 1761 *mp = m_copym(m, 0, len, copy_flag); 1762 if (copy_flag == M_TRYWAIT) 1763 SOCKBUF_LOCK(&so->so_rcv); 1764 if (*mp == NULL) { 1765 /* 1766 * m_copym() couldn't 1767 * allocate an mbuf. Adjust 1768 * uio_resid back (it was 1769 * adjusted down by len 1770 * bytes, which we didn't end 1771 * up "copying" over). 1772 */ 1773 uio->uio_resid += len; 1774 break; 1775 } 1776 } 1777 m->m_data += len; 1778 m->m_len -= len; 1779 so->so_rcv.sb_cc -= len; 1780 } 1781 } 1782 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1783 if (so->so_oobmark) { 1784 if ((flags & MSG_PEEK) == 0) { 1785 so->so_oobmark -= len; 1786 if (so->so_oobmark == 0) { 1787 so->so_rcv.sb_state |= SBS_RCVATMARK; 1788 break; 1789 } 1790 } else { 1791 offset += len; 1792 if (offset == so->so_oobmark) 1793 break; 1794 } 1795 } 1796 if (flags & MSG_EOR) 1797 break; 1798 /* 1799 * If the MSG_WAITALL flag is set (for non-atomic socket), we 1800 * must not quit until "uio->uio_resid == 0" or an error 1801 * termination. If a signal/timeout occurs, return with a 1802 * short count but without error. Keep sockbuf locked 1803 * against other readers. 1804 */ 1805 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 1806 !sosendallatonce(so) && nextrecord == NULL) { 1807 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1808 if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE) 1809 break; 1810 /* 1811 * Notify the protocol that some data has been 1812 * drained before blocking. 1813 */ 1814 if (pr->pr_flags & PR_WANTRCVD) { 1815 SOCKBUF_UNLOCK(&so->so_rcv); 1816 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1817 SOCKBUF_LOCK(&so->so_rcv); 1818 } 1819 SBLASTRECORDCHK(&so->so_rcv); 1820 SBLASTMBUFCHK(&so->so_rcv); 1821 error = sbwait(&so->so_rcv); 1822 if (error) 1823 goto release; 1824 m = so->so_rcv.sb_mb; 1825 if (m != NULL) 1826 nextrecord = m->m_nextpkt; 1827 } 1828 } 1829 1830 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1831 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 1832 flags |= MSG_TRUNC; 1833 if ((flags & MSG_PEEK) == 0) 1834 (void) sbdroprecord_locked(&so->so_rcv); 1835 } 1836 if ((flags & MSG_PEEK) == 0) { 1837 if (m == NULL) { 1838 /* 1839 * First part is an inline SB_EMPTY_FIXUP(). Second 1840 * part makes sure sb_lastrecord is up-to-date if 1841 * there is still data in the socket buffer. 1842 */ 1843 so->so_rcv.sb_mb = nextrecord; 1844 if (so->so_rcv.sb_mb == NULL) { 1845 so->so_rcv.sb_mbtail = NULL; 1846 so->so_rcv.sb_lastrecord = NULL; 1847 } else if (nextrecord->m_nextpkt == NULL) 1848 so->so_rcv.sb_lastrecord = nextrecord; 1849 } 1850 SBLASTRECORDCHK(&so->so_rcv); 1851 SBLASTMBUFCHK(&so->so_rcv); 1852 /* 1853 * If soreceive() is being done from the socket callback, 1854 * then don't need to generate ACK to peer to update window, 1855 * since ACK will be generated on return to TCP. 1856 */ 1857 if (!(flags & MSG_SOCALLBCK) && 1858 (pr->pr_flags & PR_WANTRCVD)) { 1859 SOCKBUF_UNLOCK(&so->so_rcv); 1860 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1861 SOCKBUF_LOCK(&so->so_rcv); 1862 } 1863 } 1864 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1865 if (orig_resid == uio->uio_resid && orig_resid && 1866 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 1867 sbunlock(&so->so_rcv); 1868 goto restart; 1869 } 1870 1871 if (flagsp != NULL) 1872 *flagsp |= flags; 1873 release: 1874 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1875 sbunlock(&so->so_rcv); 1876 out: 1877 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1878 SOCKBUF_UNLOCK(&so->so_rcv); 1879 return (error); 1880 } 1881 1882 int 1883 soreceive(so, psa, uio, mp0, controlp, flagsp) 1884 struct socket *so; 1885 struct sockaddr **psa; 1886 struct uio *uio; 1887 struct mbuf **mp0; 1888 struct mbuf **controlp; 1889 int *flagsp; 1890 { 1891 1892 /* XXXRW: Temporary debugging. */ 1893 KASSERT(so->so_proto->pr_usrreqs->pru_soreceive != soreceive, 1894 ("soreceive: protocol calls soreceive")); 1895 1896 return (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0, 1897 controlp, flagsp)); 1898 } 1899 1900 int 1901 soshutdown(so, how) 1902 struct socket *so; 1903 int how; 1904 { 1905 struct protosw *pr = so->so_proto; 1906 1907 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 1908 return (EINVAL); 1909 1910 if (how != SHUT_WR) 1911 sorflush(so); 1912 if (how != SHUT_RD) 1913 return ((*pr->pr_usrreqs->pru_shutdown)(so)); 1914 return (0); 1915 } 1916 1917 void 1918 sorflush(so) 1919 struct socket *so; 1920 { 1921 struct sockbuf *sb = &so->so_rcv; 1922 struct protosw *pr = so->so_proto; 1923 struct sockbuf asb; 1924 1925 /* 1926 * XXXRW: This is quite ugly. Previously, this code made a copy of 1927 * the socket buffer, then zero'd the original to clear the buffer 1928 * fields. However, with mutexes in the socket buffer, this causes 1929 * problems. We only clear the zeroable bits of the original; 1930 * however, we have to initialize and destroy the mutex in the copy 1931 * so that dom_dispose() and sbrelease() can lock t as needed. 1932 */ 1933 SOCKBUF_LOCK(sb); 1934 sb->sb_flags |= SB_NOINTR; 1935 (void) sblock(sb, M_WAITOK); 1936 /* 1937 * socantrcvmore_locked() drops the socket buffer mutex so that it 1938 * can safely perform wakeups. Re-acquire the mutex before 1939 * continuing. 1940 */ 1941 socantrcvmore_locked(so); 1942 SOCKBUF_LOCK(sb); 1943 sbunlock(sb); 1944 /* 1945 * Invalidate/clear most of the sockbuf structure, but leave selinfo 1946 * and mutex data unchanged. 1947 */ 1948 bzero(&asb, offsetof(struct sockbuf, sb_startzero)); 1949 bcopy(&sb->sb_startzero, &asb.sb_startzero, 1950 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1951 bzero(&sb->sb_startzero, 1952 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1953 SOCKBUF_UNLOCK(sb); 1954 1955 SOCKBUF_LOCK_INIT(&asb, "so_rcv"); 1956 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1957 (*pr->pr_domain->dom_dispose)(asb.sb_mb); 1958 sbrelease(&asb, so); 1959 SOCKBUF_LOCK_DESTROY(&asb); 1960 } 1961 1962 /* 1963 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 1964 * additional variant to handle the case where the option value needs to be 1965 * some kind of integer, but not a specific size. In addition to their use 1966 * here, these functions are also called by the protocol-level pr_ctloutput() 1967 * routines. 1968 */ 1969 int 1970 sooptcopyin(sopt, buf, len, minlen) 1971 struct sockopt *sopt; 1972 void *buf; 1973 size_t len; 1974 size_t minlen; 1975 { 1976 size_t valsize; 1977 1978 /* 1979 * If the user gives us more than we wanted, we ignore it, but if we 1980 * don't get the minimum length the caller wants, we return EINVAL. 1981 * On success, sopt->sopt_valsize is set to however much we actually 1982 * retrieved. 1983 */ 1984 if ((valsize = sopt->sopt_valsize) < minlen) 1985 return EINVAL; 1986 if (valsize > len) 1987 sopt->sopt_valsize = valsize = len; 1988 1989 if (sopt->sopt_td != NULL) 1990 return (copyin(sopt->sopt_val, buf, valsize)); 1991 1992 bcopy(sopt->sopt_val, buf, valsize); 1993 return (0); 1994 } 1995 1996 /* 1997 * Kernel version of setsockopt(2). 1998 * 1999 * XXX: optlen is size_t, not socklen_t 2000 */ 2001 int 2002 so_setsockopt(struct socket *so, int level, int optname, void *optval, 2003 size_t optlen) 2004 { 2005 struct sockopt sopt; 2006 2007 sopt.sopt_level = level; 2008 sopt.sopt_name = optname; 2009 sopt.sopt_dir = SOPT_SET; 2010 sopt.sopt_val = optval; 2011 sopt.sopt_valsize = optlen; 2012 sopt.sopt_td = NULL; 2013 return (sosetopt(so, &sopt)); 2014 } 2015 2016 int 2017 sosetopt(so, sopt) 2018 struct socket *so; 2019 struct sockopt *sopt; 2020 { 2021 int error, optval; 2022 struct linger l; 2023 struct timeval tv; 2024 u_long val; 2025 #ifdef MAC 2026 struct mac extmac; 2027 #endif 2028 2029 error = 0; 2030 if (sopt->sopt_level != SOL_SOCKET) { 2031 if (so->so_proto && so->so_proto->pr_ctloutput) 2032 return ((*so->so_proto->pr_ctloutput) 2033 (so, sopt)); 2034 error = ENOPROTOOPT; 2035 } else { 2036 switch (sopt->sopt_name) { 2037 #ifdef INET 2038 case SO_ACCEPTFILTER: 2039 error = do_setopt_accept_filter(so, sopt); 2040 if (error) 2041 goto bad; 2042 break; 2043 #endif 2044 case SO_LINGER: 2045 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 2046 if (error) 2047 goto bad; 2048 2049 SOCK_LOCK(so); 2050 so->so_linger = l.l_linger; 2051 if (l.l_onoff) 2052 so->so_options |= SO_LINGER; 2053 else 2054 so->so_options &= ~SO_LINGER; 2055 SOCK_UNLOCK(so); 2056 break; 2057 2058 case SO_DEBUG: 2059 case SO_KEEPALIVE: 2060 case SO_DONTROUTE: 2061 case SO_USELOOPBACK: 2062 case SO_BROADCAST: 2063 case SO_REUSEADDR: 2064 case SO_REUSEPORT: 2065 case SO_OOBINLINE: 2066 case SO_TIMESTAMP: 2067 case SO_BINTIME: 2068 case SO_NOSIGPIPE: 2069 error = sooptcopyin(sopt, &optval, sizeof optval, 2070 sizeof optval); 2071 if (error) 2072 goto bad; 2073 SOCK_LOCK(so); 2074 if (optval) 2075 so->so_options |= sopt->sopt_name; 2076 else 2077 so->so_options &= ~sopt->sopt_name; 2078 SOCK_UNLOCK(so); 2079 break; 2080 2081 case SO_SNDBUF: 2082 case SO_RCVBUF: 2083 case SO_SNDLOWAT: 2084 case SO_RCVLOWAT: 2085 error = sooptcopyin(sopt, &optval, sizeof optval, 2086 sizeof optval); 2087 if (error) 2088 goto bad; 2089 2090 /* 2091 * Values < 1 make no sense for any of these options, 2092 * so disallow them. 2093 */ 2094 if (optval < 1) { 2095 error = EINVAL; 2096 goto bad; 2097 } 2098 2099 switch (sopt->sopt_name) { 2100 case SO_SNDBUF: 2101 case SO_RCVBUF: 2102 if (sbreserve(sopt->sopt_name == SO_SNDBUF ? 2103 &so->so_snd : &so->so_rcv, (u_long)optval, 2104 so, curthread) == 0) { 2105 error = ENOBUFS; 2106 goto bad; 2107 } 2108 break; 2109 2110 /* 2111 * Make sure the low-water is never greater than the 2112 * high-water. 2113 */ 2114 case SO_SNDLOWAT: 2115 SOCKBUF_LOCK(&so->so_snd); 2116 so->so_snd.sb_lowat = 2117 (optval > so->so_snd.sb_hiwat) ? 2118 so->so_snd.sb_hiwat : optval; 2119 SOCKBUF_UNLOCK(&so->so_snd); 2120 break; 2121 case SO_RCVLOWAT: 2122 SOCKBUF_LOCK(&so->so_rcv); 2123 so->so_rcv.sb_lowat = 2124 (optval > so->so_rcv.sb_hiwat) ? 2125 so->so_rcv.sb_hiwat : optval; 2126 SOCKBUF_UNLOCK(&so->so_rcv); 2127 break; 2128 } 2129 break; 2130 2131 case SO_SNDTIMEO: 2132 case SO_RCVTIMEO: 2133 #ifdef COMPAT_IA32 2134 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 2135 struct timeval32 tv32; 2136 2137 error = sooptcopyin(sopt, &tv32, sizeof tv32, 2138 sizeof tv32); 2139 CP(tv32, tv, tv_sec); 2140 CP(tv32, tv, tv_usec); 2141 } else 2142 #endif 2143 error = sooptcopyin(sopt, &tv, sizeof tv, 2144 sizeof tv); 2145 if (error) 2146 goto bad; 2147 2148 /* assert(hz > 0); */ 2149 if (tv.tv_sec < 0 || tv.tv_sec > INT_MAX / hz || 2150 tv.tv_usec < 0 || tv.tv_usec >= 1000000) { 2151 error = EDOM; 2152 goto bad; 2153 } 2154 /* assert(tick > 0); */ 2155 /* assert(ULONG_MAX - INT_MAX >= 1000000); */ 2156 val = (u_long)(tv.tv_sec * hz) + tv.tv_usec / tick; 2157 if (val > INT_MAX) { 2158 error = EDOM; 2159 goto bad; 2160 } 2161 if (val == 0 && tv.tv_usec != 0) 2162 val = 1; 2163 2164 switch (sopt->sopt_name) { 2165 case SO_SNDTIMEO: 2166 so->so_snd.sb_timeo = val; 2167 break; 2168 case SO_RCVTIMEO: 2169 so->so_rcv.sb_timeo = val; 2170 break; 2171 } 2172 break; 2173 2174 case SO_LABEL: 2175 #ifdef MAC 2176 error = sooptcopyin(sopt, &extmac, sizeof extmac, 2177 sizeof extmac); 2178 if (error) 2179 goto bad; 2180 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 2181 so, &extmac); 2182 #else 2183 error = EOPNOTSUPP; 2184 #endif 2185 break; 2186 2187 default: 2188 error = ENOPROTOOPT; 2189 break; 2190 } 2191 if (error == 0 && so->so_proto != NULL && 2192 so->so_proto->pr_ctloutput != NULL) { 2193 (void) ((*so->so_proto->pr_ctloutput) 2194 (so, sopt)); 2195 } 2196 } 2197 bad: 2198 return (error); 2199 } 2200 2201 /* 2202 * Helper routine for getsockopt. 2203 */ 2204 int 2205 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 2206 { 2207 int error; 2208 size_t valsize; 2209 2210 error = 0; 2211 2212 /* 2213 * Documented get behavior is that we always return a value, possibly 2214 * truncated to fit in the user's buffer. Traditional behavior is 2215 * that we always tell the user precisely how much we copied, rather 2216 * than something useful like the total amount we had available for 2217 * her. Note that this interface is not idempotent; the entire 2218 * answer must generated ahead of time. 2219 */ 2220 valsize = min(len, sopt->sopt_valsize); 2221 sopt->sopt_valsize = valsize; 2222 if (sopt->sopt_val != NULL) { 2223 if (sopt->sopt_td != NULL) 2224 error = copyout(buf, sopt->sopt_val, valsize); 2225 else 2226 bcopy(buf, sopt->sopt_val, valsize); 2227 } 2228 return (error); 2229 } 2230 2231 int 2232 sogetopt(so, sopt) 2233 struct socket *so; 2234 struct sockopt *sopt; 2235 { 2236 int error, optval; 2237 struct linger l; 2238 struct timeval tv; 2239 #ifdef MAC 2240 struct mac extmac; 2241 #endif 2242 2243 error = 0; 2244 if (sopt->sopt_level != SOL_SOCKET) { 2245 if (so->so_proto && so->so_proto->pr_ctloutput) { 2246 return ((*so->so_proto->pr_ctloutput) 2247 (so, sopt)); 2248 } else 2249 return (ENOPROTOOPT); 2250 } else { 2251 switch (sopt->sopt_name) { 2252 #ifdef INET 2253 case SO_ACCEPTFILTER: 2254 error = do_getopt_accept_filter(so, sopt); 2255 break; 2256 #endif 2257 case SO_LINGER: 2258 SOCK_LOCK(so); 2259 l.l_onoff = so->so_options & SO_LINGER; 2260 l.l_linger = so->so_linger; 2261 SOCK_UNLOCK(so); 2262 error = sooptcopyout(sopt, &l, sizeof l); 2263 break; 2264 2265 case SO_USELOOPBACK: 2266 case SO_DONTROUTE: 2267 case SO_DEBUG: 2268 case SO_KEEPALIVE: 2269 case SO_REUSEADDR: 2270 case SO_REUSEPORT: 2271 case SO_BROADCAST: 2272 case SO_OOBINLINE: 2273 case SO_ACCEPTCONN: 2274 case SO_TIMESTAMP: 2275 case SO_BINTIME: 2276 case SO_NOSIGPIPE: 2277 optval = so->so_options & sopt->sopt_name; 2278 integer: 2279 error = sooptcopyout(sopt, &optval, sizeof optval); 2280 break; 2281 2282 case SO_TYPE: 2283 optval = so->so_type; 2284 goto integer; 2285 2286 case SO_ERROR: 2287 SOCK_LOCK(so); 2288 optval = so->so_error; 2289 so->so_error = 0; 2290 SOCK_UNLOCK(so); 2291 goto integer; 2292 2293 case SO_SNDBUF: 2294 optval = so->so_snd.sb_hiwat; 2295 goto integer; 2296 2297 case SO_RCVBUF: 2298 optval = so->so_rcv.sb_hiwat; 2299 goto integer; 2300 2301 case SO_SNDLOWAT: 2302 optval = so->so_snd.sb_lowat; 2303 goto integer; 2304 2305 case SO_RCVLOWAT: 2306 optval = so->so_rcv.sb_lowat; 2307 goto integer; 2308 2309 case SO_SNDTIMEO: 2310 case SO_RCVTIMEO: 2311 optval = (sopt->sopt_name == SO_SNDTIMEO ? 2312 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 2313 2314 tv.tv_sec = optval / hz; 2315 tv.tv_usec = (optval % hz) * tick; 2316 #ifdef COMPAT_IA32 2317 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 2318 struct timeval32 tv32; 2319 2320 CP(tv, tv32, tv_sec); 2321 CP(tv, tv32, tv_usec); 2322 error = sooptcopyout(sopt, &tv32, sizeof tv32); 2323 } else 2324 #endif 2325 error = sooptcopyout(sopt, &tv, sizeof tv); 2326 break; 2327 2328 case SO_LABEL: 2329 #ifdef MAC 2330 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2331 sizeof(extmac)); 2332 if (error) 2333 return (error); 2334 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 2335 so, &extmac); 2336 if (error) 2337 return (error); 2338 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2339 #else 2340 error = EOPNOTSUPP; 2341 #endif 2342 break; 2343 2344 case SO_PEERLABEL: 2345 #ifdef MAC 2346 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2347 sizeof(extmac)); 2348 if (error) 2349 return (error); 2350 error = mac_getsockopt_peerlabel( 2351 sopt->sopt_td->td_ucred, so, &extmac); 2352 if (error) 2353 return (error); 2354 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2355 #else 2356 error = EOPNOTSUPP; 2357 #endif 2358 break; 2359 2360 case SO_LISTENQLIMIT: 2361 optval = so->so_qlimit; 2362 goto integer; 2363 2364 case SO_LISTENQLEN: 2365 optval = so->so_qlen; 2366 goto integer; 2367 2368 case SO_LISTENINCQLEN: 2369 optval = so->so_incqlen; 2370 goto integer; 2371 2372 default: 2373 error = ENOPROTOOPT; 2374 break; 2375 } 2376 return (error); 2377 } 2378 } 2379 2380 /* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */ 2381 int 2382 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 2383 { 2384 struct mbuf *m, *m_prev; 2385 int sopt_size = sopt->sopt_valsize; 2386 2387 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2388 if (m == NULL) 2389 return ENOBUFS; 2390 if (sopt_size > MLEN) { 2391 MCLGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT); 2392 if ((m->m_flags & M_EXT) == 0) { 2393 m_free(m); 2394 return ENOBUFS; 2395 } 2396 m->m_len = min(MCLBYTES, sopt_size); 2397 } else { 2398 m->m_len = min(MLEN, sopt_size); 2399 } 2400 sopt_size -= m->m_len; 2401 *mp = m; 2402 m_prev = m; 2403 2404 while (sopt_size) { 2405 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2406 if (m == NULL) { 2407 m_freem(*mp); 2408 return ENOBUFS; 2409 } 2410 if (sopt_size > MLEN) { 2411 MCLGET(m, sopt->sopt_td != NULL ? M_TRYWAIT : 2412 M_DONTWAIT); 2413 if ((m->m_flags & M_EXT) == 0) { 2414 m_freem(m); 2415 m_freem(*mp); 2416 return ENOBUFS; 2417 } 2418 m->m_len = min(MCLBYTES, sopt_size); 2419 } else { 2420 m->m_len = min(MLEN, sopt_size); 2421 } 2422 sopt_size -= m->m_len; 2423 m_prev->m_next = m; 2424 m_prev = m; 2425 } 2426 return (0); 2427 } 2428 2429 /* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */ 2430 int 2431 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 2432 { 2433 struct mbuf *m0 = m; 2434 2435 if (sopt->sopt_val == NULL) 2436 return (0); 2437 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2438 if (sopt->sopt_td != NULL) { 2439 int error; 2440 2441 error = copyin(sopt->sopt_val, mtod(m, char *), 2442 m->m_len); 2443 if (error != 0) { 2444 m_freem(m0); 2445 return(error); 2446 } 2447 } else 2448 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 2449 sopt->sopt_valsize -= m->m_len; 2450 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2451 m = m->m_next; 2452 } 2453 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 2454 panic("ip6_sooptmcopyin"); 2455 return (0); 2456 } 2457 2458 /* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */ 2459 int 2460 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 2461 { 2462 struct mbuf *m0 = m; 2463 size_t valsize = 0; 2464 2465 if (sopt->sopt_val == NULL) 2466 return (0); 2467 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2468 if (sopt->sopt_td != NULL) { 2469 int error; 2470 2471 error = copyout(mtod(m, char *), sopt->sopt_val, 2472 m->m_len); 2473 if (error != 0) { 2474 m_freem(m0); 2475 return(error); 2476 } 2477 } else 2478 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 2479 sopt->sopt_valsize -= m->m_len; 2480 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2481 valsize += m->m_len; 2482 m = m->m_next; 2483 } 2484 if (m != NULL) { 2485 /* enough soopt buffer should be given from user-land */ 2486 m_freem(m0); 2487 return(EINVAL); 2488 } 2489 sopt->sopt_valsize = valsize; 2490 return (0); 2491 } 2492 2493 /* 2494 * sohasoutofband(): protocol notifies socket layer of the arrival of new 2495 * out-of-band data, which will then notify socket consumers. 2496 */ 2497 void 2498 sohasoutofband(so) 2499 struct socket *so; 2500 { 2501 if (so->so_sigio != NULL) 2502 pgsigio(&so->so_sigio, SIGURG, 0); 2503 selwakeuppri(&so->so_rcv.sb_sel, PSOCK); 2504 } 2505 2506 int 2507 sopoll(struct socket *so, int events, struct ucred *active_cred, 2508 struct thread *td) 2509 { 2510 2511 /* XXXRW: Temporary debugging. */ 2512 KASSERT(so->so_proto->pr_usrreqs->pru_sopoll != sopoll, 2513 ("sopoll: protocol calls sopoll")); 2514 2515 return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred, 2516 td)); 2517 } 2518 2519 int 2520 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, 2521 struct thread *td) 2522 { 2523 int revents = 0; 2524 2525 SOCKBUF_LOCK(&so->so_snd); 2526 SOCKBUF_LOCK(&so->so_rcv); 2527 if (events & (POLLIN | POLLRDNORM)) 2528 if (soreadable(so)) 2529 revents |= events & (POLLIN | POLLRDNORM); 2530 2531 if (events & POLLINIGNEOF) 2532 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat || 2533 !TAILQ_EMPTY(&so->so_comp) || so->so_error) 2534 revents |= POLLINIGNEOF; 2535 2536 if (events & (POLLOUT | POLLWRNORM)) 2537 if (sowriteable(so)) 2538 revents |= events & (POLLOUT | POLLWRNORM); 2539 2540 if (events & (POLLPRI | POLLRDBAND)) 2541 if (so->so_oobmark || (so->so_rcv.sb_state & SBS_RCVATMARK)) 2542 revents |= events & (POLLPRI | POLLRDBAND); 2543 2544 if (revents == 0) { 2545 if (events & 2546 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | 2547 POLLRDBAND)) { 2548 selrecord(td, &so->so_rcv.sb_sel); 2549 so->so_rcv.sb_flags |= SB_SEL; 2550 } 2551 2552 if (events & (POLLOUT | POLLWRNORM)) { 2553 selrecord(td, &so->so_snd.sb_sel); 2554 so->so_snd.sb_flags |= SB_SEL; 2555 } 2556 } 2557 2558 SOCKBUF_UNLOCK(&so->so_rcv); 2559 SOCKBUF_UNLOCK(&so->so_snd); 2560 return (revents); 2561 } 2562 2563 int 2564 soo_kqfilter(struct file *fp, struct knote *kn) 2565 { 2566 struct socket *so = kn->kn_fp->f_data; 2567 struct sockbuf *sb; 2568 2569 switch (kn->kn_filter) { 2570 case EVFILT_READ: 2571 if (so->so_options & SO_ACCEPTCONN) 2572 kn->kn_fop = &solisten_filtops; 2573 else 2574 kn->kn_fop = &soread_filtops; 2575 sb = &so->so_rcv; 2576 break; 2577 case EVFILT_WRITE: 2578 kn->kn_fop = &sowrite_filtops; 2579 sb = &so->so_snd; 2580 break; 2581 default: 2582 return (EINVAL); 2583 } 2584 2585 SOCKBUF_LOCK(sb); 2586 knlist_add(&sb->sb_sel.si_note, kn, 1); 2587 sb->sb_flags |= SB_KNOTE; 2588 SOCKBUF_UNLOCK(sb); 2589 return (0); 2590 } 2591 2592 static void 2593 filt_sordetach(struct knote *kn) 2594 { 2595 struct socket *so = kn->kn_fp->f_data; 2596 2597 SOCKBUF_LOCK(&so->so_rcv); 2598 knlist_remove(&so->so_rcv.sb_sel.si_note, kn, 1); 2599 if (knlist_empty(&so->so_rcv.sb_sel.si_note)) 2600 so->so_rcv.sb_flags &= ~SB_KNOTE; 2601 SOCKBUF_UNLOCK(&so->so_rcv); 2602 } 2603 2604 /*ARGSUSED*/ 2605 static int 2606 filt_soread(struct knote *kn, long hint) 2607 { 2608 struct socket *so; 2609 2610 so = kn->kn_fp->f_data; 2611 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2612 2613 kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl; 2614 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2615 kn->kn_flags |= EV_EOF; 2616 kn->kn_fflags = so->so_error; 2617 return (1); 2618 } else if (so->so_error) /* temporary udp error */ 2619 return (1); 2620 else if (kn->kn_sfflags & NOTE_LOWAT) 2621 return (kn->kn_data >= kn->kn_sdata); 2622 else 2623 return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat); 2624 } 2625 2626 static void 2627 filt_sowdetach(struct knote *kn) 2628 { 2629 struct socket *so = kn->kn_fp->f_data; 2630 2631 SOCKBUF_LOCK(&so->so_snd); 2632 knlist_remove(&so->so_snd.sb_sel.si_note, kn, 1); 2633 if (knlist_empty(&so->so_snd.sb_sel.si_note)) 2634 so->so_snd.sb_flags &= ~SB_KNOTE; 2635 SOCKBUF_UNLOCK(&so->so_snd); 2636 } 2637 2638 /*ARGSUSED*/ 2639 static int 2640 filt_sowrite(struct knote *kn, long hint) 2641 { 2642 struct socket *so; 2643 2644 so = kn->kn_fp->f_data; 2645 SOCKBUF_LOCK_ASSERT(&so->so_snd); 2646 kn->kn_data = sbspace(&so->so_snd); 2647 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2648 kn->kn_flags |= EV_EOF; 2649 kn->kn_fflags = so->so_error; 2650 return (1); 2651 } else if (so->so_error) /* temporary udp error */ 2652 return (1); 2653 else if (((so->so_state & SS_ISCONNECTED) == 0) && 2654 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 2655 return (0); 2656 else if (kn->kn_sfflags & NOTE_LOWAT) 2657 return (kn->kn_data >= kn->kn_sdata); 2658 else 2659 return (kn->kn_data >= so->so_snd.sb_lowat); 2660 } 2661 2662 /*ARGSUSED*/ 2663 static int 2664 filt_solisten(struct knote *kn, long hint) 2665 { 2666 struct socket *so = kn->kn_fp->f_data; 2667 2668 kn->kn_data = so->so_qlen; 2669 return (! TAILQ_EMPTY(&so->so_comp)); 2670 } 2671 2672 int 2673 socheckuid(struct socket *so, uid_t uid) 2674 { 2675 2676 if (so == NULL) 2677 return (EPERM); 2678 if (so->so_cred->cr_uid != uid) 2679 return (EPERM); 2680 return (0); 2681 } 2682 2683 static int 2684 somaxconn_sysctl(SYSCTL_HANDLER_ARGS) 2685 { 2686 int error; 2687 int val; 2688 2689 val = somaxconn; 2690 error = sysctl_handle_int(oidp, &val, sizeof(int), req); 2691 if (error || !req->newptr ) 2692 return (error); 2693 2694 if (val < 1 || val > USHRT_MAX) 2695 return (EINVAL); 2696 2697 somaxconn = val; 2698 return (0); 2699 } 2700