1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. 4 * Copyright (c) 2004 The FreeBSD Foundation 5 * Copyright (c) 2004-2006 Robert N. M. Watson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 4. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pru_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pru_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pru_attach() has 50 * been successfully called. If pru_attach() returned an error, 51 * pru_detach() will not be called. Socket layer private. 52 * 53 * pru_abort() and pru_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pru_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 */ 96 97 #include <sys/cdefs.h> 98 __FBSDID("$FreeBSD$"); 99 100 #include "opt_inet.h" 101 #include "opt_mac.h" 102 #include "opt_zero.h" 103 #include "opt_compat.h" 104 105 #include <sys/param.h> 106 #include <sys/systm.h> 107 #include <sys/fcntl.h> 108 #include <sys/limits.h> 109 #include <sys/lock.h> 110 #include <sys/mac.h> 111 #include <sys/malloc.h> 112 #include <sys/mbuf.h> 113 #include <sys/mutex.h> 114 #include <sys/domain.h> 115 #include <sys/file.h> /* for struct knote */ 116 #include <sys/kernel.h> 117 #include <sys/event.h> 118 #include <sys/eventhandler.h> 119 #include <sys/poll.h> 120 #include <sys/proc.h> 121 #include <sys/protosw.h> 122 #include <sys/socket.h> 123 #include <sys/socketvar.h> 124 #include <sys/resourcevar.h> 125 #include <sys/signalvar.h> 126 #include <sys/sysctl.h> 127 #include <sys/uio.h> 128 #include <sys/jail.h> 129 130 #include <security/mac/mac_framework.h> 131 132 #include <vm/uma.h> 133 134 #ifdef COMPAT_IA32 135 #include <sys/mount.h> 136 #include <compat/freebsd32/freebsd32.h> 137 138 extern struct sysentvec ia32_freebsd_sysvec; 139 #endif 140 141 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 142 int flags); 143 144 static void filt_sordetach(struct knote *kn); 145 static int filt_soread(struct knote *kn, long hint); 146 static void filt_sowdetach(struct knote *kn); 147 static int filt_sowrite(struct knote *kn, long hint); 148 static int filt_solisten(struct knote *kn, long hint); 149 150 static struct filterops solisten_filtops = 151 { 1, NULL, filt_sordetach, filt_solisten }; 152 static struct filterops soread_filtops = 153 { 1, NULL, filt_sordetach, filt_soread }; 154 static struct filterops sowrite_filtops = 155 { 1, NULL, filt_sowdetach, filt_sowrite }; 156 157 uma_zone_t socket_zone; 158 so_gen_t so_gencnt; /* generation count for sockets */ 159 160 int maxsockets; 161 162 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 163 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 164 165 static int somaxconn = SOMAXCONN; 166 static int somaxconn_sysctl(SYSCTL_HANDLER_ARGS); 167 /* XXX: we dont have SYSCTL_USHORT */ 168 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, CTLTYPE_UINT | CTLFLAG_RW, 169 0, sizeof(int), somaxconn_sysctl, "I", "Maximum pending socket connection " 170 "queue size"); 171 static int numopensockets; 172 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 173 &numopensockets, 0, "Number of open sockets"); 174 #ifdef ZERO_COPY_SOCKETS 175 /* These aren't static because they're used in other files. */ 176 int so_zero_copy_send = 1; 177 int so_zero_copy_receive = 1; 178 SYSCTL_NODE(_kern_ipc, OID_AUTO, zero_copy, CTLFLAG_RD, 0, 179 "Zero copy controls"); 180 SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, receive, CTLFLAG_RW, 181 &so_zero_copy_receive, 0, "Enable zero copy receive"); 182 SYSCTL_INT(_kern_ipc_zero_copy, OID_AUTO, send, CTLFLAG_RW, 183 &so_zero_copy_send, 0, "Enable zero copy send"); 184 #endif /* ZERO_COPY_SOCKETS */ 185 186 /* 187 * accept_mtx locks down per-socket fields relating to accept queues. See 188 * socketvar.h for an annotation of the protected fields of struct socket. 189 */ 190 struct mtx accept_mtx; 191 MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 192 193 /* 194 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 195 * so_gencnt field. 196 */ 197 static struct mtx so_global_mtx; 198 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 199 200 /* 201 * General IPC sysctl name space, used by sockets and a variety of other IPC 202 * types. 203 */ 204 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC"); 205 206 /* 207 * Sysctl to get and set the maximum global sockets limit. Notify protocols 208 * of the change so that they can update their dependent limits as required. 209 */ 210 static int 211 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 212 { 213 int error, newmaxsockets; 214 215 newmaxsockets = maxsockets; 216 error = sysctl_handle_int(oidp, &newmaxsockets, sizeof(int), req); 217 if (error == 0 && req->newptr) { 218 if (newmaxsockets > maxsockets) { 219 maxsockets = newmaxsockets; 220 if (maxsockets > ((maxfiles / 4) * 3)) { 221 maxfiles = (maxsockets * 5) / 4; 222 maxfilesperproc = (maxfiles * 9) / 10; 223 } 224 EVENTHANDLER_INVOKE(maxsockets_change); 225 } else 226 error = EINVAL; 227 } 228 return (error); 229 } 230 231 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, CTLTYPE_INT|CTLFLAG_RW, 232 &maxsockets, 0, sysctl_maxsockets, "IU", 233 "Maximum number of sockets avaliable"); 234 235 /* 236 * Initialise maxsockets. 237 */ 238 static void init_maxsockets(void *ignored) 239 { 240 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 241 maxsockets = imax(maxsockets, imax(maxfiles, nmbclusters)); 242 } 243 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 244 245 /* 246 * Socket operation routines. These routines are called by the routines in 247 * sys_socket.c or from a system process, and implement the semantics of 248 * socket operations by switching out to the protocol specific routines. 249 */ 250 251 /* 252 * Get a socket structure from our zone, and initialize it. Note that it 253 * would probably be better to allocate socket and PCB at the same time, but 254 * I'm not convinced that all the protocols can be easily modified to do 255 * this. 256 * 257 * soalloc() returns a socket with a ref count of 0. 258 */ 259 static struct socket * 260 soalloc(int mflags) 261 { 262 struct socket *so; 263 264 so = uma_zalloc(socket_zone, mflags | M_ZERO); 265 if (so == NULL) 266 return (NULL); 267 #ifdef MAC 268 if (mac_init_socket(so, mflags) != 0) { 269 uma_zfree(socket_zone, so); 270 return (NULL); 271 } 272 #endif 273 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 274 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 275 TAILQ_INIT(&so->so_aiojobq); 276 mtx_lock(&so_global_mtx); 277 so->so_gencnt = ++so_gencnt; 278 ++numopensockets; 279 mtx_unlock(&so_global_mtx); 280 return (so); 281 } 282 283 /* 284 * Free the storage associated with a socket at the socket layer, tear down 285 * locks, labels, etc. All protocol state is assumed already to have been 286 * torn down (and possibly never set up) by the caller. 287 */ 288 static void 289 sodealloc(struct socket *so) 290 { 291 292 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 293 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 294 295 mtx_lock(&so_global_mtx); 296 so->so_gencnt = ++so_gencnt; 297 --numopensockets; /* Could be below, but faster here. */ 298 mtx_unlock(&so_global_mtx); 299 if (so->so_rcv.sb_hiwat) 300 (void)chgsbsize(so->so_cred->cr_uidinfo, 301 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 302 if (so->so_snd.sb_hiwat) 303 (void)chgsbsize(so->so_cred->cr_uidinfo, 304 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 305 #ifdef INET 306 /* remove acccept filter if one is present. */ 307 if (so->so_accf != NULL) 308 do_setopt_accept_filter(so, NULL); 309 #endif 310 #ifdef MAC 311 mac_destroy_socket(so); 312 #endif 313 crfree(so->so_cred); 314 SOCKBUF_LOCK_DESTROY(&so->so_snd); 315 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 316 uma_zfree(socket_zone, so); 317 } 318 319 /* 320 * socreate returns a socket with a ref count of 1. The socket should be 321 * closed with soclose(). 322 */ 323 int 324 socreate(dom, aso, type, proto, cred, td) 325 int dom; 326 struct socket **aso; 327 int type; 328 int proto; 329 struct ucred *cred; 330 struct thread *td; 331 { 332 struct protosw *prp; 333 struct socket *so; 334 int error; 335 336 if (proto) 337 prp = pffindproto(dom, proto, type); 338 else 339 prp = pffindtype(dom, type); 340 341 if (prp == NULL || prp->pr_usrreqs->pru_attach == NULL || 342 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 343 return (EPROTONOSUPPORT); 344 345 if (jailed(cred) && jail_socket_unixiproute_only && 346 prp->pr_domain->dom_family != PF_LOCAL && 347 prp->pr_domain->dom_family != PF_INET && 348 prp->pr_domain->dom_family != PF_ROUTE) { 349 return (EPROTONOSUPPORT); 350 } 351 352 if (prp->pr_type != type) 353 return (EPROTOTYPE); 354 so = soalloc(M_WAITOK); 355 if (so == NULL) 356 return (ENOBUFS); 357 358 TAILQ_INIT(&so->so_incomp); 359 TAILQ_INIT(&so->so_comp); 360 so->so_type = type; 361 so->so_cred = crhold(cred); 362 so->so_proto = prp; 363 #ifdef MAC 364 mac_create_socket(cred, so); 365 #endif 366 knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv), 367 NULL, NULL, NULL); 368 knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd), 369 NULL, NULL, NULL); 370 so->so_count = 1; 371 /* 372 * Auto-sizing of socket buffers is managed by the protocols and 373 * the appropriate flags must be set in the pru_attach function. 374 */ 375 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 376 if (error) { 377 KASSERT(so->so_count == 1, ("socreate: so_count %d", 378 so->so_count)); 379 so->so_count = 0; 380 sodealloc(so); 381 return (error); 382 } 383 *aso = so; 384 return (0); 385 } 386 387 #ifdef REGRESSION 388 static int regression_sonewconn_earlytest = 1; 389 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 390 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 391 #endif 392 393 /* 394 * When an attempt at a new connection is noted on a socket which accepts 395 * connections, sonewconn is called. If the connection is possible (subject 396 * to space constraints, etc.) then we allocate a new structure, propoerly 397 * linked into the data structure of the original socket, and return this. 398 * Connstatus may be 0, or SO_ISCONFIRMING, or SO_ISCONNECTED. 399 * 400 * Note: the ref count on the socket is 0 on return. 401 */ 402 struct socket * 403 sonewconn(head, connstatus) 404 register struct socket *head; 405 int connstatus; 406 { 407 register struct socket *so; 408 int over; 409 410 ACCEPT_LOCK(); 411 over = (head->so_qlen > 3 * head->so_qlimit / 2); 412 ACCEPT_UNLOCK(); 413 #ifdef REGRESSION 414 if (regression_sonewconn_earlytest && over) 415 #else 416 if (over) 417 #endif 418 return (NULL); 419 so = soalloc(M_NOWAIT); 420 if (so == NULL) 421 return (NULL); 422 if ((head->so_options & SO_ACCEPTFILTER) != 0) 423 connstatus = 0; 424 so->so_head = head; 425 so->so_type = head->so_type; 426 so->so_options = head->so_options &~ SO_ACCEPTCONN; 427 so->so_linger = head->so_linger; 428 so->so_state = head->so_state | SS_NOFDREF; 429 so->so_proto = head->so_proto; 430 so->so_cred = crhold(head->so_cred); 431 #ifdef MAC 432 SOCK_LOCK(head); 433 mac_create_socket_from_socket(head, so); 434 SOCK_UNLOCK(head); 435 #endif 436 knlist_init(&so->so_rcv.sb_sel.si_note, SOCKBUF_MTX(&so->so_rcv), 437 NULL, NULL, NULL); 438 knlist_init(&so->so_snd.sb_sel.si_note, SOCKBUF_MTX(&so->so_snd), 439 NULL, NULL, NULL); 440 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat) || 441 (*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 442 sodealloc(so); 443 return (NULL); 444 } 445 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 446 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 447 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 448 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 449 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 450 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 451 so->so_state |= connstatus; 452 ACCEPT_LOCK(); 453 if (connstatus) { 454 TAILQ_INSERT_TAIL(&head->so_comp, so, so_list); 455 so->so_qstate |= SQ_COMP; 456 head->so_qlen++; 457 } else { 458 /* 459 * Keep removing sockets from the head until there's room for 460 * us to insert on the tail. In pre-locking revisions, this 461 * was a simple if(), but as we could be racing with other 462 * threads and soabort() requires dropping locks, we must 463 * loop waiting for the condition to be true. 464 */ 465 while (head->so_incqlen > head->so_qlimit) { 466 struct socket *sp; 467 sp = TAILQ_FIRST(&head->so_incomp); 468 TAILQ_REMOVE(&head->so_incomp, sp, so_list); 469 head->so_incqlen--; 470 sp->so_qstate &= ~SQ_INCOMP; 471 sp->so_head = NULL; 472 ACCEPT_UNLOCK(); 473 soabort(sp); 474 ACCEPT_LOCK(); 475 } 476 TAILQ_INSERT_TAIL(&head->so_incomp, so, so_list); 477 so->so_qstate |= SQ_INCOMP; 478 head->so_incqlen++; 479 } 480 ACCEPT_UNLOCK(); 481 if (connstatus) { 482 sorwakeup(head); 483 wakeup_one(&head->so_timeo); 484 } 485 return (so); 486 } 487 488 int 489 sobind(so, nam, td) 490 struct socket *so; 491 struct sockaddr *nam; 492 struct thread *td; 493 { 494 495 return ((*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td)); 496 } 497 498 /* 499 * solisten() transitions a socket from a non-listening state to a listening 500 * state, but can also be used to update the listen queue depth on an 501 * existing listen socket. The protocol will call back into the sockets 502 * layer using solisten_proto_check() and solisten_proto() to check and set 503 * socket-layer listen state. Call backs are used so that the protocol can 504 * acquire both protocol and socket layer locks in whatever order is required 505 * by the protocol. 506 * 507 * Protocol implementors are advised to hold the socket lock across the 508 * socket-layer test and set to avoid races at the socket layer. 509 */ 510 int 511 solisten(so, backlog, td) 512 struct socket *so; 513 int backlog; 514 struct thread *td; 515 { 516 517 return ((*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td)); 518 } 519 520 int 521 solisten_proto_check(so) 522 struct socket *so; 523 { 524 525 SOCK_LOCK_ASSERT(so); 526 527 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 528 SS_ISDISCONNECTING)) 529 return (EINVAL); 530 return (0); 531 } 532 533 void 534 solisten_proto(so, backlog) 535 struct socket *so; 536 int backlog; 537 { 538 539 SOCK_LOCK_ASSERT(so); 540 541 if (backlog < 0 || backlog > somaxconn) 542 backlog = somaxconn; 543 so->so_qlimit = backlog; 544 so->so_options |= SO_ACCEPTCONN; 545 } 546 547 /* 548 * Attempt to free a socket. This should really be sotryfree(). 549 * 550 * sofree() will succeed if: 551 * 552 * - There are no outstanding file descriptor references or related consumers 553 * (so_count == 0). 554 * 555 * - The socket has been closed by user space, if ever open (SS_NOFDREF). 556 * 557 * - The protocol does not have an outstanding strong reference on the socket 558 * (SS_PROTOREF). 559 * 560 * - The socket is not in a completed connection queue, so a process has been 561 * notified that it is present. If it is removed, the user process may 562 * block in accept() despite select() saying the socket was ready. 563 * 564 * Otherwise, it will quietly abort so that a future call to sofree(), when 565 * conditions are right, can succeed. 566 */ 567 void 568 sofree(so) 569 struct socket *so; 570 { 571 struct protosw *pr = so->so_proto; 572 struct socket *head; 573 574 ACCEPT_LOCK_ASSERT(); 575 SOCK_LOCK_ASSERT(so); 576 577 if ((so->so_state & SS_NOFDREF) == 0 || so->so_count != 0 || 578 (so->so_state & SS_PROTOREF) || (so->so_qstate & SQ_COMP)) { 579 SOCK_UNLOCK(so); 580 ACCEPT_UNLOCK(); 581 return; 582 } 583 584 head = so->so_head; 585 if (head != NULL) { 586 KASSERT((so->so_qstate & SQ_COMP) != 0 || 587 (so->so_qstate & SQ_INCOMP) != 0, 588 ("sofree: so_head != NULL, but neither SQ_COMP nor " 589 "SQ_INCOMP")); 590 KASSERT((so->so_qstate & SQ_COMP) == 0 || 591 (so->so_qstate & SQ_INCOMP) == 0, 592 ("sofree: so->so_qstate is SQ_COMP and also SQ_INCOMP")); 593 TAILQ_REMOVE(&head->so_incomp, so, so_list); 594 head->so_incqlen--; 595 so->so_qstate &= ~SQ_INCOMP; 596 so->so_head = NULL; 597 } 598 KASSERT((so->so_qstate & SQ_COMP) == 0 && 599 (so->so_qstate & SQ_INCOMP) == 0, 600 ("sofree: so_head == NULL, but still SQ_COMP(%d) or SQ_INCOMP(%d)", 601 so->so_qstate & SQ_COMP, so->so_qstate & SQ_INCOMP)); 602 if (so->so_options & SO_ACCEPTCONN) { 603 KASSERT((TAILQ_EMPTY(&so->so_comp)), ("sofree: so_comp populated")); 604 KASSERT((TAILQ_EMPTY(&so->so_incomp)), ("sofree: so_comp populated")); 605 } 606 SOCK_UNLOCK(so); 607 ACCEPT_UNLOCK(); 608 609 /* 610 * From this point on, we assume that no other references to this 611 * socket exist anywhere else in the stack. Therefore, no locks need 612 * to be acquired or held. 613 * 614 * We used to do a lot of socket buffer and socket locking here, as 615 * well as invoke sorflush() and perform wakeups. The direct call to 616 * dom_dispose() and sbrelease_internal() are an inlining of what was 617 * necessary from sorflush(). 618 * 619 * Notice that the socket buffer and kqueue state are torn down 620 * before calling pru_detach. This means that protocols shold not 621 * assume they can perform socket wakeups, etc, in their detach 622 * code. 623 */ 624 KASSERT((so->so_snd.sb_flags & SB_LOCK) == 0, ("sofree: snd sblock")); 625 KASSERT((so->so_rcv.sb_flags & SB_LOCK) == 0, ("sofree: rcv sblock")); 626 sbdestroy(&so->so_snd, so); 627 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 628 (*pr->pr_domain->dom_dispose)(so->so_rcv.sb_mb); 629 sbdestroy(&so->so_rcv, so); 630 if (pr->pr_usrreqs->pru_detach != NULL) 631 (*pr->pr_usrreqs->pru_detach)(so); 632 knlist_destroy(&so->so_rcv.sb_sel.si_note); 633 knlist_destroy(&so->so_snd.sb_sel.si_note); 634 sodealloc(so); 635 } 636 637 /* 638 * Close a socket on last file table reference removal. Initiate disconnect 639 * if connected. Free socket when disconnect complete. 640 * 641 * This function will sorele() the socket. Note that soclose() may be called 642 * prior to the ref count reaching zero. The actual socket structure will 643 * not be freed until the ref count reaches zero. 644 */ 645 int 646 soclose(so) 647 struct socket *so; 648 { 649 int error = 0; 650 651 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 652 653 funsetown(&so->so_sigio); 654 if (so->so_state & SS_ISCONNECTED) { 655 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 656 error = sodisconnect(so); 657 if (error) 658 goto drop; 659 } 660 if (so->so_options & SO_LINGER) { 661 if ((so->so_state & SS_ISDISCONNECTING) && 662 (so->so_state & SS_NBIO)) 663 goto drop; 664 while (so->so_state & SS_ISCONNECTED) { 665 error = tsleep(&so->so_timeo, 666 PSOCK | PCATCH, "soclos", so->so_linger * hz); 667 if (error) 668 break; 669 } 670 } 671 } 672 673 drop: 674 if (so->so_proto->pr_usrreqs->pru_close != NULL) 675 (*so->so_proto->pr_usrreqs->pru_close)(so); 676 if (so->so_options & SO_ACCEPTCONN) { 677 struct socket *sp; 678 ACCEPT_LOCK(); 679 while ((sp = TAILQ_FIRST(&so->so_incomp)) != NULL) { 680 TAILQ_REMOVE(&so->so_incomp, sp, so_list); 681 so->so_incqlen--; 682 sp->so_qstate &= ~SQ_INCOMP; 683 sp->so_head = NULL; 684 ACCEPT_UNLOCK(); 685 soabort(sp); 686 ACCEPT_LOCK(); 687 } 688 while ((sp = TAILQ_FIRST(&so->so_comp)) != NULL) { 689 TAILQ_REMOVE(&so->so_comp, sp, so_list); 690 so->so_qlen--; 691 sp->so_qstate &= ~SQ_COMP; 692 sp->so_head = NULL; 693 ACCEPT_UNLOCK(); 694 soabort(sp); 695 ACCEPT_LOCK(); 696 } 697 ACCEPT_UNLOCK(); 698 } 699 ACCEPT_LOCK(); 700 SOCK_LOCK(so); 701 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 702 so->so_state |= SS_NOFDREF; 703 sorele(so); 704 return (error); 705 } 706 707 /* 708 * soabort() is used to abruptly tear down a connection, such as when a 709 * resource limit is reached (listen queue depth exceeded), or if a listen 710 * socket is closed while there are sockets waiting to be accepted. 711 * 712 * This interface is tricky, because it is called on an unreferenced socket, 713 * and must be called only by a thread that has actually removed the socket 714 * from the listen queue it was on, or races with other threads are risked. 715 * 716 * This interface will call into the protocol code, so must not be called 717 * with any socket locks held. Protocols do call it while holding their own 718 * recursible protocol mutexes, but this is something that should be subject 719 * to review in the future. 720 */ 721 void 722 soabort(so) 723 struct socket *so; 724 { 725 726 /* 727 * In as much as is possible, assert that no references to this 728 * socket are held. This is not quite the same as asserting that the 729 * current thread is responsible for arranging for no references, but 730 * is as close as we can get for now. 731 */ 732 KASSERT(so->so_count == 0, ("soabort: so_count")); 733 KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF")); 734 KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF")); 735 KASSERT((so->so_state & SQ_COMP) == 0, ("soabort: SQ_COMP")); 736 KASSERT((so->so_state & SQ_INCOMP) == 0, ("soabort: SQ_INCOMP")); 737 738 if (so->so_proto->pr_usrreqs->pru_abort != NULL) 739 (*so->so_proto->pr_usrreqs->pru_abort)(so); 740 ACCEPT_LOCK(); 741 SOCK_LOCK(so); 742 sofree(so); 743 } 744 745 int 746 soaccept(so, nam) 747 struct socket *so; 748 struct sockaddr **nam; 749 { 750 int error; 751 752 SOCK_LOCK(so); 753 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 754 so->so_state &= ~SS_NOFDREF; 755 SOCK_UNLOCK(so); 756 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 757 return (error); 758 } 759 760 int 761 soconnect(so, nam, td) 762 struct socket *so; 763 struct sockaddr *nam; 764 struct thread *td; 765 { 766 int error; 767 768 if (so->so_options & SO_ACCEPTCONN) 769 return (EOPNOTSUPP); 770 /* 771 * If protocol is connection-based, can only connect once. 772 * Otherwise, if connected, try to disconnect first. This allows 773 * user to disconnect by connecting to, e.g., a null address. 774 */ 775 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 776 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 777 (error = sodisconnect(so)))) { 778 error = EISCONN; 779 } else { 780 /* 781 * Prevent accumulated error from previous connection from 782 * biting us. 783 */ 784 so->so_error = 0; 785 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, nam, td); 786 } 787 788 return (error); 789 } 790 791 int 792 soconnect2(so1, so2) 793 struct socket *so1; 794 struct socket *so2; 795 { 796 797 return ((*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2)); 798 } 799 800 int 801 sodisconnect(so) 802 struct socket *so; 803 { 804 int error; 805 806 if ((so->so_state & SS_ISCONNECTED) == 0) 807 return (ENOTCONN); 808 if (so->so_state & SS_ISDISCONNECTING) 809 return (EALREADY); 810 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 811 return (error); 812 } 813 814 #ifdef ZERO_COPY_SOCKETS 815 struct so_zerocopy_stats{ 816 int size_ok; 817 int align_ok; 818 int found_ifp; 819 }; 820 struct so_zerocopy_stats so_zerocp_stats = {0,0,0}; 821 #include <netinet/in.h> 822 #include <net/route.h> 823 #include <netinet/in_pcb.h> 824 #include <vm/vm.h> 825 #include <vm/vm_page.h> 826 #include <vm/vm_object.h> 827 828 /* 829 * sosend_copyin() is only used if zero copy sockets are enabled. Otherwise 830 * sosend_dgram() and sosend_generic() use m_uiotombuf(). 831 * 832 * sosend_copyin() accepts a uio and prepares an mbuf chain holding part or 833 * all of the data referenced by the uio. If desired, it uses zero-copy. 834 * *space will be updated to reflect data copied in. 835 * 836 * NB: If atomic I/O is requested, the caller must already have checked that 837 * space can hold resid bytes. 838 * 839 * NB: In the event of an error, the caller may need to free the partial 840 * chain pointed to by *mpp. The contents of both *uio and *space may be 841 * modified even in the case of an error. 842 */ 843 static int 844 sosend_copyin(struct uio *uio, struct mbuf **retmp, int atomic, long *space, 845 int flags) 846 { 847 struct mbuf *m, **mp, *top; 848 long len, resid; 849 int error; 850 #ifdef ZERO_COPY_SOCKETS 851 int cow_send; 852 #endif 853 854 *retmp = top = NULL; 855 mp = ⊤ 856 len = 0; 857 resid = uio->uio_resid; 858 error = 0; 859 do { 860 #ifdef ZERO_COPY_SOCKETS 861 cow_send = 0; 862 #endif /* ZERO_COPY_SOCKETS */ 863 if (resid >= MINCLSIZE) { 864 #ifdef ZERO_COPY_SOCKETS 865 if (top == NULL) { 866 MGETHDR(m, M_TRYWAIT, MT_DATA); 867 if (m == NULL) { 868 error = ENOBUFS; 869 goto out; 870 } 871 m->m_pkthdr.len = 0; 872 m->m_pkthdr.rcvif = NULL; 873 } else { 874 MGET(m, M_TRYWAIT, MT_DATA); 875 if (m == NULL) { 876 error = ENOBUFS; 877 goto out; 878 } 879 } 880 if (so_zero_copy_send && 881 resid>=PAGE_SIZE && 882 *space>=PAGE_SIZE && 883 uio->uio_iov->iov_len>=PAGE_SIZE) { 884 so_zerocp_stats.size_ok++; 885 so_zerocp_stats.align_ok++; 886 cow_send = socow_setup(m, uio); 887 len = cow_send; 888 } 889 if (!cow_send) { 890 MCLGET(m, M_TRYWAIT); 891 if ((m->m_flags & M_EXT) == 0) { 892 m_free(m); 893 m = NULL; 894 } else { 895 len = min(min(MCLBYTES, resid), 896 *space); 897 } 898 } 899 #else /* ZERO_COPY_SOCKETS */ 900 if (top == NULL) { 901 m = m_getcl(M_TRYWAIT, MT_DATA, M_PKTHDR); 902 m->m_pkthdr.len = 0; 903 m->m_pkthdr.rcvif = NULL; 904 } else 905 m = m_getcl(M_TRYWAIT, MT_DATA, 0); 906 len = min(min(MCLBYTES, resid), *space); 907 #endif /* ZERO_COPY_SOCKETS */ 908 } else { 909 if (top == NULL) { 910 m = m_gethdr(M_TRYWAIT, MT_DATA); 911 m->m_pkthdr.len = 0; 912 m->m_pkthdr.rcvif = NULL; 913 914 len = min(min(MHLEN, resid), *space); 915 /* 916 * For datagram protocols, leave room 917 * for protocol headers in first mbuf. 918 */ 919 if (atomic && m && len < MHLEN) 920 MH_ALIGN(m, len); 921 } else { 922 m = m_get(M_TRYWAIT, MT_DATA); 923 len = min(min(MLEN, resid), *space); 924 } 925 } 926 if (m == NULL) { 927 error = ENOBUFS; 928 goto out; 929 } 930 931 *space -= len; 932 #ifdef ZERO_COPY_SOCKETS 933 if (cow_send) 934 error = 0; 935 else 936 #endif /* ZERO_COPY_SOCKETS */ 937 error = uiomove(mtod(m, void *), (int)len, uio); 938 resid = uio->uio_resid; 939 m->m_len = len; 940 *mp = m; 941 top->m_pkthdr.len += len; 942 if (error) 943 goto out; 944 mp = &m->m_next; 945 if (resid <= 0) { 946 if (flags & MSG_EOR) 947 top->m_flags |= M_EOR; 948 break; 949 } 950 } while (*space > 0 && atomic); 951 out: 952 *retmp = top; 953 return (error); 954 } 955 #endif /*ZERO_COPY_SOCKETS*/ 956 957 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? M_NOWAIT : M_WAITOK) 958 959 int 960 sosend_dgram(so, addr, uio, top, control, flags, td) 961 struct socket *so; 962 struct sockaddr *addr; 963 struct uio *uio; 964 struct mbuf *top; 965 struct mbuf *control; 966 int flags; 967 struct thread *td; 968 { 969 long space, resid; 970 int clen = 0, error, dontroute; 971 #ifdef ZERO_COPY_SOCKETS 972 int atomic = sosendallatonce(so) || top; 973 #endif 974 975 KASSERT(so->so_type == SOCK_DGRAM, ("sodgram_send: !SOCK_DGRAM")); 976 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 977 ("sodgram_send: !PR_ATOMIC")); 978 979 if (uio != NULL) 980 resid = uio->uio_resid; 981 else 982 resid = top->m_pkthdr.len; 983 /* 984 * In theory resid should be unsigned. However, space must be 985 * signed, as it might be less than 0 if we over-committed, and we 986 * must use a signed comparison of space and resid. On the other 987 * hand, a negative resid causes us to loop sending 0-length 988 * segments to the protocol. 989 * 990 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 991 * type sockets since that's an error. 992 */ 993 if (resid < 0) { 994 error = EINVAL; 995 goto out; 996 } 997 998 dontroute = 999 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 1000 if (td != NULL) 1001 td->td_proc->p_stats->p_ru.ru_msgsnd++; 1002 if (control != NULL) 1003 clen = control->m_len; 1004 1005 SOCKBUF_LOCK(&so->so_snd); 1006 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1007 SOCKBUF_UNLOCK(&so->so_snd); 1008 error = EPIPE; 1009 goto out; 1010 } 1011 if (so->so_error) { 1012 error = so->so_error; 1013 so->so_error = 0; 1014 SOCKBUF_UNLOCK(&so->so_snd); 1015 goto out; 1016 } 1017 if ((so->so_state & SS_ISCONNECTED) == 0) { 1018 /* 1019 * `sendto' and `sendmsg' is allowed on a connection-based 1020 * socket if it supports implied connect. Return ENOTCONN if 1021 * not connected and no address is supplied. 1022 */ 1023 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1024 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1025 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1026 !(resid == 0 && clen != 0)) { 1027 SOCKBUF_UNLOCK(&so->so_snd); 1028 error = ENOTCONN; 1029 goto out; 1030 } 1031 } else if (addr == NULL) { 1032 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1033 error = ENOTCONN; 1034 else 1035 error = EDESTADDRREQ; 1036 SOCKBUF_UNLOCK(&so->so_snd); 1037 goto out; 1038 } 1039 } 1040 1041 /* 1042 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 1043 * problem and need fixing. 1044 */ 1045 space = sbspace(&so->so_snd); 1046 if (flags & MSG_OOB) 1047 space += 1024; 1048 space -= clen; 1049 SOCKBUF_UNLOCK(&so->so_snd); 1050 if (resid > space) { 1051 error = EMSGSIZE; 1052 goto out; 1053 } 1054 if (uio == NULL) { 1055 resid = 0; 1056 if (flags & MSG_EOR) 1057 top->m_flags |= M_EOR; 1058 } else { 1059 #ifdef ZERO_COPY_SOCKETS 1060 error = sosend_copyin(uio, &top, atomic, &space, flags); 1061 if (error) 1062 goto out; 1063 #else 1064 /* 1065 * Copy the data from userland into a mbuf chain. 1066 * If no data is to be copied in, a single empty mbuf 1067 * is returned. 1068 */ 1069 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 1070 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 1071 if (top == NULL) { 1072 error = EFAULT; /* only possible error */ 1073 goto out; 1074 } 1075 space -= resid - uio->uio_resid; 1076 #endif 1077 resid = uio->uio_resid; 1078 } 1079 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 1080 /* 1081 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 1082 * than with. 1083 */ 1084 if (dontroute) { 1085 SOCK_LOCK(so); 1086 so->so_options |= SO_DONTROUTE; 1087 SOCK_UNLOCK(so); 1088 } 1089 /* 1090 * XXX all the SBS_CANTSENDMORE checks previously done could be out 1091 * of date. We could have recieved a reset packet in an interrupt or 1092 * maybe we slept while doing page faults in uiomove() etc. We could 1093 * probably recheck again inside the locking protection here, but 1094 * there are probably other places that this also happens. We must 1095 * rethink this. 1096 */ 1097 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1098 (flags & MSG_OOB) ? PRUS_OOB : 1099 /* 1100 * If the user set MSG_EOF, the protocol understands this flag and 1101 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 1102 */ 1103 ((flags & MSG_EOF) && 1104 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1105 (resid <= 0)) ? 1106 PRUS_EOF : 1107 /* If there is more to send set PRUS_MORETOCOME */ 1108 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1109 top, addr, control, td); 1110 if (dontroute) { 1111 SOCK_LOCK(so); 1112 so->so_options &= ~SO_DONTROUTE; 1113 SOCK_UNLOCK(so); 1114 } 1115 clen = 0; 1116 control = NULL; 1117 top = NULL; 1118 out: 1119 if (top != NULL) 1120 m_freem(top); 1121 if (control != NULL) 1122 m_freem(control); 1123 return (error); 1124 } 1125 1126 /* 1127 * Send on a socket. If send must go all at once and message is larger than 1128 * send buffering, then hard error. Lock against other senders. If must go 1129 * all at once and not enough room now, then inform user that this would 1130 * block and do nothing. Otherwise, if nonblocking, send as much as 1131 * possible. The data to be sent is described by "uio" if nonzero, otherwise 1132 * by the mbuf chain "top" (which must be null if uio is not). Data provided 1133 * in mbuf chain must be small enough to send all at once. 1134 * 1135 * Returns nonzero on error, timeout or signal; callers must check for short 1136 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 1137 * on return. 1138 */ 1139 #define snderr(errno) { error = (errno); goto release; } 1140 int 1141 sosend_generic(so, addr, uio, top, control, flags, td) 1142 struct socket *so; 1143 struct sockaddr *addr; 1144 struct uio *uio; 1145 struct mbuf *top; 1146 struct mbuf *control; 1147 int flags; 1148 struct thread *td; 1149 { 1150 long space, resid; 1151 int clen = 0, error, dontroute; 1152 int atomic = sosendallatonce(so) || top; 1153 1154 if (uio != NULL) 1155 resid = uio->uio_resid; 1156 else 1157 resid = top->m_pkthdr.len; 1158 /* 1159 * In theory resid should be unsigned. However, space must be 1160 * signed, as it might be less than 0 if we over-committed, and we 1161 * must use a signed comparison of space and resid. On the other 1162 * hand, a negative resid causes us to loop sending 0-length 1163 * segments to the protocol. 1164 * 1165 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 1166 * type sockets since that's an error. 1167 */ 1168 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 1169 error = EINVAL; 1170 goto out; 1171 } 1172 1173 dontroute = 1174 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1175 (so->so_proto->pr_flags & PR_ATOMIC); 1176 if (td != NULL) 1177 td->td_proc->p_stats->p_ru.ru_msgsnd++; 1178 if (control != NULL) 1179 clen = control->m_len; 1180 1181 SOCKBUF_LOCK(&so->so_snd); 1182 restart: 1183 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1184 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1185 if (error) 1186 goto out_locked; 1187 do { 1188 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1189 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 1190 snderr(EPIPE); 1191 if (so->so_error) { 1192 error = so->so_error; 1193 so->so_error = 0; 1194 goto release; 1195 } 1196 if ((so->so_state & SS_ISCONNECTED) == 0) { 1197 /* 1198 * `sendto' and `sendmsg' is allowed on a connection- 1199 * based socket if it supports implied connect. 1200 * Return ENOTCONN if not connected and no address is 1201 * supplied. 1202 */ 1203 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1204 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1205 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1206 !(resid == 0 && clen != 0)) 1207 snderr(ENOTCONN); 1208 } else if (addr == NULL) 1209 snderr(so->so_proto->pr_flags & PR_CONNREQUIRED ? 1210 ENOTCONN : EDESTADDRREQ); 1211 } 1212 space = sbspace(&so->so_snd); 1213 if (flags & MSG_OOB) 1214 space += 1024; 1215 if ((atomic && resid > so->so_snd.sb_hiwat) || 1216 clen > so->so_snd.sb_hiwat) 1217 snderr(EMSGSIZE); 1218 if (space < resid + clen && 1219 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1220 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) 1221 snderr(EWOULDBLOCK); 1222 sbunlock(&so->so_snd); 1223 error = sbwait(&so->so_snd); 1224 if (error) 1225 goto out_locked; 1226 goto restart; 1227 } 1228 SOCKBUF_UNLOCK(&so->so_snd); 1229 space -= clen; 1230 do { 1231 if (uio == NULL) { 1232 resid = 0; 1233 if (flags & MSG_EOR) 1234 top->m_flags |= M_EOR; 1235 } else { 1236 #ifdef ZERO_COPY_SOCKETS 1237 error = sosend_copyin(uio, &top, atomic, 1238 &space, flags); 1239 if (error != 0) { 1240 SOCKBUF_LOCK(&so->so_snd); 1241 goto release; 1242 } 1243 #else 1244 /* 1245 * Copy the data from userland into a mbuf 1246 * chain. If no data is to be copied in, 1247 * a single empty mbuf is returned. 1248 */ 1249 top = m_uiotombuf(uio, M_WAITOK, space, 1250 (atomic ? max_hdr : 0), 1251 (atomic ? M_PKTHDR : 0) | 1252 ((flags & MSG_EOR) ? M_EOR : 0)); 1253 if (top == NULL) { 1254 SOCKBUF_LOCK(&so->so_snd); 1255 error = EFAULT; /* only possible error */ 1256 goto release; 1257 } 1258 space -= resid - uio->uio_resid; 1259 #endif 1260 resid = uio->uio_resid; 1261 } 1262 if (dontroute) { 1263 SOCK_LOCK(so); 1264 so->so_options |= SO_DONTROUTE; 1265 SOCK_UNLOCK(so); 1266 } 1267 /* 1268 * XXX all the SBS_CANTSENDMORE checks previously 1269 * done could be out of date. We could have recieved 1270 * a reset packet in an interrupt or maybe we slept 1271 * while doing page faults in uiomove() etc. We 1272 * could probably recheck again inside the locking 1273 * protection here, but there are probably other 1274 * places that this also happens. We must rethink 1275 * this. 1276 */ 1277 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1278 (flags & MSG_OOB) ? PRUS_OOB : 1279 /* 1280 * If the user set MSG_EOF, the protocol understands 1281 * this flag and nothing left to send then use 1282 * PRU_SEND_EOF instead of PRU_SEND. 1283 */ 1284 ((flags & MSG_EOF) && 1285 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1286 (resid <= 0)) ? 1287 PRUS_EOF : 1288 /* If there is more to send set PRUS_MORETOCOME. */ 1289 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1290 top, addr, control, td); 1291 if (dontroute) { 1292 SOCK_LOCK(so); 1293 so->so_options &= ~SO_DONTROUTE; 1294 SOCK_UNLOCK(so); 1295 } 1296 clen = 0; 1297 control = NULL; 1298 top = NULL; 1299 if (error) { 1300 SOCKBUF_LOCK(&so->so_snd); 1301 goto release; 1302 } 1303 } while (resid && space > 0); 1304 SOCKBUF_LOCK(&so->so_snd); 1305 } while (resid); 1306 1307 release: 1308 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1309 sbunlock(&so->so_snd); 1310 out_locked: 1311 SOCKBUF_LOCK_ASSERT(&so->so_snd); 1312 SOCKBUF_UNLOCK(&so->so_snd); 1313 out: 1314 if (top != NULL) 1315 m_freem(top); 1316 if (control != NULL) 1317 m_freem(control); 1318 return (error); 1319 } 1320 #undef snderr 1321 1322 int 1323 sosend(so, addr, uio, top, control, flags, td) 1324 struct socket *so; 1325 struct sockaddr *addr; 1326 struct uio *uio; 1327 struct mbuf *top; 1328 struct mbuf *control; 1329 int flags; 1330 struct thread *td; 1331 { 1332 1333 /* XXXRW: Temporary debugging. */ 1334 KASSERT(so->so_proto->pr_usrreqs->pru_sosend != sosend, 1335 ("sosend: protocol calls sosend")); 1336 1337 return (so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, top, 1338 control, flags, td)); 1339 } 1340 1341 /* 1342 * The part of soreceive() that implements reading non-inline out-of-band 1343 * data from a socket. For more complete comments, see soreceive(), from 1344 * which this code originated. 1345 * 1346 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1347 * unable to return an mbuf chain to the caller. 1348 */ 1349 static int 1350 soreceive_rcvoob(so, uio, flags) 1351 struct socket *so; 1352 struct uio *uio; 1353 int flags; 1354 { 1355 struct protosw *pr = so->so_proto; 1356 struct mbuf *m; 1357 int error; 1358 1359 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1360 1361 m = m_get(M_TRYWAIT, MT_DATA); 1362 if (m == NULL) 1363 return (ENOBUFS); 1364 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1365 if (error) 1366 goto bad; 1367 do { 1368 #ifdef ZERO_COPY_SOCKETS 1369 if (so_zero_copy_receive) { 1370 int disposable; 1371 1372 if ((m->m_flags & M_EXT) 1373 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1374 disposable = 1; 1375 else 1376 disposable = 0; 1377 1378 error = uiomoveco(mtod(m, void *), 1379 min(uio->uio_resid, m->m_len), 1380 uio, disposable); 1381 } else 1382 #endif /* ZERO_COPY_SOCKETS */ 1383 error = uiomove(mtod(m, void *), 1384 (int) min(uio->uio_resid, m->m_len), uio); 1385 m = m_free(m); 1386 } while (uio->uio_resid && error == 0 && m); 1387 bad: 1388 if (m != NULL) 1389 m_freem(m); 1390 return (error); 1391 } 1392 1393 /* 1394 * Following replacement or removal of the first mbuf on the first mbuf chain 1395 * of a socket buffer, push necessary state changes back into the socket 1396 * buffer so that other consumers see the values consistently. 'nextrecord' 1397 * is the callers locally stored value of the original value of 1398 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1399 * NOTE: 'nextrecord' may be NULL. 1400 */ 1401 static __inline void 1402 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1403 { 1404 1405 SOCKBUF_LOCK_ASSERT(sb); 1406 /* 1407 * First, update for the new value of nextrecord. If necessary, make 1408 * it the first record. 1409 */ 1410 if (sb->sb_mb != NULL) 1411 sb->sb_mb->m_nextpkt = nextrecord; 1412 else 1413 sb->sb_mb = nextrecord; 1414 1415 /* 1416 * Now update any dependent socket buffer fields to reflect the new 1417 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1418 * addition of a second clause that takes care of the case where 1419 * sb_mb has been updated, but remains the last record. 1420 */ 1421 if (sb->sb_mb == NULL) { 1422 sb->sb_mbtail = NULL; 1423 sb->sb_lastrecord = NULL; 1424 } else if (sb->sb_mb->m_nextpkt == NULL) 1425 sb->sb_lastrecord = sb->sb_mb; 1426 } 1427 1428 1429 /* 1430 * Implement receive operations on a socket. We depend on the way that 1431 * records are added to the sockbuf by sbappend. In particular, each record 1432 * (mbufs linked through m_next) must begin with an address if the protocol 1433 * so specifies, followed by an optional mbuf or mbufs containing ancillary 1434 * data, and then zero or more mbufs of data. In order to allow parallelism 1435 * between network receive and copying to user space, as well as avoid 1436 * sleeping with a mutex held, we release the socket buffer mutex during the 1437 * user space copy. Although the sockbuf is locked, new data may still be 1438 * appended, and thus we must maintain consistency of the sockbuf during that 1439 * time. 1440 * 1441 * The caller may receive the data as a single mbuf chain by supplying an 1442 * mbuf **mp0 for use in returning the chain. The uio is then used only for 1443 * the count in uio_resid. 1444 */ 1445 int 1446 soreceive_generic(so, psa, uio, mp0, controlp, flagsp) 1447 struct socket *so; 1448 struct sockaddr **psa; 1449 struct uio *uio; 1450 struct mbuf **mp0; 1451 struct mbuf **controlp; 1452 int *flagsp; 1453 { 1454 struct mbuf *m, **mp; 1455 int flags, len, error, offset; 1456 struct protosw *pr = so->so_proto; 1457 struct mbuf *nextrecord; 1458 int moff, type = 0; 1459 int orig_resid = uio->uio_resid; 1460 1461 mp = mp0; 1462 if (psa != NULL) 1463 *psa = NULL; 1464 if (controlp != NULL) 1465 *controlp = NULL; 1466 if (flagsp != NULL) 1467 flags = *flagsp &~ MSG_EOR; 1468 else 1469 flags = 0; 1470 if (flags & MSG_OOB) 1471 return (soreceive_rcvoob(so, uio, flags)); 1472 if (mp != NULL) 1473 *mp = NULL; 1474 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1475 && uio->uio_resid) 1476 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1477 1478 SOCKBUF_LOCK(&so->so_rcv); 1479 restart: 1480 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1481 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1482 if (error) 1483 goto out; 1484 1485 m = so->so_rcv.sb_mb; 1486 /* 1487 * If we have less data than requested, block awaiting more (subject 1488 * to any timeout) if: 1489 * 1. the current count is less than the low water mark, or 1490 * 2. MSG_WAITALL is set, and it is possible to do the entire 1491 * receive operation at once if we block (resid <= hiwat). 1492 * 3. MSG_DONTWAIT is not set 1493 * If MSG_WAITALL is set but resid is larger than the receive buffer, 1494 * we have to do the receive in sections, and thus risk returning a 1495 * short count if a timeout or signal occurs after we start. 1496 */ 1497 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1498 so->so_rcv.sb_cc < uio->uio_resid) && 1499 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat || 1500 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) && 1501 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1502 KASSERT(m != NULL || !so->so_rcv.sb_cc, 1503 ("receive: m == %p so->so_rcv.sb_cc == %u", 1504 m, so->so_rcv.sb_cc)); 1505 if (so->so_error) { 1506 if (m != NULL) 1507 goto dontblock; 1508 error = so->so_error; 1509 if ((flags & MSG_PEEK) == 0) 1510 so->so_error = 0; 1511 goto release; 1512 } 1513 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1514 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1515 if (m) 1516 goto dontblock; 1517 else 1518 goto release; 1519 } 1520 for (; m != NULL; m = m->m_next) 1521 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1522 m = so->so_rcv.sb_mb; 1523 goto dontblock; 1524 } 1525 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1526 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1527 error = ENOTCONN; 1528 goto release; 1529 } 1530 if (uio->uio_resid == 0) 1531 goto release; 1532 if ((so->so_state & SS_NBIO) || 1533 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1534 error = EWOULDBLOCK; 1535 goto release; 1536 } 1537 SBLASTRECORDCHK(&so->so_rcv); 1538 SBLASTMBUFCHK(&so->so_rcv); 1539 sbunlock(&so->so_rcv); 1540 error = sbwait(&so->so_rcv); 1541 if (error) 1542 goto out; 1543 goto restart; 1544 } 1545 dontblock: 1546 /* 1547 * From this point onward, we maintain 'nextrecord' as a cache of the 1548 * pointer to the next record in the socket buffer. We must keep the 1549 * various socket buffer pointers and local stack versions of the 1550 * pointers in sync, pushing out modifications before dropping the 1551 * socket buffer mutex, and re-reading them when picking it up. 1552 * 1553 * Otherwise, we will race with the network stack appending new data 1554 * or records onto the socket buffer by using inconsistent/stale 1555 * versions of the field, possibly resulting in socket buffer 1556 * corruption. 1557 * 1558 * By holding the high-level sblock(), we prevent simultaneous 1559 * readers from pulling off the front of the socket buffer. 1560 */ 1561 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1562 if (uio->uio_td) 1563 uio->uio_td->td_proc->p_stats->p_ru.ru_msgrcv++; 1564 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1565 SBLASTRECORDCHK(&so->so_rcv); 1566 SBLASTMBUFCHK(&so->so_rcv); 1567 nextrecord = m->m_nextpkt; 1568 if (pr->pr_flags & PR_ADDR) { 1569 KASSERT(m->m_type == MT_SONAME, 1570 ("m->m_type == %d", m->m_type)); 1571 orig_resid = 0; 1572 if (psa != NULL) 1573 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1574 M_NOWAIT); 1575 if (flags & MSG_PEEK) { 1576 m = m->m_next; 1577 } else { 1578 sbfree(&so->so_rcv, m); 1579 so->so_rcv.sb_mb = m_free(m); 1580 m = so->so_rcv.sb_mb; 1581 sockbuf_pushsync(&so->so_rcv, nextrecord); 1582 } 1583 } 1584 1585 /* 1586 * Process one or more MT_CONTROL mbufs present before any data mbufs 1587 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1588 * just copy the data; if !MSG_PEEK, we call into the protocol to 1589 * perform externalization (or freeing if controlp == NULL). 1590 */ 1591 if (m != NULL && m->m_type == MT_CONTROL) { 1592 struct mbuf *cm = NULL, *cmn; 1593 struct mbuf **cme = &cm; 1594 1595 do { 1596 if (flags & MSG_PEEK) { 1597 if (controlp != NULL) { 1598 *controlp = m_copy(m, 0, m->m_len); 1599 controlp = &(*controlp)->m_next; 1600 } 1601 m = m->m_next; 1602 } else { 1603 sbfree(&so->so_rcv, m); 1604 so->so_rcv.sb_mb = m->m_next; 1605 m->m_next = NULL; 1606 *cme = m; 1607 cme = &(*cme)->m_next; 1608 m = so->so_rcv.sb_mb; 1609 } 1610 } while (m != NULL && m->m_type == MT_CONTROL); 1611 if ((flags & MSG_PEEK) == 0) 1612 sockbuf_pushsync(&so->so_rcv, nextrecord); 1613 while (cm != NULL) { 1614 cmn = cm->m_next; 1615 cm->m_next = NULL; 1616 if (pr->pr_domain->dom_externalize != NULL) { 1617 SOCKBUF_UNLOCK(&so->so_rcv); 1618 error = (*pr->pr_domain->dom_externalize) 1619 (cm, controlp); 1620 SOCKBUF_LOCK(&so->so_rcv); 1621 } else if (controlp != NULL) 1622 *controlp = cm; 1623 else 1624 m_freem(cm); 1625 if (controlp != NULL) { 1626 orig_resid = 0; 1627 while (*controlp != NULL) 1628 controlp = &(*controlp)->m_next; 1629 } 1630 cm = cmn; 1631 } 1632 if (m != NULL) 1633 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1634 else 1635 nextrecord = so->so_rcv.sb_mb; 1636 orig_resid = 0; 1637 } 1638 if (m != NULL) { 1639 if ((flags & MSG_PEEK) == 0) { 1640 KASSERT(m->m_nextpkt == nextrecord, 1641 ("soreceive: post-control, nextrecord !sync")); 1642 if (nextrecord == NULL) { 1643 KASSERT(so->so_rcv.sb_mb == m, 1644 ("soreceive: post-control, sb_mb!=m")); 1645 KASSERT(so->so_rcv.sb_lastrecord == m, 1646 ("soreceive: post-control, lastrecord!=m")); 1647 } 1648 } 1649 type = m->m_type; 1650 if (type == MT_OOBDATA) 1651 flags |= MSG_OOB; 1652 } else { 1653 if ((flags & MSG_PEEK) == 0) { 1654 KASSERT(so->so_rcv.sb_mb == nextrecord, 1655 ("soreceive: sb_mb != nextrecord")); 1656 if (so->so_rcv.sb_mb == NULL) { 1657 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1658 ("soreceive: sb_lastercord != NULL")); 1659 } 1660 } 1661 } 1662 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1663 SBLASTRECORDCHK(&so->so_rcv); 1664 SBLASTMBUFCHK(&so->so_rcv); 1665 1666 /* 1667 * Now continue to read any data mbufs off of the head of the socket 1668 * buffer until the read request is satisfied. Note that 'type' is 1669 * used to store the type of any mbuf reads that have happened so far 1670 * such that soreceive() can stop reading if the type changes, which 1671 * causes soreceive() to return only one of regular data and inline 1672 * out-of-band data in a single socket receive operation. 1673 */ 1674 moff = 0; 1675 offset = 0; 1676 while (m != NULL && uio->uio_resid > 0 && error == 0) { 1677 /* 1678 * If the type of mbuf has changed since the last mbuf 1679 * examined ('type'), end the receive operation. 1680 */ 1681 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1682 if (m->m_type == MT_OOBDATA) { 1683 if (type != MT_OOBDATA) 1684 break; 1685 } else if (type == MT_OOBDATA) 1686 break; 1687 else 1688 KASSERT(m->m_type == MT_DATA, 1689 ("m->m_type == %d", m->m_type)); 1690 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1691 len = uio->uio_resid; 1692 if (so->so_oobmark && len > so->so_oobmark - offset) 1693 len = so->so_oobmark - offset; 1694 if (len > m->m_len - moff) 1695 len = m->m_len - moff; 1696 /* 1697 * If mp is set, just pass back the mbufs. Otherwise copy 1698 * them out via the uio, then free. Sockbuf must be 1699 * consistent here (points to current mbuf, it points to next 1700 * record) when we drop priority; we must note any additions 1701 * to the sockbuf when we block interrupts again. 1702 */ 1703 if (mp == NULL) { 1704 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1705 SBLASTRECORDCHK(&so->so_rcv); 1706 SBLASTMBUFCHK(&so->so_rcv); 1707 SOCKBUF_UNLOCK(&so->so_rcv); 1708 #ifdef ZERO_COPY_SOCKETS 1709 if (so_zero_copy_receive) { 1710 int disposable; 1711 1712 if ((m->m_flags & M_EXT) 1713 && (m->m_ext.ext_type == EXT_DISPOSABLE)) 1714 disposable = 1; 1715 else 1716 disposable = 0; 1717 1718 error = uiomoveco(mtod(m, char *) + moff, 1719 (int)len, uio, 1720 disposable); 1721 } else 1722 #endif /* ZERO_COPY_SOCKETS */ 1723 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1724 SOCKBUF_LOCK(&so->so_rcv); 1725 if (error) { 1726 /* 1727 * The MT_SONAME mbuf has already been removed 1728 * from the record, so it is necessary to 1729 * remove the data mbufs, if any, to preserve 1730 * the invariant in the case of PR_ADDR that 1731 * requires MT_SONAME mbufs at the head of 1732 * each record. 1733 */ 1734 if (m && pr->pr_flags & PR_ATOMIC && 1735 ((flags & MSG_PEEK) == 0)) 1736 (void)sbdroprecord_locked(&so->so_rcv); 1737 goto release; 1738 } 1739 } else 1740 uio->uio_resid -= len; 1741 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1742 if (len == m->m_len - moff) { 1743 if (m->m_flags & M_EOR) 1744 flags |= MSG_EOR; 1745 if (flags & MSG_PEEK) { 1746 m = m->m_next; 1747 moff = 0; 1748 } else { 1749 nextrecord = m->m_nextpkt; 1750 sbfree(&so->so_rcv, m); 1751 if (mp != NULL) { 1752 *mp = m; 1753 mp = &m->m_next; 1754 so->so_rcv.sb_mb = m = m->m_next; 1755 *mp = NULL; 1756 } else { 1757 so->so_rcv.sb_mb = m_free(m); 1758 m = so->so_rcv.sb_mb; 1759 } 1760 sockbuf_pushsync(&so->so_rcv, nextrecord); 1761 SBLASTRECORDCHK(&so->so_rcv); 1762 SBLASTMBUFCHK(&so->so_rcv); 1763 } 1764 } else { 1765 if (flags & MSG_PEEK) 1766 moff += len; 1767 else { 1768 if (mp != NULL) { 1769 int copy_flag; 1770 1771 if (flags & MSG_DONTWAIT) 1772 copy_flag = M_DONTWAIT; 1773 else 1774 copy_flag = M_TRYWAIT; 1775 if (copy_flag == M_TRYWAIT) 1776 SOCKBUF_UNLOCK(&so->so_rcv); 1777 *mp = m_copym(m, 0, len, copy_flag); 1778 if (copy_flag == M_TRYWAIT) 1779 SOCKBUF_LOCK(&so->so_rcv); 1780 if (*mp == NULL) { 1781 /* 1782 * m_copym() couldn't 1783 * allocate an mbuf. Adjust 1784 * uio_resid back (it was 1785 * adjusted down by len 1786 * bytes, which we didn't end 1787 * up "copying" over). 1788 */ 1789 uio->uio_resid += len; 1790 break; 1791 } 1792 } 1793 m->m_data += len; 1794 m->m_len -= len; 1795 so->so_rcv.sb_cc -= len; 1796 } 1797 } 1798 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1799 if (so->so_oobmark) { 1800 if ((flags & MSG_PEEK) == 0) { 1801 so->so_oobmark -= len; 1802 if (so->so_oobmark == 0) { 1803 so->so_rcv.sb_state |= SBS_RCVATMARK; 1804 break; 1805 } 1806 } else { 1807 offset += len; 1808 if (offset == so->so_oobmark) 1809 break; 1810 } 1811 } 1812 if (flags & MSG_EOR) 1813 break; 1814 /* 1815 * If the MSG_WAITALL flag is set (for non-atomic socket), we 1816 * must not quit until "uio->uio_resid == 0" or an error 1817 * termination. If a signal/timeout occurs, return with a 1818 * short count but without error. Keep sockbuf locked 1819 * against other readers. 1820 */ 1821 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 1822 !sosendallatonce(so) && nextrecord == NULL) { 1823 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1824 if (so->so_error || so->so_rcv.sb_state & SBS_CANTRCVMORE) 1825 break; 1826 /* 1827 * Notify the protocol that some data has been 1828 * drained before blocking. 1829 */ 1830 if (pr->pr_flags & PR_WANTRCVD) { 1831 SOCKBUF_UNLOCK(&so->so_rcv); 1832 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1833 SOCKBUF_LOCK(&so->so_rcv); 1834 } 1835 SBLASTRECORDCHK(&so->so_rcv); 1836 SBLASTMBUFCHK(&so->so_rcv); 1837 error = sbwait(&so->so_rcv); 1838 if (error) 1839 goto release; 1840 m = so->so_rcv.sb_mb; 1841 if (m != NULL) 1842 nextrecord = m->m_nextpkt; 1843 } 1844 } 1845 1846 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1847 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 1848 flags |= MSG_TRUNC; 1849 if ((flags & MSG_PEEK) == 0) 1850 (void) sbdroprecord_locked(&so->so_rcv); 1851 } 1852 if ((flags & MSG_PEEK) == 0) { 1853 if (m == NULL) { 1854 /* 1855 * First part is an inline SB_EMPTY_FIXUP(). Second 1856 * part makes sure sb_lastrecord is up-to-date if 1857 * there is still data in the socket buffer. 1858 */ 1859 so->so_rcv.sb_mb = nextrecord; 1860 if (so->so_rcv.sb_mb == NULL) { 1861 so->so_rcv.sb_mbtail = NULL; 1862 so->so_rcv.sb_lastrecord = NULL; 1863 } else if (nextrecord->m_nextpkt == NULL) 1864 so->so_rcv.sb_lastrecord = nextrecord; 1865 } 1866 SBLASTRECORDCHK(&so->so_rcv); 1867 SBLASTMBUFCHK(&so->so_rcv); 1868 /* 1869 * If soreceive() is being done from the socket callback, 1870 * then don't need to generate ACK to peer to update window, 1871 * since ACK will be generated on return to TCP. 1872 */ 1873 if (!(flags & MSG_SOCALLBCK) && 1874 (pr->pr_flags & PR_WANTRCVD)) { 1875 SOCKBUF_UNLOCK(&so->so_rcv); 1876 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 1877 SOCKBUF_LOCK(&so->so_rcv); 1878 } 1879 } 1880 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1881 if (orig_resid == uio->uio_resid && orig_resid && 1882 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 1883 sbunlock(&so->so_rcv); 1884 goto restart; 1885 } 1886 1887 if (flagsp != NULL) 1888 *flagsp |= flags; 1889 release: 1890 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1891 sbunlock(&so->so_rcv); 1892 out: 1893 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1894 SOCKBUF_UNLOCK(&so->so_rcv); 1895 return (error); 1896 } 1897 1898 int 1899 soreceive(so, psa, uio, mp0, controlp, flagsp) 1900 struct socket *so; 1901 struct sockaddr **psa; 1902 struct uio *uio; 1903 struct mbuf **mp0; 1904 struct mbuf **controlp; 1905 int *flagsp; 1906 { 1907 1908 /* XXXRW: Temporary debugging. */ 1909 KASSERT(so->so_proto->pr_usrreqs->pru_soreceive != soreceive, 1910 ("soreceive: protocol calls soreceive")); 1911 1912 return (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0, 1913 controlp, flagsp)); 1914 } 1915 1916 int 1917 soshutdown(so, how) 1918 struct socket *so; 1919 int how; 1920 { 1921 struct protosw *pr = so->so_proto; 1922 1923 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 1924 return (EINVAL); 1925 1926 if (how != SHUT_WR) 1927 sorflush(so); 1928 if (how != SHUT_RD) 1929 return ((*pr->pr_usrreqs->pru_shutdown)(so)); 1930 return (0); 1931 } 1932 1933 void 1934 sorflush(so) 1935 struct socket *so; 1936 { 1937 struct sockbuf *sb = &so->so_rcv; 1938 struct protosw *pr = so->so_proto; 1939 struct sockbuf asb; 1940 1941 /* 1942 * XXXRW: This is quite ugly. Previously, this code made a copy of 1943 * the socket buffer, then zero'd the original to clear the buffer 1944 * fields. However, with mutexes in the socket buffer, this causes 1945 * problems. We only clear the zeroable bits of the original; 1946 * however, we have to initialize and destroy the mutex in the copy 1947 * so that dom_dispose() and sbrelease() can lock t as needed. 1948 */ 1949 SOCKBUF_LOCK(sb); 1950 sb->sb_flags |= SB_NOINTR; 1951 (void) sblock(sb, M_WAITOK); 1952 /* 1953 * socantrcvmore_locked() drops the socket buffer mutex so that it 1954 * can safely perform wakeups. Re-acquire the mutex before 1955 * continuing. 1956 */ 1957 socantrcvmore_locked(so); 1958 SOCKBUF_LOCK(sb); 1959 sbunlock(sb); 1960 /* 1961 * Invalidate/clear most of the sockbuf structure, but leave selinfo 1962 * and mutex data unchanged. 1963 */ 1964 bzero(&asb, offsetof(struct sockbuf, sb_startzero)); 1965 bcopy(&sb->sb_startzero, &asb.sb_startzero, 1966 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1967 bzero(&sb->sb_startzero, 1968 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 1969 SOCKBUF_UNLOCK(sb); 1970 1971 SOCKBUF_LOCK_INIT(&asb, "so_rcv"); 1972 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1973 (*pr->pr_domain->dom_dispose)(asb.sb_mb); 1974 sbrelease(&asb, so); 1975 SOCKBUF_LOCK_DESTROY(&asb); 1976 } 1977 1978 /* 1979 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 1980 * additional variant to handle the case where the option value needs to be 1981 * some kind of integer, but not a specific size. In addition to their use 1982 * here, these functions are also called by the protocol-level pr_ctloutput() 1983 * routines. 1984 */ 1985 int 1986 sooptcopyin(sopt, buf, len, minlen) 1987 struct sockopt *sopt; 1988 void *buf; 1989 size_t len; 1990 size_t minlen; 1991 { 1992 size_t valsize; 1993 1994 /* 1995 * If the user gives us more than we wanted, we ignore it, but if we 1996 * don't get the minimum length the caller wants, we return EINVAL. 1997 * On success, sopt->sopt_valsize is set to however much we actually 1998 * retrieved. 1999 */ 2000 if ((valsize = sopt->sopt_valsize) < minlen) 2001 return EINVAL; 2002 if (valsize > len) 2003 sopt->sopt_valsize = valsize = len; 2004 2005 if (sopt->sopt_td != NULL) 2006 return (copyin(sopt->sopt_val, buf, valsize)); 2007 2008 bcopy(sopt->sopt_val, buf, valsize); 2009 return (0); 2010 } 2011 2012 /* 2013 * Kernel version of setsockopt(2). 2014 * 2015 * XXX: optlen is size_t, not socklen_t 2016 */ 2017 int 2018 so_setsockopt(struct socket *so, int level, int optname, void *optval, 2019 size_t optlen) 2020 { 2021 struct sockopt sopt; 2022 2023 sopt.sopt_level = level; 2024 sopt.sopt_name = optname; 2025 sopt.sopt_dir = SOPT_SET; 2026 sopt.sopt_val = optval; 2027 sopt.sopt_valsize = optlen; 2028 sopt.sopt_td = NULL; 2029 return (sosetopt(so, &sopt)); 2030 } 2031 2032 int 2033 sosetopt(so, sopt) 2034 struct socket *so; 2035 struct sockopt *sopt; 2036 { 2037 int error, optval; 2038 struct linger l; 2039 struct timeval tv; 2040 u_long val; 2041 #ifdef MAC 2042 struct mac extmac; 2043 #endif 2044 2045 error = 0; 2046 if (sopt->sopt_level != SOL_SOCKET) { 2047 if (so->so_proto && so->so_proto->pr_ctloutput) 2048 return ((*so->so_proto->pr_ctloutput) 2049 (so, sopt)); 2050 error = ENOPROTOOPT; 2051 } else { 2052 switch (sopt->sopt_name) { 2053 #ifdef INET 2054 case SO_ACCEPTFILTER: 2055 error = do_setopt_accept_filter(so, sopt); 2056 if (error) 2057 goto bad; 2058 break; 2059 #endif 2060 case SO_LINGER: 2061 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 2062 if (error) 2063 goto bad; 2064 2065 SOCK_LOCK(so); 2066 so->so_linger = l.l_linger; 2067 if (l.l_onoff) 2068 so->so_options |= SO_LINGER; 2069 else 2070 so->so_options &= ~SO_LINGER; 2071 SOCK_UNLOCK(so); 2072 break; 2073 2074 case SO_DEBUG: 2075 case SO_KEEPALIVE: 2076 case SO_DONTROUTE: 2077 case SO_USELOOPBACK: 2078 case SO_BROADCAST: 2079 case SO_REUSEADDR: 2080 case SO_REUSEPORT: 2081 case SO_OOBINLINE: 2082 case SO_TIMESTAMP: 2083 case SO_BINTIME: 2084 case SO_NOSIGPIPE: 2085 error = sooptcopyin(sopt, &optval, sizeof optval, 2086 sizeof optval); 2087 if (error) 2088 goto bad; 2089 SOCK_LOCK(so); 2090 if (optval) 2091 so->so_options |= sopt->sopt_name; 2092 else 2093 so->so_options &= ~sopt->sopt_name; 2094 SOCK_UNLOCK(so); 2095 break; 2096 2097 case SO_SNDBUF: 2098 case SO_RCVBUF: 2099 case SO_SNDLOWAT: 2100 case SO_RCVLOWAT: 2101 error = sooptcopyin(sopt, &optval, sizeof optval, 2102 sizeof optval); 2103 if (error) 2104 goto bad; 2105 2106 /* 2107 * Values < 1 make no sense for any of these options, 2108 * so disallow them. 2109 */ 2110 if (optval < 1) { 2111 error = EINVAL; 2112 goto bad; 2113 } 2114 2115 switch (sopt->sopt_name) { 2116 case SO_SNDBUF: 2117 case SO_RCVBUF: 2118 if (sbreserve(sopt->sopt_name == SO_SNDBUF ? 2119 &so->so_snd : &so->so_rcv, (u_long)optval, 2120 so, curthread) == 0) { 2121 error = ENOBUFS; 2122 goto bad; 2123 } 2124 (sopt->sopt_name == SO_SNDBUF ? &so->so_snd : 2125 &so->so_rcv)->sb_flags &= ~SB_AUTOSIZE; 2126 break; 2127 2128 /* 2129 * Make sure the low-water is never greater than the 2130 * high-water. 2131 */ 2132 case SO_SNDLOWAT: 2133 SOCKBUF_LOCK(&so->so_snd); 2134 so->so_snd.sb_lowat = 2135 (optval > so->so_snd.sb_hiwat) ? 2136 so->so_snd.sb_hiwat : optval; 2137 SOCKBUF_UNLOCK(&so->so_snd); 2138 break; 2139 case SO_RCVLOWAT: 2140 SOCKBUF_LOCK(&so->so_rcv); 2141 so->so_rcv.sb_lowat = 2142 (optval > so->so_rcv.sb_hiwat) ? 2143 so->so_rcv.sb_hiwat : optval; 2144 SOCKBUF_UNLOCK(&so->so_rcv); 2145 break; 2146 } 2147 break; 2148 2149 case SO_SNDTIMEO: 2150 case SO_RCVTIMEO: 2151 #ifdef COMPAT_IA32 2152 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 2153 struct timeval32 tv32; 2154 2155 error = sooptcopyin(sopt, &tv32, sizeof tv32, 2156 sizeof tv32); 2157 CP(tv32, tv, tv_sec); 2158 CP(tv32, tv, tv_usec); 2159 } else 2160 #endif 2161 error = sooptcopyin(sopt, &tv, sizeof tv, 2162 sizeof tv); 2163 if (error) 2164 goto bad; 2165 2166 /* assert(hz > 0); */ 2167 if (tv.tv_sec < 0 || tv.tv_sec > INT_MAX / hz || 2168 tv.tv_usec < 0 || tv.tv_usec >= 1000000) { 2169 error = EDOM; 2170 goto bad; 2171 } 2172 /* assert(tick > 0); */ 2173 /* assert(ULONG_MAX - INT_MAX >= 1000000); */ 2174 val = (u_long)(tv.tv_sec * hz) + tv.tv_usec / tick; 2175 if (val > INT_MAX) { 2176 error = EDOM; 2177 goto bad; 2178 } 2179 if (val == 0 && tv.tv_usec != 0) 2180 val = 1; 2181 2182 switch (sopt->sopt_name) { 2183 case SO_SNDTIMEO: 2184 so->so_snd.sb_timeo = val; 2185 break; 2186 case SO_RCVTIMEO: 2187 so->so_rcv.sb_timeo = val; 2188 break; 2189 } 2190 break; 2191 2192 case SO_LABEL: 2193 #ifdef MAC 2194 error = sooptcopyin(sopt, &extmac, sizeof extmac, 2195 sizeof extmac); 2196 if (error) 2197 goto bad; 2198 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 2199 so, &extmac); 2200 #else 2201 error = EOPNOTSUPP; 2202 #endif 2203 break; 2204 2205 default: 2206 error = ENOPROTOOPT; 2207 break; 2208 } 2209 if (error == 0 && so->so_proto != NULL && 2210 so->so_proto->pr_ctloutput != NULL) { 2211 (void) ((*so->so_proto->pr_ctloutput) 2212 (so, sopt)); 2213 } 2214 } 2215 bad: 2216 return (error); 2217 } 2218 2219 /* 2220 * Helper routine for getsockopt. 2221 */ 2222 int 2223 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 2224 { 2225 int error; 2226 size_t valsize; 2227 2228 error = 0; 2229 2230 /* 2231 * Documented get behavior is that we always return a value, possibly 2232 * truncated to fit in the user's buffer. Traditional behavior is 2233 * that we always tell the user precisely how much we copied, rather 2234 * than something useful like the total amount we had available for 2235 * her. Note that this interface is not idempotent; the entire 2236 * answer must generated ahead of time. 2237 */ 2238 valsize = min(len, sopt->sopt_valsize); 2239 sopt->sopt_valsize = valsize; 2240 if (sopt->sopt_val != NULL) { 2241 if (sopt->sopt_td != NULL) 2242 error = copyout(buf, sopt->sopt_val, valsize); 2243 else 2244 bcopy(buf, sopt->sopt_val, valsize); 2245 } 2246 return (error); 2247 } 2248 2249 int 2250 sogetopt(so, sopt) 2251 struct socket *so; 2252 struct sockopt *sopt; 2253 { 2254 int error, optval; 2255 struct linger l; 2256 struct timeval tv; 2257 #ifdef MAC 2258 struct mac extmac; 2259 #endif 2260 2261 error = 0; 2262 if (sopt->sopt_level != SOL_SOCKET) { 2263 if (so->so_proto && so->so_proto->pr_ctloutput) { 2264 return ((*so->so_proto->pr_ctloutput) 2265 (so, sopt)); 2266 } else 2267 return (ENOPROTOOPT); 2268 } else { 2269 switch (sopt->sopt_name) { 2270 #ifdef INET 2271 case SO_ACCEPTFILTER: 2272 error = do_getopt_accept_filter(so, sopt); 2273 break; 2274 #endif 2275 case SO_LINGER: 2276 SOCK_LOCK(so); 2277 l.l_onoff = so->so_options & SO_LINGER; 2278 l.l_linger = so->so_linger; 2279 SOCK_UNLOCK(so); 2280 error = sooptcopyout(sopt, &l, sizeof l); 2281 break; 2282 2283 case SO_USELOOPBACK: 2284 case SO_DONTROUTE: 2285 case SO_DEBUG: 2286 case SO_KEEPALIVE: 2287 case SO_REUSEADDR: 2288 case SO_REUSEPORT: 2289 case SO_BROADCAST: 2290 case SO_OOBINLINE: 2291 case SO_ACCEPTCONN: 2292 case SO_TIMESTAMP: 2293 case SO_BINTIME: 2294 case SO_NOSIGPIPE: 2295 optval = so->so_options & sopt->sopt_name; 2296 integer: 2297 error = sooptcopyout(sopt, &optval, sizeof optval); 2298 break; 2299 2300 case SO_TYPE: 2301 optval = so->so_type; 2302 goto integer; 2303 2304 case SO_ERROR: 2305 SOCK_LOCK(so); 2306 optval = so->so_error; 2307 so->so_error = 0; 2308 SOCK_UNLOCK(so); 2309 goto integer; 2310 2311 case SO_SNDBUF: 2312 optval = so->so_snd.sb_hiwat; 2313 goto integer; 2314 2315 case SO_RCVBUF: 2316 optval = so->so_rcv.sb_hiwat; 2317 goto integer; 2318 2319 case SO_SNDLOWAT: 2320 optval = so->so_snd.sb_lowat; 2321 goto integer; 2322 2323 case SO_RCVLOWAT: 2324 optval = so->so_rcv.sb_lowat; 2325 goto integer; 2326 2327 case SO_SNDTIMEO: 2328 case SO_RCVTIMEO: 2329 optval = (sopt->sopt_name == SO_SNDTIMEO ? 2330 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 2331 2332 tv.tv_sec = optval / hz; 2333 tv.tv_usec = (optval % hz) * tick; 2334 #ifdef COMPAT_IA32 2335 if (curthread->td_proc->p_sysent == &ia32_freebsd_sysvec) { 2336 struct timeval32 tv32; 2337 2338 CP(tv, tv32, tv_sec); 2339 CP(tv, tv32, tv_usec); 2340 error = sooptcopyout(sopt, &tv32, sizeof tv32); 2341 } else 2342 #endif 2343 error = sooptcopyout(sopt, &tv, sizeof tv); 2344 break; 2345 2346 case SO_LABEL: 2347 #ifdef MAC 2348 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2349 sizeof(extmac)); 2350 if (error) 2351 return (error); 2352 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 2353 so, &extmac); 2354 if (error) 2355 return (error); 2356 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2357 #else 2358 error = EOPNOTSUPP; 2359 #endif 2360 break; 2361 2362 case SO_PEERLABEL: 2363 #ifdef MAC 2364 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 2365 sizeof(extmac)); 2366 if (error) 2367 return (error); 2368 error = mac_getsockopt_peerlabel( 2369 sopt->sopt_td->td_ucred, so, &extmac); 2370 if (error) 2371 return (error); 2372 error = sooptcopyout(sopt, &extmac, sizeof extmac); 2373 #else 2374 error = EOPNOTSUPP; 2375 #endif 2376 break; 2377 2378 case SO_LISTENQLIMIT: 2379 optval = so->so_qlimit; 2380 goto integer; 2381 2382 case SO_LISTENQLEN: 2383 optval = so->so_qlen; 2384 goto integer; 2385 2386 case SO_LISTENINCQLEN: 2387 optval = so->so_incqlen; 2388 goto integer; 2389 2390 default: 2391 error = ENOPROTOOPT; 2392 break; 2393 } 2394 return (error); 2395 } 2396 } 2397 2398 /* XXX; prepare mbuf for (__FreeBSD__ < 3) routines. */ 2399 int 2400 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 2401 { 2402 struct mbuf *m, *m_prev; 2403 int sopt_size = sopt->sopt_valsize; 2404 2405 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2406 if (m == NULL) 2407 return ENOBUFS; 2408 if (sopt_size > MLEN) { 2409 MCLGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT); 2410 if ((m->m_flags & M_EXT) == 0) { 2411 m_free(m); 2412 return ENOBUFS; 2413 } 2414 m->m_len = min(MCLBYTES, sopt_size); 2415 } else { 2416 m->m_len = min(MLEN, sopt_size); 2417 } 2418 sopt_size -= m->m_len; 2419 *mp = m; 2420 m_prev = m; 2421 2422 while (sopt_size) { 2423 MGET(m, sopt->sopt_td ? M_TRYWAIT : M_DONTWAIT, MT_DATA); 2424 if (m == NULL) { 2425 m_freem(*mp); 2426 return ENOBUFS; 2427 } 2428 if (sopt_size > MLEN) { 2429 MCLGET(m, sopt->sopt_td != NULL ? M_TRYWAIT : 2430 M_DONTWAIT); 2431 if ((m->m_flags & M_EXT) == 0) { 2432 m_freem(m); 2433 m_freem(*mp); 2434 return ENOBUFS; 2435 } 2436 m->m_len = min(MCLBYTES, sopt_size); 2437 } else { 2438 m->m_len = min(MLEN, sopt_size); 2439 } 2440 sopt_size -= m->m_len; 2441 m_prev->m_next = m; 2442 m_prev = m; 2443 } 2444 return (0); 2445 } 2446 2447 /* XXX; copyin sopt data into mbuf chain for (__FreeBSD__ < 3) routines. */ 2448 int 2449 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 2450 { 2451 struct mbuf *m0 = m; 2452 2453 if (sopt->sopt_val == NULL) 2454 return (0); 2455 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2456 if (sopt->sopt_td != NULL) { 2457 int error; 2458 2459 error = copyin(sopt->sopt_val, mtod(m, char *), 2460 m->m_len); 2461 if (error != 0) { 2462 m_freem(m0); 2463 return(error); 2464 } 2465 } else 2466 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 2467 sopt->sopt_valsize -= m->m_len; 2468 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2469 m = m->m_next; 2470 } 2471 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 2472 panic("ip6_sooptmcopyin"); 2473 return (0); 2474 } 2475 2476 /* XXX; copyout mbuf chain data into soopt for (__FreeBSD__ < 3) routines. */ 2477 int 2478 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 2479 { 2480 struct mbuf *m0 = m; 2481 size_t valsize = 0; 2482 2483 if (sopt->sopt_val == NULL) 2484 return (0); 2485 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 2486 if (sopt->sopt_td != NULL) { 2487 int error; 2488 2489 error = copyout(mtod(m, char *), sopt->sopt_val, 2490 m->m_len); 2491 if (error != 0) { 2492 m_freem(m0); 2493 return(error); 2494 } 2495 } else 2496 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 2497 sopt->sopt_valsize -= m->m_len; 2498 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 2499 valsize += m->m_len; 2500 m = m->m_next; 2501 } 2502 if (m != NULL) { 2503 /* enough soopt buffer should be given from user-land */ 2504 m_freem(m0); 2505 return(EINVAL); 2506 } 2507 sopt->sopt_valsize = valsize; 2508 return (0); 2509 } 2510 2511 /* 2512 * sohasoutofband(): protocol notifies socket layer of the arrival of new 2513 * out-of-band data, which will then notify socket consumers. 2514 */ 2515 void 2516 sohasoutofband(so) 2517 struct socket *so; 2518 { 2519 if (so->so_sigio != NULL) 2520 pgsigio(&so->so_sigio, SIGURG, 0); 2521 selwakeuppri(&so->so_rcv.sb_sel, PSOCK); 2522 } 2523 2524 int 2525 sopoll(struct socket *so, int events, struct ucred *active_cred, 2526 struct thread *td) 2527 { 2528 2529 /* XXXRW: Temporary debugging. */ 2530 KASSERT(so->so_proto->pr_usrreqs->pru_sopoll != sopoll, 2531 ("sopoll: protocol calls sopoll")); 2532 2533 return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred, 2534 td)); 2535 } 2536 2537 int 2538 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, 2539 struct thread *td) 2540 { 2541 int revents = 0; 2542 2543 SOCKBUF_LOCK(&so->so_snd); 2544 SOCKBUF_LOCK(&so->so_rcv); 2545 if (events & (POLLIN | POLLRDNORM)) 2546 if (soreadable(so)) 2547 revents |= events & (POLLIN | POLLRDNORM); 2548 2549 if (events & POLLINIGNEOF) 2550 if (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat || 2551 !TAILQ_EMPTY(&so->so_comp) || so->so_error) 2552 revents |= POLLINIGNEOF; 2553 2554 if (events & (POLLOUT | POLLWRNORM)) 2555 if (sowriteable(so)) 2556 revents |= events & (POLLOUT | POLLWRNORM); 2557 2558 if (events & (POLLPRI | POLLRDBAND)) 2559 if (so->so_oobmark || (so->so_rcv.sb_state & SBS_RCVATMARK)) 2560 revents |= events & (POLLPRI | POLLRDBAND); 2561 2562 if (revents == 0) { 2563 if (events & 2564 (POLLIN | POLLINIGNEOF | POLLPRI | POLLRDNORM | 2565 POLLRDBAND)) { 2566 selrecord(td, &so->so_rcv.sb_sel); 2567 so->so_rcv.sb_flags |= SB_SEL; 2568 } 2569 2570 if (events & (POLLOUT | POLLWRNORM)) { 2571 selrecord(td, &so->so_snd.sb_sel); 2572 so->so_snd.sb_flags |= SB_SEL; 2573 } 2574 } 2575 2576 SOCKBUF_UNLOCK(&so->so_rcv); 2577 SOCKBUF_UNLOCK(&so->so_snd); 2578 return (revents); 2579 } 2580 2581 int 2582 soo_kqfilter(struct file *fp, struct knote *kn) 2583 { 2584 struct socket *so = kn->kn_fp->f_data; 2585 struct sockbuf *sb; 2586 2587 switch (kn->kn_filter) { 2588 case EVFILT_READ: 2589 if (so->so_options & SO_ACCEPTCONN) 2590 kn->kn_fop = &solisten_filtops; 2591 else 2592 kn->kn_fop = &soread_filtops; 2593 sb = &so->so_rcv; 2594 break; 2595 case EVFILT_WRITE: 2596 kn->kn_fop = &sowrite_filtops; 2597 sb = &so->so_snd; 2598 break; 2599 default: 2600 return (EINVAL); 2601 } 2602 2603 SOCKBUF_LOCK(sb); 2604 knlist_add(&sb->sb_sel.si_note, kn, 1); 2605 sb->sb_flags |= SB_KNOTE; 2606 SOCKBUF_UNLOCK(sb); 2607 return (0); 2608 } 2609 2610 static void 2611 filt_sordetach(struct knote *kn) 2612 { 2613 struct socket *so = kn->kn_fp->f_data; 2614 2615 SOCKBUF_LOCK(&so->so_rcv); 2616 knlist_remove(&so->so_rcv.sb_sel.si_note, kn, 1); 2617 if (knlist_empty(&so->so_rcv.sb_sel.si_note)) 2618 so->so_rcv.sb_flags &= ~SB_KNOTE; 2619 SOCKBUF_UNLOCK(&so->so_rcv); 2620 } 2621 2622 /*ARGSUSED*/ 2623 static int 2624 filt_soread(struct knote *kn, long hint) 2625 { 2626 struct socket *so; 2627 2628 so = kn->kn_fp->f_data; 2629 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2630 2631 kn->kn_data = so->so_rcv.sb_cc - so->so_rcv.sb_ctl; 2632 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 2633 kn->kn_flags |= EV_EOF; 2634 kn->kn_fflags = so->so_error; 2635 return (1); 2636 } else if (so->so_error) /* temporary udp error */ 2637 return (1); 2638 else if (kn->kn_sfflags & NOTE_LOWAT) 2639 return (kn->kn_data >= kn->kn_sdata); 2640 else 2641 return (so->so_rcv.sb_cc >= so->so_rcv.sb_lowat); 2642 } 2643 2644 static void 2645 filt_sowdetach(struct knote *kn) 2646 { 2647 struct socket *so = kn->kn_fp->f_data; 2648 2649 SOCKBUF_LOCK(&so->so_snd); 2650 knlist_remove(&so->so_snd.sb_sel.si_note, kn, 1); 2651 if (knlist_empty(&so->so_snd.sb_sel.si_note)) 2652 so->so_snd.sb_flags &= ~SB_KNOTE; 2653 SOCKBUF_UNLOCK(&so->so_snd); 2654 } 2655 2656 /*ARGSUSED*/ 2657 static int 2658 filt_sowrite(struct knote *kn, long hint) 2659 { 2660 struct socket *so; 2661 2662 so = kn->kn_fp->f_data; 2663 SOCKBUF_LOCK_ASSERT(&so->so_snd); 2664 kn->kn_data = sbspace(&so->so_snd); 2665 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 2666 kn->kn_flags |= EV_EOF; 2667 kn->kn_fflags = so->so_error; 2668 return (1); 2669 } else if (so->so_error) /* temporary udp error */ 2670 return (1); 2671 else if (((so->so_state & SS_ISCONNECTED) == 0) && 2672 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 2673 return (0); 2674 else if (kn->kn_sfflags & NOTE_LOWAT) 2675 return (kn->kn_data >= kn->kn_sdata); 2676 else 2677 return (kn->kn_data >= so->so_snd.sb_lowat); 2678 } 2679 2680 /*ARGSUSED*/ 2681 static int 2682 filt_solisten(struct knote *kn, long hint) 2683 { 2684 struct socket *so = kn->kn_fp->f_data; 2685 2686 kn->kn_data = so->so_qlen; 2687 return (! TAILQ_EMPTY(&so->so_comp)); 2688 } 2689 2690 int 2691 socheckuid(struct socket *so, uid_t uid) 2692 { 2693 2694 if (so == NULL) 2695 return (EPERM); 2696 if (so->so_cred->cr_uid != uid) 2697 return (EPERM); 2698 return (0); 2699 } 2700 2701 static int 2702 somaxconn_sysctl(SYSCTL_HANDLER_ARGS) 2703 { 2704 int error; 2705 int val; 2706 2707 val = somaxconn; 2708 error = sysctl_handle_int(oidp, &val, sizeof(int), req); 2709 if (error || !req->newptr ) 2710 return (error); 2711 2712 if (val < 1 || val > USHRT_MAX) 2713 return (EINVAL); 2714 2715 somaxconn = val; 2716 return (0); 2717 } 2718