1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. 4 * Copyright (c) 2004 The FreeBSD Foundation 5 * Copyright (c) 2004-2008 Robert N. M. Watson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pru_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pru_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pru_attach() has 50 * been successfully called. If pru_attach() returned an error, 51 * pru_detach() will not be called. Socket layer private. 52 * 53 * pru_abort() and pru_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pru_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 * 96 * NOTE: With regard to VNETs the general rule is that callers do not set 97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 98 * sofree() (and with that sorele(), sotryfree()), as well as sonewconn() 99 * and sorflush(), which are usually called from a pre-set VNET context. 100 * sopoll() currently does not need a VNET context to be set. 101 */ 102 103 #include <sys/cdefs.h> 104 __FBSDID("$FreeBSD$"); 105 106 #include "opt_inet.h" 107 #include "opt_inet6.h" 108 #include "opt_compat.h" 109 #include "opt_sctp.h" 110 111 #include <sys/param.h> 112 #include <sys/systm.h> 113 #include <sys/fcntl.h> 114 #include <sys/limits.h> 115 #include <sys/lock.h> 116 #include <sys/mac.h> 117 #include <sys/malloc.h> 118 #include <sys/mbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/domain.h> 121 #include <sys/file.h> /* for struct knote */ 122 #include <sys/hhook.h> 123 #include <sys/kernel.h> 124 #include <sys/khelp.h> 125 #include <sys/event.h> 126 #include <sys/eventhandler.h> 127 #include <sys/poll.h> 128 #include <sys/proc.h> 129 #include <sys/protosw.h> 130 #include <sys/socket.h> 131 #include <sys/socketvar.h> 132 #include <sys/resourcevar.h> 133 #include <net/route.h> 134 #include <sys/signalvar.h> 135 #include <sys/stat.h> 136 #include <sys/sx.h> 137 #include <sys/sysctl.h> 138 #include <sys/taskqueue.h> 139 #include <sys/uio.h> 140 #include <sys/jail.h> 141 #include <sys/syslog.h> 142 #include <netinet/in.h> 143 144 #include <net/vnet.h> 145 146 #include <security/mac/mac_framework.h> 147 148 #include <vm/uma.h> 149 150 #ifdef COMPAT_FREEBSD32 151 #include <sys/mount.h> 152 #include <sys/sysent.h> 153 #include <compat/freebsd32/freebsd32.h> 154 #endif 155 156 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 157 int flags); 158 static void so_rdknl_lock(void *); 159 static void so_rdknl_unlock(void *); 160 static void so_rdknl_assert_locked(void *); 161 static void so_rdknl_assert_unlocked(void *); 162 static void so_wrknl_lock(void *); 163 static void so_wrknl_unlock(void *); 164 static void so_wrknl_assert_locked(void *); 165 static void so_wrknl_assert_unlocked(void *); 166 167 static void filt_sordetach(struct knote *kn); 168 static int filt_soread(struct knote *kn, long hint); 169 static void filt_sowdetach(struct knote *kn); 170 static int filt_sowrite(struct knote *kn, long hint); 171 static int filt_soempty(struct knote *kn, long hint); 172 static int inline hhook_run_socket(struct socket *so, void *hctx, int32_t h_id); 173 fo_kqfilter_t soo_kqfilter; 174 175 static struct filterops soread_filtops = { 176 .f_isfd = 1, 177 .f_detach = filt_sordetach, 178 .f_event = filt_soread, 179 }; 180 static struct filterops sowrite_filtops = { 181 .f_isfd = 1, 182 .f_detach = filt_sowdetach, 183 .f_event = filt_sowrite, 184 }; 185 static struct filterops soempty_filtops = { 186 .f_isfd = 1, 187 .f_detach = filt_sowdetach, 188 .f_event = filt_soempty, 189 }; 190 191 so_gen_t so_gencnt; /* generation count for sockets */ 192 193 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 194 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 195 196 #define VNET_SO_ASSERT(so) \ 197 VNET_ASSERT(curvnet != NULL, \ 198 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 199 200 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 201 #define V_socket_hhh VNET(socket_hhh) 202 203 /* 204 * Limit on the number of connections in the listen queue waiting 205 * for accept(2). 206 * NB: The original sysctl somaxconn is still available but hidden 207 * to prevent confusion about the actual purpose of this number. 208 */ 209 static u_int somaxconn = SOMAXCONN; 210 211 static int 212 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 213 { 214 int error; 215 int val; 216 217 val = somaxconn; 218 error = sysctl_handle_int(oidp, &val, 0, req); 219 if (error || !req->newptr ) 220 return (error); 221 222 /* 223 * The purpose of the UINT_MAX / 3 limit, is so that the formula 224 * 3 * so_qlimit / 2 225 * below, will not overflow. 226 */ 227 228 if (val < 1 || val > UINT_MAX / 3) 229 return (EINVAL); 230 231 somaxconn = val; 232 return (0); 233 } 234 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, CTLTYPE_UINT | CTLFLAG_RW, 235 0, sizeof(int), sysctl_somaxconn, "I", 236 "Maximum listen socket pending connection accept queue size"); 237 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 238 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP, 239 0, sizeof(int), sysctl_somaxconn, "I", 240 "Maximum listen socket pending connection accept queue size (compat)"); 241 242 static int numopensockets; 243 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 244 &numopensockets, 0, "Number of open sockets"); 245 246 /* 247 * accept_mtx locks down per-socket fields relating to accept queues. See 248 * socketvar.h for an annotation of the protected fields of struct socket. 249 */ 250 struct mtx accept_mtx; 251 MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 252 253 /* 254 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 255 * so_gencnt field. 256 */ 257 static struct mtx so_global_mtx; 258 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 259 260 /* 261 * General IPC sysctl name space, used by sockets and a variety of other IPC 262 * types. 263 */ 264 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC"); 265 266 /* 267 * Initialize the socket subsystem and set up the socket 268 * memory allocator. 269 */ 270 static uma_zone_t socket_zone; 271 int maxsockets; 272 273 static void 274 socket_zone_change(void *tag) 275 { 276 277 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 278 } 279 280 static void 281 socket_hhook_register(int subtype) 282 { 283 284 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 285 &V_socket_hhh[subtype], 286 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 287 printf("%s: WARNING: unable to register hook\n", __func__); 288 } 289 290 static void 291 socket_hhook_deregister(int subtype) 292 { 293 294 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 295 printf("%s: WARNING: unable to deregister hook\n", __func__); 296 } 297 298 static void 299 socket_init(void *tag) 300 { 301 302 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 303 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 304 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 305 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 306 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 307 EVENTHANDLER_PRI_FIRST); 308 } 309 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 310 311 static void 312 socket_vnet_init(const void *unused __unused) 313 { 314 int i; 315 316 /* We expect a contiguous range */ 317 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 318 socket_hhook_register(i); 319 } 320 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 321 socket_vnet_init, NULL); 322 323 static void 324 socket_vnet_uninit(const void *unused __unused) 325 { 326 int i; 327 328 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 329 socket_hhook_deregister(i); 330 } 331 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 332 socket_vnet_uninit, NULL); 333 334 /* 335 * Initialise maxsockets. This SYSINIT must be run after 336 * tunable_mbinit(). 337 */ 338 static void 339 init_maxsockets(void *ignored) 340 { 341 342 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 343 maxsockets = imax(maxsockets, maxfiles); 344 } 345 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 346 347 /* 348 * Sysctl to get and set the maximum global sockets limit. Notify protocols 349 * of the change so that they can update their dependent limits as required. 350 */ 351 static int 352 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 353 { 354 int error, newmaxsockets; 355 356 newmaxsockets = maxsockets; 357 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 358 if (error == 0 && req->newptr) { 359 if (newmaxsockets > maxsockets && 360 newmaxsockets <= maxfiles) { 361 maxsockets = newmaxsockets; 362 EVENTHANDLER_INVOKE(maxsockets_change); 363 } else 364 error = EINVAL; 365 } 366 return (error); 367 } 368 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, CTLTYPE_INT|CTLFLAG_RW, 369 &maxsockets, 0, sysctl_maxsockets, "IU", 370 "Maximum number of sockets available"); 371 372 /* 373 * Socket operation routines. These routines are called by the routines in 374 * sys_socket.c or from a system process, and implement the semantics of 375 * socket operations by switching out to the protocol specific routines. 376 */ 377 378 /* 379 * Get a socket structure from our zone, and initialize it. Note that it 380 * would probably be better to allocate socket and PCB at the same time, but 381 * I'm not convinced that all the protocols can be easily modified to do 382 * this. 383 * 384 * soalloc() returns a socket with a ref count of 0. 385 */ 386 static struct socket * 387 soalloc(struct vnet *vnet) 388 { 389 struct socket *so; 390 391 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 392 if (so == NULL) 393 return (NULL); 394 #ifdef MAC 395 if (mac_socket_init(so, M_NOWAIT) != 0) { 396 uma_zfree(socket_zone, so); 397 return (NULL); 398 } 399 #endif 400 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 401 uma_zfree(socket_zone, so); 402 return (NULL); 403 } 404 405 /* 406 * The socket locking protocol allows to lock 2 sockets at a time, 407 * however, the first one must be a listening socket. WITNESS lacks 408 * a feature to change class of an existing lock, so we use DUPOK. 409 */ 410 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 411 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 412 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 413 so->so_rcv.sb_sel = &so->so_rdsel; 414 so->so_snd.sb_sel = &so->so_wrsel; 415 sx_init(&so->so_snd.sb_sx, "so_snd_sx"); 416 sx_init(&so->so_rcv.sb_sx, "so_rcv_sx"); 417 TAILQ_INIT(&so->so_snd.sb_aiojobq); 418 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 419 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 420 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 421 #ifdef VIMAGE 422 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 423 __func__, __LINE__, so)); 424 so->so_vnet = vnet; 425 #endif 426 /* We shouldn't need the so_global_mtx */ 427 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 428 /* Do we need more comprehensive error returns? */ 429 uma_zfree(socket_zone, so); 430 return (NULL); 431 } 432 mtx_lock(&so_global_mtx); 433 so->so_gencnt = ++so_gencnt; 434 ++numopensockets; 435 #ifdef VIMAGE 436 vnet->vnet_sockcnt++; 437 #endif 438 mtx_unlock(&so_global_mtx); 439 440 return (so); 441 } 442 443 /* 444 * Free the storage associated with a socket at the socket layer, tear down 445 * locks, labels, etc. All protocol state is assumed already to have been 446 * torn down (and possibly never set up) by the caller. 447 */ 448 static void 449 sodealloc(struct socket *so) 450 { 451 452 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 453 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 454 455 mtx_lock(&so_global_mtx); 456 so->so_gencnt = ++so_gencnt; 457 --numopensockets; /* Could be below, but faster here. */ 458 #ifdef VIMAGE 459 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 460 __func__, __LINE__, so)); 461 so->so_vnet->vnet_sockcnt--; 462 #endif 463 mtx_unlock(&so_global_mtx); 464 #ifdef MAC 465 mac_socket_destroy(so); 466 #endif 467 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 468 469 crfree(so->so_cred); 470 khelp_destroy_osd(&so->osd); 471 if (SOLISTENING(so)) { 472 if (so->sol_accept_filter != NULL) 473 accept_filt_setopt(so, NULL); 474 } else { 475 if (so->so_rcv.sb_hiwat) 476 (void)chgsbsize(so->so_cred->cr_uidinfo, 477 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 478 if (so->so_snd.sb_hiwat) 479 (void)chgsbsize(so->so_cred->cr_uidinfo, 480 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 481 sx_destroy(&so->so_snd.sb_sx); 482 sx_destroy(&so->so_rcv.sb_sx); 483 SOCKBUF_LOCK_DESTROY(&so->so_snd); 484 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 485 } 486 mtx_destroy(&so->so_lock); 487 uma_zfree(socket_zone, so); 488 } 489 490 /* 491 * socreate returns a socket with a ref count of 1. The socket should be 492 * closed with soclose(). 493 */ 494 int 495 socreate(int dom, struct socket **aso, int type, int proto, 496 struct ucred *cred, struct thread *td) 497 { 498 struct protosw *prp; 499 struct socket *so; 500 int error; 501 502 if (proto) 503 prp = pffindproto(dom, proto, type); 504 else 505 prp = pffindtype(dom, type); 506 507 if (prp == NULL) { 508 /* No support for domain. */ 509 if (pffinddomain(dom) == NULL) 510 return (EAFNOSUPPORT); 511 /* No support for socket type. */ 512 if (proto == 0 && type != 0) 513 return (EPROTOTYPE); 514 return (EPROTONOSUPPORT); 515 } 516 if (prp->pr_usrreqs->pru_attach == NULL || 517 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 518 return (EPROTONOSUPPORT); 519 520 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 521 return (EPROTONOSUPPORT); 522 523 if (prp->pr_type != type) 524 return (EPROTOTYPE); 525 so = soalloc(CRED_TO_VNET(cred)); 526 if (so == NULL) 527 return (ENOBUFS); 528 529 so->so_type = type; 530 so->so_cred = crhold(cred); 531 if ((prp->pr_domain->dom_family == PF_INET) || 532 (prp->pr_domain->dom_family == PF_INET6) || 533 (prp->pr_domain->dom_family == PF_ROUTE)) 534 so->so_fibnum = td->td_proc->p_fibnum; 535 else 536 so->so_fibnum = 0; 537 so->so_proto = prp; 538 #ifdef MAC 539 mac_socket_create(cred, so); 540 #endif 541 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 542 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 543 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 544 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 545 /* 546 * Auto-sizing of socket buffers is managed by the protocols and 547 * the appropriate flags must be set in the pru_attach function. 548 */ 549 CURVNET_SET(so->so_vnet); 550 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 551 CURVNET_RESTORE(); 552 if (error) { 553 sodealloc(so); 554 return (error); 555 } 556 soref(so); 557 *aso = so; 558 return (0); 559 } 560 561 #ifdef REGRESSION 562 static int regression_sonewconn_earlytest = 1; 563 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 564 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 565 #endif 566 567 /* 568 * When an attempt at a new connection is noted on a socket which accepts 569 * connections, sonewconn is called. If the connection is possible (subject 570 * to space constraints, etc.) then we allocate a new structure, properly 571 * linked into the data structure of the original socket, and return this. 572 * Connstatus may be 0, or SS_ISCONFIRMING, or SS_ISCONNECTED. 573 * 574 * Note: the ref count on the socket is 0 on return. 575 */ 576 struct socket * 577 sonewconn(struct socket *head, int connstatus) 578 { 579 static struct timeval lastover; 580 static struct timeval overinterval = { 60, 0 }; 581 static int overcount; 582 583 struct socket *so; 584 u_int over; 585 586 SOLISTEN_LOCK(head); 587 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 588 SOLISTEN_UNLOCK(head); 589 #ifdef REGRESSION 590 if (regression_sonewconn_earlytest && over) { 591 #else 592 if (over) { 593 #endif 594 overcount++; 595 596 if (ratecheck(&lastover, &overinterval)) { 597 log(LOG_DEBUG, "%s: pcb %p: Listen queue overflow: " 598 "%i already in queue awaiting acceptance " 599 "(%d occurrences)\n", 600 __func__, head->so_pcb, head->sol_qlen, overcount); 601 602 overcount = 0; 603 } 604 605 return (NULL); 606 } 607 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 608 __func__, head)); 609 so = soalloc(head->so_vnet); 610 if (so == NULL) { 611 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 612 "limit reached or out of memory\n", 613 __func__, head->so_pcb); 614 return (NULL); 615 } 616 so->so_listen = head; 617 so->so_type = head->so_type; 618 so->so_linger = head->so_linger; 619 so->so_state = head->so_state | SS_NOFDREF; 620 so->so_fibnum = head->so_fibnum; 621 so->so_proto = head->so_proto; 622 so->so_cred = crhold(head->so_cred); 623 #ifdef MAC 624 mac_socket_newconn(head, so); 625 #endif 626 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 627 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 628 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 629 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 630 VNET_SO_ASSERT(head); 631 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { 632 sodealloc(so); 633 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 634 __func__, head->so_pcb); 635 return (NULL); 636 } 637 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 638 sodealloc(so); 639 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 640 __func__, head->so_pcb); 641 return (NULL); 642 } 643 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 644 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 645 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 646 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 647 so->so_rcv.sb_flags |= head->sol_sbrcv_flags & SB_AUTOSIZE; 648 so->so_snd.sb_flags |= head->sol_sbsnd_flags & SB_AUTOSIZE; 649 650 SOLISTEN_LOCK(head); 651 if (head->sol_accept_filter != NULL) 652 connstatus = 0; 653 so->so_state |= connstatus; 654 so->so_options = head->so_options & ~SO_ACCEPTCONN; 655 soref(head); /* A socket on (in)complete queue refs head. */ 656 if (connstatus) { 657 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 658 so->so_qstate = SQ_COMP; 659 head->sol_qlen++; 660 solisten_wakeup(head); /* unlocks */ 661 } else { 662 /* 663 * Keep removing sockets from the head until there's room for 664 * us to insert on the tail. In pre-locking revisions, this 665 * was a simple if(), but as we could be racing with other 666 * threads and soabort() requires dropping locks, we must 667 * loop waiting for the condition to be true. 668 */ 669 while (head->sol_incqlen > head->sol_qlimit) { 670 struct socket *sp; 671 672 sp = TAILQ_FIRST(&head->sol_incomp); 673 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 674 head->sol_incqlen--; 675 SOCK_LOCK(sp); 676 sp->so_qstate = SQ_NONE; 677 sp->so_listen = NULL; 678 SOCK_UNLOCK(sp); 679 sorele(head); /* does SOLISTEN_UNLOCK, head stays */ 680 soabort(sp); 681 SOLISTEN_LOCK(head); 682 } 683 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 684 so->so_qstate = SQ_INCOMP; 685 head->sol_incqlen++; 686 SOLISTEN_UNLOCK(head); 687 } 688 return (so); 689 } 690 691 #ifdef SCTP 692 /* 693 * Socket part of sctp_peeloff(). Detach a new socket from an 694 * association. The new socket is returned with a reference. 695 */ 696 struct socket * 697 sopeeloff(struct socket *head) 698 { 699 struct socket *so; 700 701 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 702 __func__, __LINE__, head)); 703 so = soalloc(head->so_vnet); 704 if (so == NULL) { 705 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 706 "limit reached or out of memory\n", 707 __func__, head->so_pcb); 708 return (NULL); 709 } 710 so->so_type = head->so_type; 711 so->so_options = head->so_options; 712 so->so_linger = head->so_linger; 713 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 714 so->so_fibnum = head->so_fibnum; 715 so->so_proto = head->so_proto; 716 so->so_cred = crhold(head->so_cred); 717 #ifdef MAC 718 mac_socket_newconn(head, so); 719 #endif 720 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 721 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 722 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 723 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 724 VNET_SO_ASSERT(head); 725 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 726 sodealloc(so); 727 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 728 __func__, head->so_pcb); 729 return (NULL); 730 } 731 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 732 sodealloc(so); 733 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 734 __func__, head->so_pcb); 735 return (NULL); 736 } 737 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 738 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 739 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 740 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 741 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 742 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 743 744 soref(so); 745 746 return (so); 747 } 748 #endif /* SCTP */ 749 750 int 751 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 752 { 753 int error; 754 755 CURVNET_SET(so->so_vnet); 756 error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td); 757 CURVNET_RESTORE(); 758 return (error); 759 } 760 761 int 762 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 763 { 764 int error; 765 766 CURVNET_SET(so->so_vnet); 767 error = (*so->so_proto->pr_usrreqs->pru_bindat)(fd, so, nam, td); 768 CURVNET_RESTORE(); 769 return (error); 770 } 771 772 /* 773 * solisten() transitions a socket from a non-listening state to a listening 774 * state, but can also be used to update the listen queue depth on an 775 * existing listen socket. The protocol will call back into the sockets 776 * layer using solisten_proto_check() and solisten_proto() to check and set 777 * socket-layer listen state. Call backs are used so that the protocol can 778 * acquire both protocol and socket layer locks in whatever order is required 779 * by the protocol. 780 * 781 * Protocol implementors are advised to hold the socket lock across the 782 * socket-layer test and set to avoid races at the socket layer. 783 */ 784 int 785 solisten(struct socket *so, int backlog, struct thread *td) 786 { 787 int error; 788 789 CURVNET_SET(so->so_vnet); 790 error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td); 791 CURVNET_RESTORE(); 792 return (error); 793 } 794 795 int 796 solisten_proto_check(struct socket *so) 797 { 798 799 SOCK_LOCK_ASSERT(so); 800 801 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 802 SS_ISDISCONNECTING)) 803 return (EINVAL); 804 return (0); 805 } 806 807 void 808 solisten_proto(struct socket *so, int backlog) 809 { 810 int sbrcv_lowat, sbsnd_lowat; 811 u_int sbrcv_hiwat, sbsnd_hiwat; 812 short sbrcv_flags, sbsnd_flags; 813 sbintime_t sbrcv_timeo, sbsnd_timeo; 814 815 SOCK_LOCK_ASSERT(so); 816 817 if (SOLISTENING(so)) 818 goto listening; 819 820 /* 821 * Change this socket to listening state. 822 */ 823 sbrcv_lowat = so->so_rcv.sb_lowat; 824 sbsnd_lowat = so->so_snd.sb_lowat; 825 sbrcv_hiwat = so->so_rcv.sb_hiwat; 826 sbsnd_hiwat = so->so_snd.sb_hiwat; 827 sbrcv_flags = so->so_rcv.sb_flags; 828 sbsnd_flags = so->so_snd.sb_flags; 829 sbrcv_timeo = so->so_rcv.sb_timeo; 830 sbsnd_timeo = so->so_snd.sb_timeo; 831 832 sbdestroy(&so->so_snd, so); 833 sbdestroy(&so->so_rcv, so); 834 sx_destroy(&so->so_snd.sb_sx); 835 sx_destroy(&so->so_rcv.sb_sx); 836 SOCKBUF_LOCK_DESTROY(&so->so_snd); 837 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 838 839 #ifdef INVARIANTS 840 bzero(&so->so_rcv, 841 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 842 #endif 843 844 so->sol_sbrcv_lowat = sbrcv_lowat; 845 so->sol_sbsnd_lowat = sbsnd_lowat; 846 so->sol_sbrcv_hiwat = sbrcv_hiwat; 847 so->sol_sbsnd_hiwat = sbsnd_hiwat; 848 so->sol_sbrcv_flags = sbrcv_flags; 849 so->sol_sbsnd_flags = sbsnd_flags; 850 so->sol_sbrcv_timeo = sbrcv_timeo; 851 so->sol_sbsnd_timeo = sbsnd_timeo; 852 853 so->sol_qlen = so->sol_incqlen = 0; 854 TAILQ_INIT(&so->sol_incomp); 855 TAILQ_INIT(&so->sol_comp); 856 857 so->sol_accept_filter = NULL; 858 so->sol_accept_filter_arg = NULL; 859 so->sol_accept_filter_str = NULL; 860 861 so->so_options |= SO_ACCEPTCONN; 862 863 listening: 864 if (backlog < 0 || backlog > somaxconn) 865 backlog = somaxconn; 866 so->sol_qlimit = backlog; 867 } 868 869 /* 870 * Wakeup listeners/subsystems once we have a complete connection. 871 * Enters with lock, returns unlocked. 872 */ 873 void 874 solisten_wakeup(struct socket *sol) 875 { 876 877 if (sol->sol_upcall != NULL) 878 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 879 else { 880 selwakeuppri(&sol->so_rdsel, PSOCK); 881 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 882 } 883 SOLISTEN_UNLOCK(sol); 884 wakeup_one(&sol->sol_comp); 885 } 886 887 /* 888 * Return single connection off a listening socket queue. Main consumer of 889 * the function is kern_accept4(). Some modules, that do their own accept 890 * management also use the function. 891 * 892 * Listening socket must be locked on entry and is returned unlocked on 893 * return. 894 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 895 */ 896 int 897 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 898 { 899 struct socket *so; 900 int error; 901 902 SOLISTEN_LOCK_ASSERT(head); 903 904 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 905 head->so_error == 0) { 906 error = msleep(&head->sol_comp, &head->so_lock, PSOCK | PCATCH, 907 "accept", 0); 908 if (error != 0) { 909 SOLISTEN_UNLOCK(head); 910 return (error); 911 } 912 } 913 if (head->so_error) { 914 error = head->so_error; 915 head->so_error = 0; 916 SOLISTEN_UNLOCK(head); 917 return (error); 918 } 919 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) { 920 SOLISTEN_UNLOCK(head); 921 return (EWOULDBLOCK); 922 } 923 so = TAILQ_FIRST(&head->sol_comp); 924 SOCK_LOCK(so); 925 KASSERT(so->so_qstate == SQ_COMP, 926 ("%s: so %p not SQ_COMP", __func__, so)); 927 soref(so); 928 head->sol_qlen--; 929 so->so_qstate = SQ_NONE; 930 so->so_listen = NULL; 931 TAILQ_REMOVE(&head->sol_comp, so, so_list); 932 if (flags & ACCEPT4_INHERIT) 933 so->so_state |= (head->so_state & SS_NBIO); 934 else 935 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 936 SOCK_UNLOCK(so); 937 sorele(head); 938 939 *ret = so; 940 return (0); 941 } 942 943 /* 944 * Evaluate the reference count and named references on a socket; if no 945 * references remain, free it. This should be called whenever a reference is 946 * released, such as in sorele(), but also when named reference flags are 947 * cleared in socket or protocol code. 948 * 949 * sofree() will free the socket if: 950 * 951 * - There are no outstanding file descriptor references or related consumers 952 * (so_count == 0). 953 * 954 * - The socket has been closed by user space, if ever open (SS_NOFDREF). 955 * 956 * - The protocol does not have an outstanding strong reference on the socket 957 * (SS_PROTOREF). 958 * 959 * - The socket is not in a completed connection queue, so a process has been 960 * notified that it is present. If it is removed, the user process may 961 * block in accept() despite select() saying the socket was ready. 962 */ 963 void 964 sofree(struct socket *so) 965 { 966 struct protosw *pr = so->so_proto; 967 968 SOCK_LOCK_ASSERT(so); 969 970 if ((so->so_state & SS_NOFDREF) == 0 || so->so_count != 0 || 971 (so->so_state & SS_PROTOREF) || (so->so_qstate == SQ_COMP)) { 972 SOCK_UNLOCK(so); 973 return; 974 } 975 976 if (!SOLISTENING(so) && so->so_qstate == SQ_INCOMP) { 977 struct socket *sol; 978 979 sol = so->so_listen; 980 KASSERT(sol, ("%s: so %p on incomp of NULL", __func__, so)); 981 982 /* 983 * To solve race between close of a listening socket and 984 * a socket on its incomplete queue, we need to lock both. 985 * The order is first listening socket, then regular. 986 * Since we don't have SS_NOFDREF neither SS_PROTOREF, this 987 * function and the listening socket are the only pointers 988 * to so. To preserve so and sol, we reference both and then 989 * relock. 990 * After relock the socket may not move to so_comp since it 991 * doesn't have PCB already, but it may be removed from 992 * so_incomp. If that happens, we share responsiblity on 993 * freeing the socket, but soclose() has already removed 994 * it from queue. 995 */ 996 soref(sol); 997 soref(so); 998 SOCK_UNLOCK(so); 999 SOLISTEN_LOCK(sol); 1000 SOCK_LOCK(so); 1001 if (so->so_qstate == SQ_INCOMP) { 1002 KASSERT(so->so_listen == sol, 1003 ("%s: so %p migrated out of sol %p", 1004 __func__, so, sol)); 1005 TAILQ_REMOVE(&sol->sol_incomp, so, so_list); 1006 sol->sol_incqlen--; 1007 /* This is guarenteed not to be the last. */ 1008 refcount_release(&sol->so_count); 1009 so->so_qstate = SQ_NONE; 1010 so->so_listen = NULL; 1011 } else 1012 KASSERT(so->so_listen == NULL, 1013 ("%s: so %p not on (in)comp with so_listen", 1014 __func__, so)); 1015 sorele(sol); 1016 KASSERT(so->so_count == 1, 1017 ("%s: so %p count %u", __func__, so, so->so_count)); 1018 so->so_count = 0; 1019 } 1020 if (SOLISTENING(so)) 1021 so->so_error = ECONNABORTED; 1022 SOCK_UNLOCK(so); 1023 1024 VNET_SO_ASSERT(so); 1025 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1026 (*pr->pr_domain->dom_dispose)(so); 1027 if (pr->pr_usrreqs->pru_detach != NULL) 1028 (*pr->pr_usrreqs->pru_detach)(so); 1029 1030 /* 1031 * From this point on, we assume that no other references to this 1032 * socket exist anywhere else in the stack. Therefore, no locks need 1033 * to be acquired or held. 1034 * 1035 * We used to do a lot of socket buffer and socket locking here, as 1036 * well as invoke sorflush() and perform wakeups. The direct call to 1037 * dom_dispose() and sbrelease_internal() are an inlining of what was 1038 * necessary from sorflush(). 1039 * 1040 * Notice that the socket buffer and kqueue state are torn down 1041 * before calling pru_detach. This means that protocols shold not 1042 * assume they can perform socket wakeups, etc, in their detach code. 1043 */ 1044 if (!SOLISTENING(so)) { 1045 sbdestroy(&so->so_snd, so); 1046 sbdestroy(&so->so_rcv, so); 1047 } 1048 seldrain(&so->so_rdsel); 1049 seldrain(&so->so_wrsel); 1050 knlist_destroy(&so->so_rdsel.si_note); 1051 knlist_destroy(&so->so_wrsel.si_note); 1052 sodealloc(so); 1053 } 1054 1055 /* 1056 * Close a socket on last file table reference removal. Initiate disconnect 1057 * if connected. Free socket when disconnect complete. 1058 * 1059 * This function will sorele() the socket. Note that soclose() may be called 1060 * prior to the ref count reaching zero. The actual socket structure will 1061 * not be freed until the ref count reaches zero. 1062 */ 1063 int 1064 soclose(struct socket *so) 1065 { 1066 struct accept_queue lqueue; 1067 bool listening; 1068 int error = 0; 1069 1070 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 1071 1072 CURVNET_SET(so->so_vnet); 1073 funsetown(&so->so_sigio); 1074 if (so->so_state & SS_ISCONNECTED) { 1075 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1076 error = sodisconnect(so); 1077 if (error) { 1078 if (error == ENOTCONN) 1079 error = 0; 1080 goto drop; 1081 } 1082 } 1083 if (so->so_options & SO_LINGER) { 1084 if ((so->so_state & SS_ISDISCONNECTING) && 1085 (so->so_state & SS_NBIO)) 1086 goto drop; 1087 while (so->so_state & SS_ISCONNECTED) { 1088 error = tsleep(&so->so_timeo, 1089 PSOCK | PCATCH, "soclos", 1090 so->so_linger * hz); 1091 if (error) 1092 break; 1093 } 1094 } 1095 } 1096 1097 drop: 1098 if (so->so_proto->pr_usrreqs->pru_close != NULL) 1099 (*so->so_proto->pr_usrreqs->pru_close)(so); 1100 1101 SOCK_LOCK(so); 1102 if ((listening = (so->so_options & SO_ACCEPTCONN))) { 1103 struct socket *sp; 1104 1105 TAILQ_INIT(&lqueue); 1106 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 1107 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 1108 1109 so->sol_qlen = so->sol_incqlen = 0; 1110 1111 TAILQ_FOREACH(sp, &lqueue, so_list) { 1112 SOCK_LOCK(sp); 1113 sp->so_qstate = SQ_NONE; 1114 sp->so_listen = NULL; 1115 SOCK_UNLOCK(sp); 1116 /* Guaranteed not to be the last. */ 1117 refcount_release(&so->so_count); 1118 } 1119 } 1120 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 1121 so->so_state |= SS_NOFDREF; 1122 sorele(so); 1123 if (listening) { 1124 struct socket *sp; 1125 1126 TAILQ_FOREACH(sp, &lqueue, so_list) { 1127 SOCK_LOCK(sp); 1128 if (sp->so_count == 0) { 1129 SOCK_UNLOCK(sp); 1130 soabort(sp); 1131 } else 1132 /* sp is now in sofree() */ 1133 SOCK_UNLOCK(sp); 1134 } 1135 } 1136 CURVNET_RESTORE(); 1137 return (error); 1138 } 1139 1140 /* 1141 * soabort() is used to abruptly tear down a connection, such as when a 1142 * resource limit is reached (listen queue depth exceeded), or if a listen 1143 * socket is closed while there are sockets waiting to be accepted. 1144 * 1145 * This interface is tricky, because it is called on an unreferenced socket, 1146 * and must be called only by a thread that has actually removed the socket 1147 * from the listen queue it was on, or races with other threads are risked. 1148 * 1149 * This interface will call into the protocol code, so must not be called 1150 * with any socket locks held. Protocols do call it while holding their own 1151 * recursible protocol mutexes, but this is something that should be subject 1152 * to review in the future. 1153 */ 1154 void 1155 soabort(struct socket *so) 1156 { 1157 1158 /* 1159 * In as much as is possible, assert that no references to this 1160 * socket are held. This is not quite the same as asserting that the 1161 * current thread is responsible for arranging for no references, but 1162 * is as close as we can get for now. 1163 */ 1164 KASSERT(so->so_count == 0, ("soabort: so_count")); 1165 KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF")); 1166 KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF")); 1167 KASSERT(so->so_qstate == SQ_NONE, ("soabort: !SQ_NONE")); 1168 VNET_SO_ASSERT(so); 1169 1170 if (so->so_proto->pr_usrreqs->pru_abort != NULL) 1171 (*so->so_proto->pr_usrreqs->pru_abort)(so); 1172 SOCK_LOCK(so); 1173 sofree(so); 1174 } 1175 1176 int 1177 soaccept(struct socket *so, struct sockaddr **nam) 1178 { 1179 int error; 1180 1181 SOCK_LOCK(so); 1182 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 1183 so->so_state &= ~SS_NOFDREF; 1184 SOCK_UNLOCK(so); 1185 1186 CURVNET_SET(so->so_vnet); 1187 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 1188 CURVNET_RESTORE(); 1189 return (error); 1190 } 1191 1192 int 1193 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 1194 { 1195 1196 return (soconnectat(AT_FDCWD, so, nam, td)); 1197 } 1198 1199 int 1200 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1201 { 1202 int error; 1203 1204 if (so->so_options & SO_ACCEPTCONN) 1205 return (EOPNOTSUPP); 1206 1207 CURVNET_SET(so->so_vnet); 1208 /* 1209 * If protocol is connection-based, can only connect once. 1210 * Otherwise, if connected, try to disconnect first. This allows 1211 * user to disconnect by connecting to, e.g., a null address. 1212 */ 1213 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 1214 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 1215 (error = sodisconnect(so)))) { 1216 error = EISCONN; 1217 } else { 1218 /* 1219 * Prevent accumulated error from previous connection from 1220 * biting us. 1221 */ 1222 so->so_error = 0; 1223 if (fd == AT_FDCWD) { 1224 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, 1225 nam, td); 1226 } else { 1227 error = (*so->so_proto->pr_usrreqs->pru_connectat)(fd, 1228 so, nam, td); 1229 } 1230 } 1231 CURVNET_RESTORE(); 1232 1233 return (error); 1234 } 1235 1236 int 1237 soconnect2(struct socket *so1, struct socket *so2) 1238 { 1239 int error; 1240 1241 CURVNET_SET(so1->so_vnet); 1242 error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2); 1243 CURVNET_RESTORE(); 1244 return (error); 1245 } 1246 1247 int 1248 sodisconnect(struct socket *so) 1249 { 1250 int error; 1251 1252 if ((so->so_state & SS_ISCONNECTED) == 0) 1253 return (ENOTCONN); 1254 if (so->so_state & SS_ISDISCONNECTING) 1255 return (EALREADY); 1256 VNET_SO_ASSERT(so); 1257 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 1258 return (error); 1259 } 1260 1261 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) 1262 1263 int 1264 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1265 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1266 { 1267 long space; 1268 ssize_t resid; 1269 int clen = 0, error, dontroute; 1270 1271 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 1272 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 1273 ("sosend_dgram: !PR_ATOMIC")); 1274 1275 if (uio != NULL) 1276 resid = uio->uio_resid; 1277 else 1278 resid = top->m_pkthdr.len; 1279 /* 1280 * In theory resid should be unsigned. However, space must be 1281 * signed, as it might be less than 0 if we over-committed, and we 1282 * must use a signed comparison of space and resid. On the other 1283 * hand, a negative resid causes us to loop sending 0-length 1284 * segments to the protocol. 1285 */ 1286 if (resid < 0) { 1287 error = EINVAL; 1288 goto out; 1289 } 1290 1291 dontroute = 1292 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 1293 if (td != NULL) 1294 td->td_ru.ru_msgsnd++; 1295 if (control != NULL) 1296 clen = control->m_len; 1297 1298 SOCKBUF_LOCK(&so->so_snd); 1299 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1300 SOCKBUF_UNLOCK(&so->so_snd); 1301 error = EPIPE; 1302 goto out; 1303 } 1304 if (so->so_error) { 1305 error = so->so_error; 1306 so->so_error = 0; 1307 SOCKBUF_UNLOCK(&so->so_snd); 1308 goto out; 1309 } 1310 if ((so->so_state & SS_ISCONNECTED) == 0) { 1311 /* 1312 * `sendto' and `sendmsg' is allowed on a connection-based 1313 * socket if it supports implied connect. Return ENOTCONN if 1314 * not connected and no address is supplied. 1315 */ 1316 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1317 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1318 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1319 !(resid == 0 && clen != 0)) { 1320 SOCKBUF_UNLOCK(&so->so_snd); 1321 error = ENOTCONN; 1322 goto out; 1323 } 1324 } else if (addr == NULL) { 1325 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1326 error = ENOTCONN; 1327 else 1328 error = EDESTADDRREQ; 1329 SOCKBUF_UNLOCK(&so->so_snd); 1330 goto out; 1331 } 1332 } 1333 1334 /* 1335 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 1336 * problem and need fixing. 1337 */ 1338 space = sbspace(&so->so_snd); 1339 if (flags & MSG_OOB) 1340 space += 1024; 1341 space -= clen; 1342 SOCKBUF_UNLOCK(&so->so_snd); 1343 if (resid > space) { 1344 error = EMSGSIZE; 1345 goto out; 1346 } 1347 if (uio == NULL) { 1348 resid = 0; 1349 if (flags & MSG_EOR) 1350 top->m_flags |= M_EOR; 1351 } else { 1352 /* 1353 * Copy the data from userland into a mbuf chain. 1354 * If no data is to be copied in, a single empty mbuf 1355 * is returned. 1356 */ 1357 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 1358 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 1359 if (top == NULL) { 1360 error = EFAULT; /* only possible error */ 1361 goto out; 1362 } 1363 space -= resid - uio->uio_resid; 1364 resid = uio->uio_resid; 1365 } 1366 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 1367 /* 1368 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 1369 * than with. 1370 */ 1371 if (dontroute) { 1372 SOCK_LOCK(so); 1373 so->so_options |= SO_DONTROUTE; 1374 SOCK_UNLOCK(so); 1375 } 1376 /* 1377 * XXX all the SBS_CANTSENDMORE checks previously done could be out 1378 * of date. We could have received a reset packet in an interrupt or 1379 * maybe we slept while doing page faults in uiomove() etc. We could 1380 * probably recheck again inside the locking protection here, but 1381 * there are probably other places that this also happens. We must 1382 * rethink this. 1383 */ 1384 VNET_SO_ASSERT(so); 1385 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1386 (flags & MSG_OOB) ? PRUS_OOB : 1387 /* 1388 * If the user set MSG_EOF, the protocol understands this flag and 1389 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 1390 */ 1391 ((flags & MSG_EOF) && 1392 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1393 (resid <= 0)) ? 1394 PRUS_EOF : 1395 /* If there is more to send set PRUS_MORETOCOME */ 1396 (flags & MSG_MORETOCOME) || 1397 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1398 top, addr, control, td); 1399 if (dontroute) { 1400 SOCK_LOCK(so); 1401 so->so_options &= ~SO_DONTROUTE; 1402 SOCK_UNLOCK(so); 1403 } 1404 clen = 0; 1405 control = NULL; 1406 top = NULL; 1407 out: 1408 if (top != NULL) 1409 m_freem(top); 1410 if (control != NULL) 1411 m_freem(control); 1412 return (error); 1413 } 1414 1415 /* 1416 * Send on a socket. If send must go all at once and message is larger than 1417 * send buffering, then hard error. Lock against other senders. If must go 1418 * all at once and not enough room now, then inform user that this would 1419 * block and do nothing. Otherwise, if nonblocking, send as much as 1420 * possible. The data to be sent is described by "uio" if nonzero, otherwise 1421 * by the mbuf chain "top" (which must be null if uio is not). Data provided 1422 * in mbuf chain must be small enough to send all at once. 1423 * 1424 * Returns nonzero on error, timeout or signal; callers must check for short 1425 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 1426 * on return. 1427 */ 1428 int 1429 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 1430 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1431 { 1432 long space; 1433 ssize_t resid; 1434 int clen = 0, error, dontroute; 1435 int atomic = sosendallatonce(so) || top; 1436 1437 if (uio != NULL) 1438 resid = uio->uio_resid; 1439 else 1440 resid = top->m_pkthdr.len; 1441 /* 1442 * In theory resid should be unsigned. However, space must be 1443 * signed, as it might be less than 0 if we over-committed, and we 1444 * must use a signed comparison of space and resid. On the other 1445 * hand, a negative resid causes us to loop sending 0-length 1446 * segments to the protocol. 1447 * 1448 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 1449 * type sockets since that's an error. 1450 */ 1451 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 1452 error = EINVAL; 1453 goto out; 1454 } 1455 1456 dontroute = 1457 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1458 (so->so_proto->pr_flags & PR_ATOMIC); 1459 if (td != NULL) 1460 td->td_ru.ru_msgsnd++; 1461 if (control != NULL) 1462 clen = control->m_len; 1463 1464 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1465 if (error) 1466 goto out; 1467 1468 restart: 1469 do { 1470 SOCKBUF_LOCK(&so->so_snd); 1471 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1472 SOCKBUF_UNLOCK(&so->so_snd); 1473 error = EPIPE; 1474 goto release; 1475 } 1476 if (so->so_error) { 1477 error = so->so_error; 1478 so->so_error = 0; 1479 SOCKBUF_UNLOCK(&so->so_snd); 1480 goto release; 1481 } 1482 if ((so->so_state & SS_ISCONNECTED) == 0) { 1483 /* 1484 * `sendto' and `sendmsg' is allowed on a connection- 1485 * based socket if it supports implied connect. 1486 * Return ENOTCONN if not connected and no address is 1487 * supplied. 1488 */ 1489 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1490 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1491 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1492 !(resid == 0 && clen != 0)) { 1493 SOCKBUF_UNLOCK(&so->so_snd); 1494 error = ENOTCONN; 1495 goto release; 1496 } 1497 } else if (addr == NULL) { 1498 SOCKBUF_UNLOCK(&so->so_snd); 1499 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1500 error = ENOTCONN; 1501 else 1502 error = EDESTADDRREQ; 1503 goto release; 1504 } 1505 } 1506 space = sbspace(&so->so_snd); 1507 if (flags & MSG_OOB) 1508 space += 1024; 1509 if ((atomic && resid > so->so_snd.sb_hiwat) || 1510 clen > so->so_snd.sb_hiwat) { 1511 SOCKBUF_UNLOCK(&so->so_snd); 1512 error = EMSGSIZE; 1513 goto release; 1514 } 1515 if (space < resid + clen && 1516 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1517 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) { 1518 SOCKBUF_UNLOCK(&so->so_snd); 1519 error = EWOULDBLOCK; 1520 goto release; 1521 } 1522 error = sbwait(&so->so_snd); 1523 SOCKBUF_UNLOCK(&so->so_snd); 1524 if (error) 1525 goto release; 1526 goto restart; 1527 } 1528 SOCKBUF_UNLOCK(&so->so_snd); 1529 space -= clen; 1530 do { 1531 if (uio == NULL) { 1532 resid = 0; 1533 if (flags & MSG_EOR) 1534 top->m_flags |= M_EOR; 1535 } else { 1536 /* 1537 * Copy the data from userland into a mbuf 1538 * chain. If resid is 0, which can happen 1539 * only if we have control to send, then 1540 * a single empty mbuf is returned. This 1541 * is a workaround to prevent protocol send 1542 * methods to panic. 1543 */ 1544 top = m_uiotombuf(uio, M_WAITOK, space, 1545 (atomic ? max_hdr : 0), 1546 (atomic ? M_PKTHDR : 0) | 1547 ((flags & MSG_EOR) ? M_EOR : 0)); 1548 if (top == NULL) { 1549 error = EFAULT; /* only possible error */ 1550 goto release; 1551 } 1552 space -= resid - uio->uio_resid; 1553 resid = uio->uio_resid; 1554 } 1555 if (dontroute) { 1556 SOCK_LOCK(so); 1557 so->so_options |= SO_DONTROUTE; 1558 SOCK_UNLOCK(so); 1559 } 1560 /* 1561 * XXX all the SBS_CANTSENDMORE checks previously 1562 * done could be out of date. We could have received 1563 * a reset packet in an interrupt or maybe we slept 1564 * while doing page faults in uiomove() etc. We 1565 * could probably recheck again inside the locking 1566 * protection here, but there are probably other 1567 * places that this also happens. We must rethink 1568 * this. 1569 */ 1570 VNET_SO_ASSERT(so); 1571 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1572 (flags & MSG_OOB) ? PRUS_OOB : 1573 /* 1574 * If the user set MSG_EOF, the protocol understands 1575 * this flag and nothing left to send then use 1576 * PRU_SEND_EOF instead of PRU_SEND. 1577 */ 1578 ((flags & MSG_EOF) && 1579 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1580 (resid <= 0)) ? 1581 PRUS_EOF : 1582 /* If there is more to send set PRUS_MORETOCOME. */ 1583 (flags & MSG_MORETOCOME) || 1584 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1585 top, addr, control, td); 1586 if (dontroute) { 1587 SOCK_LOCK(so); 1588 so->so_options &= ~SO_DONTROUTE; 1589 SOCK_UNLOCK(so); 1590 } 1591 clen = 0; 1592 control = NULL; 1593 top = NULL; 1594 if (error) 1595 goto release; 1596 } while (resid && space > 0); 1597 } while (resid); 1598 1599 release: 1600 sbunlock(&so->so_snd); 1601 out: 1602 if (top != NULL) 1603 m_freem(top); 1604 if (control != NULL) 1605 m_freem(control); 1606 return (error); 1607 } 1608 1609 int 1610 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 1611 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1612 { 1613 int error; 1614 1615 CURVNET_SET(so->so_vnet); 1616 if (!SOLISTENING(so)) 1617 error = so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, 1618 top, control, flags, td); 1619 else { 1620 m_freem(top); 1621 m_freem(control); 1622 error = ENOTCONN; 1623 } 1624 CURVNET_RESTORE(); 1625 return (error); 1626 } 1627 1628 /* 1629 * The part of soreceive() that implements reading non-inline out-of-band 1630 * data from a socket. For more complete comments, see soreceive(), from 1631 * which this code originated. 1632 * 1633 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1634 * unable to return an mbuf chain to the caller. 1635 */ 1636 static int 1637 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 1638 { 1639 struct protosw *pr = so->so_proto; 1640 struct mbuf *m; 1641 int error; 1642 1643 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1644 VNET_SO_ASSERT(so); 1645 1646 m = m_get(M_WAITOK, MT_DATA); 1647 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1648 if (error) 1649 goto bad; 1650 do { 1651 error = uiomove(mtod(m, void *), 1652 (int) min(uio->uio_resid, m->m_len), uio); 1653 m = m_free(m); 1654 } while (uio->uio_resid && error == 0 && m); 1655 bad: 1656 if (m != NULL) 1657 m_freem(m); 1658 return (error); 1659 } 1660 1661 /* 1662 * Following replacement or removal of the first mbuf on the first mbuf chain 1663 * of a socket buffer, push necessary state changes back into the socket 1664 * buffer so that other consumers see the values consistently. 'nextrecord' 1665 * is the callers locally stored value of the original value of 1666 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1667 * NOTE: 'nextrecord' may be NULL. 1668 */ 1669 static __inline void 1670 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1671 { 1672 1673 SOCKBUF_LOCK_ASSERT(sb); 1674 /* 1675 * First, update for the new value of nextrecord. If necessary, make 1676 * it the first record. 1677 */ 1678 if (sb->sb_mb != NULL) 1679 sb->sb_mb->m_nextpkt = nextrecord; 1680 else 1681 sb->sb_mb = nextrecord; 1682 1683 /* 1684 * Now update any dependent socket buffer fields to reflect the new 1685 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1686 * addition of a second clause that takes care of the case where 1687 * sb_mb has been updated, but remains the last record. 1688 */ 1689 if (sb->sb_mb == NULL) { 1690 sb->sb_mbtail = NULL; 1691 sb->sb_lastrecord = NULL; 1692 } else if (sb->sb_mb->m_nextpkt == NULL) 1693 sb->sb_lastrecord = sb->sb_mb; 1694 } 1695 1696 /* 1697 * Implement receive operations on a socket. We depend on the way that 1698 * records are added to the sockbuf by sbappend. In particular, each record 1699 * (mbufs linked through m_next) must begin with an address if the protocol 1700 * so specifies, followed by an optional mbuf or mbufs containing ancillary 1701 * data, and then zero or more mbufs of data. In order to allow parallelism 1702 * between network receive and copying to user space, as well as avoid 1703 * sleeping with a mutex held, we release the socket buffer mutex during the 1704 * user space copy. Although the sockbuf is locked, new data may still be 1705 * appended, and thus we must maintain consistency of the sockbuf during that 1706 * time. 1707 * 1708 * The caller may receive the data as a single mbuf chain by supplying an 1709 * mbuf **mp0 for use in returning the chain. The uio is then used only for 1710 * the count in uio_resid. 1711 */ 1712 int 1713 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 1714 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1715 { 1716 struct mbuf *m, **mp; 1717 int flags, error, offset; 1718 ssize_t len; 1719 struct protosw *pr = so->so_proto; 1720 struct mbuf *nextrecord; 1721 int moff, type = 0; 1722 ssize_t orig_resid = uio->uio_resid; 1723 1724 mp = mp0; 1725 if (psa != NULL) 1726 *psa = NULL; 1727 if (controlp != NULL) 1728 *controlp = NULL; 1729 if (flagsp != NULL) 1730 flags = *flagsp &~ MSG_EOR; 1731 else 1732 flags = 0; 1733 if (flags & MSG_OOB) 1734 return (soreceive_rcvoob(so, uio, flags)); 1735 if (mp != NULL) 1736 *mp = NULL; 1737 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1738 && uio->uio_resid) { 1739 VNET_SO_ASSERT(so); 1740 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1741 } 1742 1743 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1744 if (error) 1745 return (error); 1746 1747 restart: 1748 SOCKBUF_LOCK(&so->so_rcv); 1749 m = so->so_rcv.sb_mb; 1750 /* 1751 * If we have less data than requested, block awaiting more (subject 1752 * to any timeout) if: 1753 * 1. the current count is less than the low water mark, or 1754 * 2. MSG_DONTWAIT is not set 1755 */ 1756 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1757 sbavail(&so->so_rcv) < uio->uio_resid) && 1758 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 1759 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1760 KASSERT(m != NULL || !sbavail(&so->so_rcv), 1761 ("receive: m == %p sbavail == %u", 1762 m, sbavail(&so->so_rcv))); 1763 if (so->so_error) { 1764 if (m != NULL) 1765 goto dontblock; 1766 error = so->so_error; 1767 if ((flags & MSG_PEEK) == 0) 1768 so->so_error = 0; 1769 SOCKBUF_UNLOCK(&so->so_rcv); 1770 goto release; 1771 } 1772 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1773 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1774 if (m == NULL) { 1775 SOCKBUF_UNLOCK(&so->so_rcv); 1776 goto release; 1777 } else 1778 goto dontblock; 1779 } 1780 for (; m != NULL; m = m->m_next) 1781 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1782 m = so->so_rcv.sb_mb; 1783 goto dontblock; 1784 } 1785 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1786 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1787 SOCKBUF_UNLOCK(&so->so_rcv); 1788 error = ENOTCONN; 1789 goto release; 1790 } 1791 if (uio->uio_resid == 0) { 1792 SOCKBUF_UNLOCK(&so->so_rcv); 1793 goto release; 1794 } 1795 if ((so->so_state & SS_NBIO) || 1796 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1797 SOCKBUF_UNLOCK(&so->so_rcv); 1798 error = EWOULDBLOCK; 1799 goto release; 1800 } 1801 SBLASTRECORDCHK(&so->so_rcv); 1802 SBLASTMBUFCHK(&so->so_rcv); 1803 error = sbwait(&so->so_rcv); 1804 SOCKBUF_UNLOCK(&so->so_rcv); 1805 if (error) 1806 goto release; 1807 goto restart; 1808 } 1809 dontblock: 1810 /* 1811 * From this point onward, we maintain 'nextrecord' as a cache of the 1812 * pointer to the next record in the socket buffer. We must keep the 1813 * various socket buffer pointers and local stack versions of the 1814 * pointers in sync, pushing out modifications before dropping the 1815 * socket buffer mutex, and re-reading them when picking it up. 1816 * 1817 * Otherwise, we will race with the network stack appending new data 1818 * or records onto the socket buffer by using inconsistent/stale 1819 * versions of the field, possibly resulting in socket buffer 1820 * corruption. 1821 * 1822 * By holding the high-level sblock(), we prevent simultaneous 1823 * readers from pulling off the front of the socket buffer. 1824 */ 1825 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1826 if (uio->uio_td) 1827 uio->uio_td->td_ru.ru_msgrcv++; 1828 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1829 SBLASTRECORDCHK(&so->so_rcv); 1830 SBLASTMBUFCHK(&so->so_rcv); 1831 nextrecord = m->m_nextpkt; 1832 if (pr->pr_flags & PR_ADDR) { 1833 KASSERT(m->m_type == MT_SONAME, 1834 ("m->m_type == %d", m->m_type)); 1835 orig_resid = 0; 1836 if (psa != NULL) 1837 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1838 M_NOWAIT); 1839 if (flags & MSG_PEEK) { 1840 m = m->m_next; 1841 } else { 1842 sbfree(&so->so_rcv, m); 1843 so->so_rcv.sb_mb = m_free(m); 1844 m = so->so_rcv.sb_mb; 1845 sockbuf_pushsync(&so->so_rcv, nextrecord); 1846 } 1847 } 1848 1849 /* 1850 * Process one or more MT_CONTROL mbufs present before any data mbufs 1851 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1852 * just copy the data; if !MSG_PEEK, we call into the protocol to 1853 * perform externalization (or freeing if controlp == NULL). 1854 */ 1855 if (m != NULL && m->m_type == MT_CONTROL) { 1856 struct mbuf *cm = NULL, *cmn; 1857 struct mbuf **cme = &cm; 1858 1859 do { 1860 if (flags & MSG_PEEK) { 1861 if (controlp != NULL) { 1862 *controlp = m_copym(m, 0, m->m_len, 1863 M_NOWAIT); 1864 controlp = &(*controlp)->m_next; 1865 } 1866 m = m->m_next; 1867 } else { 1868 sbfree(&so->so_rcv, m); 1869 so->so_rcv.sb_mb = m->m_next; 1870 m->m_next = NULL; 1871 *cme = m; 1872 cme = &(*cme)->m_next; 1873 m = so->so_rcv.sb_mb; 1874 } 1875 } while (m != NULL && m->m_type == MT_CONTROL); 1876 if ((flags & MSG_PEEK) == 0) 1877 sockbuf_pushsync(&so->so_rcv, nextrecord); 1878 while (cm != NULL) { 1879 cmn = cm->m_next; 1880 cm->m_next = NULL; 1881 if (pr->pr_domain->dom_externalize != NULL) { 1882 SOCKBUF_UNLOCK(&so->so_rcv); 1883 VNET_SO_ASSERT(so); 1884 error = (*pr->pr_domain->dom_externalize) 1885 (cm, controlp, flags); 1886 SOCKBUF_LOCK(&so->so_rcv); 1887 } else if (controlp != NULL) 1888 *controlp = cm; 1889 else 1890 m_freem(cm); 1891 if (controlp != NULL) { 1892 orig_resid = 0; 1893 while (*controlp != NULL) 1894 controlp = &(*controlp)->m_next; 1895 } 1896 cm = cmn; 1897 } 1898 if (m != NULL) 1899 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1900 else 1901 nextrecord = so->so_rcv.sb_mb; 1902 orig_resid = 0; 1903 } 1904 if (m != NULL) { 1905 if ((flags & MSG_PEEK) == 0) { 1906 KASSERT(m->m_nextpkt == nextrecord, 1907 ("soreceive: post-control, nextrecord !sync")); 1908 if (nextrecord == NULL) { 1909 KASSERT(so->so_rcv.sb_mb == m, 1910 ("soreceive: post-control, sb_mb!=m")); 1911 KASSERT(so->so_rcv.sb_lastrecord == m, 1912 ("soreceive: post-control, lastrecord!=m")); 1913 } 1914 } 1915 type = m->m_type; 1916 if (type == MT_OOBDATA) 1917 flags |= MSG_OOB; 1918 } else { 1919 if ((flags & MSG_PEEK) == 0) { 1920 KASSERT(so->so_rcv.sb_mb == nextrecord, 1921 ("soreceive: sb_mb != nextrecord")); 1922 if (so->so_rcv.sb_mb == NULL) { 1923 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1924 ("soreceive: sb_lastercord != NULL")); 1925 } 1926 } 1927 } 1928 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1929 SBLASTRECORDCHK(&so->so_rcv); 1930 SBLASTMBUFCHK(&so->so_rcv); 1931 1932 /* 1933 * Now continue to read any data mbufs off of the head of the socket 1934 * buffer until the read request is satisfied. Note that 'type' is 1935 * used to store the type of any mbuf reads that have happened so far 1936 * such that soreceive() can stop reading if the type changes, which 1937 * causes soreceive() to return only one of regular data and inline 1938 * out-of-band data in a single socket receive operation. 1939 */ 1940 moff = 0; 1941 offset = 0; 1942 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0 1943 && error == 0) { 1944 /* 1945 * If the type of mbuf has changed since the last mbuf 1946 * examined ('type'), end the receive operation. 1947 */ 1948 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1949 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 1950 if (type != m->m_type) 1951 break; 1952 } else if (type == MT_OOBDATA) 1953 break; 1954 else 1955 KASSERT(m->m_type == MT_DATA, 1956 ("m->m_type == %d", m->m_type)); 1957 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1958 len = uio->uio_resid; 1959 if (so->so_oobmark && len > so->so_oobmark - offset) 1960 len = so->so_oobmark - offset; 1961 if (len > m->m_len - moff) 1962 len = m->m_len - moff; 1963 /* 1964 * If mp is set, just pass back the mbufs. Otherwise copy 1965 * them out via the uio, then free. Sockbuf must be 1966 * consistent here (points to current mbuf, it points to next 1967 * record) when we drop priority; we must note any additions 1968 * to the sockbuf when we block interrupts again. 1969 */ 1970 if (mp == NULL) { 1971 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1972 SBLASTRECORDCHK(&so->so_rcv); 1973 SBLASTMBUFCHK(&so->so_rcv); 1974 SOCKBUF_UNLOCK(&so->so_rcv); 1975 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1976 SOCKBUF_LOCK(&so->so_rcv); 1977 if (error) { 1978 /* 1979 * The MT_SONAME mbuf has already been removed 1980 * from the record, so it is necessary to 1981 * remove the data mbufs, if any, to preserve 1982 * the invariant in the case of PR_ADDR that 1983 * requires MT_SONAME mbufs at the head of 1984 * each record. 1985 */ 1986 if (pr->pr_flags & PR_ATOMIC && 1987 ((flags & MSG_PEEK) == 0)) 1988 (void)sbdroprecord_locked(&so->so_rcv); 1989 SOCKBUF_UNLOCK(&so->so_rcv); 1990 goto release; 1991 } 1992 } else 1993 uio->uio_resid -= len; 1994 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1995 if (len == m->m_len - moff) { 1996 if (m->m_flags & M_EOR) 1997 flags |= MSG_EOR; 1998 if (flags & MSG_PEEK) { 1999 m = m->m_next; 2000 moff = 0; 2001 } else { 2002 nextrecord = m->m_nextpkt; 2003 sbfree(&so->so_rcv, m); 2004 if (mp != NULL) { 2005 m->m_nextpkt = NULL; 2006 *mp = m; 2007 mp = &m->m_next; 2008 so->so_rcv.sb_mb = m = m->m_next; 2009 *mp = NULL; 2010 } else { 2011 so->so_rcv.sb_mb = m_free(m); 2012 m = so->so_rcv.sb_mb; 2013 } 2014 sockbuf_pushsync(&so->so_rcv, nextrecord); 2015 SBLASTRECORDCHK(&so->so_rcv); 2016 SBLASTMBUFCHK(&so->so_rcv); 2017 } 2018 } else { 2019 if (flags & MSG_PEEK) 2020 moff += len; 2021 else { 2022 if (mp != NULL) { 2023 if (flags & MSG_DONTWAIT) { 2024 *mp = m_copym(m, 0, len, 2025 M_NOWAIT); 2026 if (*mp == NULL) { 2027 /* 2028 * m_copym() couldn't 2029 * allocate an mbuf. 2030 * Adjust uio_resid back 2031 * (it was adjusted 2032 * down by len bytes, 2033 * which we didn't end 2034 * up "copying" over). 2035 */ 2036 uio->uio_resid += len; 2037 break; 2038 } 2039 } else { 2040 SOCKBUF_UNLOCK(&so->so_rcv); 2041 *mp = m_copym(m, 0, len, 2042 M_WAITOK); 2043 SOCKBUF_LOCK(&so->so_rcv); 2044 } 2045 } 2046 sbcut_locked(&so->so_rcv, len); 2047 } 2048 } 2049 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2050 if (so->so_oobmark) { 2051 if ((flags & MSG_PEEK) == 0) { 2052 so->so_oobmark -= len; 2053 if (so->so_oobmark == 0) { 2054 so->so_rcv.sb_state |= SBS_RCVATMARK; 2055 break; 2056 } 2057 } else { 2058 offset += len; 2059 if (offset == so->so_oobmark) 2060 break; 2061 } 2062 } 2063 if (flags & MSG_EOR) 2064 break; 2065 /* 2066 * If the MSG_WAITALL flag is set (for non-atomic socket), we 2067 * must not quit until "uio->uio_resid == 0" or an error 2068 * termination. If a signal/timeout occurs, return with a 2069 * short count but without error. Keep sockbuf locked 2070 * against other readers. 2071 */ 2072 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 2073 !sosendallatonce(so) && nextrecord == NULL) { 2074 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2075 if (so->so_error || 2076 so->so_rcv.sb_state & SBS_CANTRCVMORE) 2077 break; 2078 /* 2079 * Notify the protocol that some data has been 2080 * drained before blocking. 2081 */ 2082 if (pr->pr_flags & PR_WANTRCVD) { 2083 SOCKBUF_UNLOCK(&so->so_rcv); 2084 VNET_SO_ASSERT(so); 2085 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2086 SOCKBUF_LOCK(&so->so_rcv); 2087 } 2088 SBLASTRECORDCHK(&so->so_rcv); 2089 SBLASTMBUFCHK(&so->so_rcv); 2090 /* 2091 * We could receive some data while was notifying 2092 * the protocol. Skip blocking in this case. 2093 */ 2094 if (so->so_rcv.sb_mb == NULL) { 2095 error = sbwait(&so->so_rcv); 2096 if (error) { 2097 SOCKBUF_UNLOCK(&so->so_rcv); 2098 goto release; 2099 } 2100 } 2101 m = so->so_rcv.sb_mb; 2102 if (m != NULL) 2103 nextrecord = m->m_nextpkt; 2104 } 2105 } 2106 2107 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2108 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 2109 flags |= MSG_TRUNC; 2110 if ((flags & MSG_PEEK) == 0) 2111 (void) sbdroprecord_locked(&so->so_rcv); 2112 } 2113 if ((flags & MSG_PEEK) == 0) { 2114 if (m == NULL) { 2115 /* 2116 * First part is an inline SB_EMPTY_FIXUP(). Second 2117 * part makes sure sb_lastrecord is up-to-date if 2118 * there is still data in the socket buffer. 2119 */ 2120 so->so_rcv.sb_mb = nextrecord; 2121 if (so->so_rcv.sb_mb == NULL) { 2122 so->so_rcv.sb_mbtail = NULL; 2123 so->so_rcv.sb_lastrecord = NULL; 2124 } else if (nextrecord->m_nextpkt == NULL) 2125 so->so_rcv.sb_lastrecord = nextrecord; 2126 } 2127 SBLASTRECORDCHK(&so->so_rcv); 2128 SBLASTMBUFCHK(&so->so_rcv); 2129 /* 2130 * If soreceive() is being done from the socket callback, 2131 * then don't need to generate ACK to peer to update window, 2132 * since ACK will be generated on return to TCP. 2133 */ 2134 if (!(flags & MSG_SOCALLBCK) && 2135 (pr->pr_flags & PR_WANTRCVD)) { 2136 SOCKBUF_UNLOCK(&so->so_rcv); 2137 VNET_SO_ASSERT(so); 2138 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2139 SOCKBUF_LOCK(&so->so_rcv); 2140 } 2141 } 2142 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2143 if (orig_resid == uio->uio_resid && orig_resid && 2144 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 2145 SOCKBUF_UNLOCK(&so->so_rcv); 2146 goto restart; 2147 } 2148 SOCKBUF_UNLOCK(&so->so_rcv); 2149 2150 if (flagsp != NULL) 2151 *flagsp |= flags; 2152 release: 2153 sbunlock(&so->so_rcv); 2154 return (error); 2155 } 2156 2157 /* 2158 * Optimized version of soreceive() for stream (TCP) sockets. 2159 * XXXAO: (MSG_WAITALL | MSG_PEEK) isn't properly handled. 2160 */ 2161 int 2162 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 2163 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2164 { 2165 int len = 0, error = 0, flags, oresid; 2166 struct sockbuf *sb; 2167 struct mbuf *m, *n = NULL; 2168 2169 /* We only do stream sockets. */ 2170 if (so->so_type != SOCK_STREAM) 2171 return (EINVAL); 2172 if (psa != NULL) 2173 *psa = NULL; 2174 if (controlp != NULL) 2175 return (EINVAL); 2176 if (flagsp != NULL) 2177 flags = *flagsp &~ MSG_EOR; 2178 else 2179 flags = 0; 2180 if (flags & MSG_OOB) 2181 return (soreceive_rcvoob(so, uio, flags)); 2182 if (mp0 != NULL) 2183 *mp0 = NULL; 2184 2185 sb = &so->so_rcv; 2186 2187 /* Prevent other readers from entering the socket. */ 2188 error = sblock(sb, SBLOCKWAIT(flags)); 2189 if (error) 2190 goto out; 2191 SOCKBUF_LOCK(sb); 2192 2193 /* Easy one, no space to copyout anything. */ 2194 if (uio->uio_resid == 0) { 2195 error = EINVAL; 2196 goto out; 2197 } 2198 oresid = uio->uio_resid; 2199 2200 /* We will never ever get anything unless we are or were connected. */ 2201 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 2202 error = ENOTCONN; 2203 goto out; 2204 } 2205 2206 restart: 2207 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2208 2209 /* Abort if socket has reported problems. */ 2210 if (so->so_error) { 2211 if (sbavail(sb) > 0) 2212 goto deliver; 2213 if (oresid > uio->uio_resid) 2214 goto out; 2215 error = so->so_error; 2216 if (!(flags & MSG_PEEK)) 2217 so->so_error = 0; 2218 goto out; 2219 } 2220 2221 /* Door is closed. Deliver what is left, if any. */ 2222 if (sb->sb_state & SBS_CANTRCVMORE) { 2223 if (sbavail(sb) > 0) 2224 goto deliver; 2225 else 2226 goto out; 2227 } 2228 2229 /* Socket buffer is empty and we shall not block. */ 2230 if (sbavail(sb) == 0 && 2231 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 2232 error = EAGAIN; 2233 goto out; 2234 } 2235 2236 /* Socket buffer got some data that we shall deliver now. */ 2237 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 2238 ((so->so_state & SS_NBIO) || 2239 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 2240 sbavail(sb) >= sb->sb_lowat || 2241 sbavail(sb) >= uio->uio_resid || 2242 sbavail(sb) >= sb->sb_hiwat) ) { 2243 goto deliver; 2244 } 2245 2246 /* On MSG_WAITALL we must wait until all data or error arrives. */ 2247 if ((flags & MSG_WAITALL) && 2248 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 2249 goto deliver; 2250 2251 /* 2252 * Wait and block until (more) data comes in. 2253 * NB: Drops the sockbuf lock during wait. 2254 */ 2255 error = sbwait(sb); 2256 if (error) 2257 goto out; 2258 goto restart; 2259 2260 deliver: 2261 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2262 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 2263 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 2264 2265 /* Statistics. */ 2266 if (uio->uio_td) 2267 uio->uio_td->td_ru.ru_msgrcv++; 2268 2269 /* Fill uio until full or current end of socket buffer is reached. */ 2270 len = min(uio->uio_resid, sbavail(sb)); 2271 if (mp0 != NULL) { 2272 /* Dequeue as many mbufs as possible. */ 2273 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 2274 if (*mp0 == NULL) 2275 *mp0 = sb->sb_mb; 2276 else 2277 m_cat(*mp0, sb->sb_mb); 2278 for (m = sb->sb_mb; 2279 m != NULL && m->m_len <= len; 2280 m = m->m_next) { 2281 KASSERT(!(m->m_flags & M_NOTAVAIL), 2282 ("%s: m %p not available", __func__, m)); 2283 len -= m->m_len; 2284 uio->uio_resid -= m->m_len; 2285 sbfree(sb, m); 2286 n = m; 2287 } 2288 n->m_next = NULL; 2289 sb->sb_mb = m; 2290 sb->sb_lastrecord = sb->sb_mb; 2291 if (sb->sb_mb == NULL) 2292 SB_EMPTY_FIXUP(sb); 2293 } 2294 /* Copy the remainder. */ 2295 if (len > 0) { 2296 KASSERT(sb->sb_mb != NULL, 2297 ("%s: len > 0 && sb->sb_mb empty", __func__)); 2298 2299 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 2300 if (m == NULL) 2301 len = 0; /* Don't flush data from sockbuf. */ 2302 else 2303 uio->uio_resid -= len; 2304 if (*mp0 != NULL) 2305 m_cat(*mp0, m); 2306 else 2307 *mp0 = m; 2308 if (*mp0 == NULL) { 2309 error = ENOBUFS; 2310 goto out; 2311 } 2312 } 2313 } else { 2314 /* NB: Must unlock socket buffer as uiomove may sleep. */ 2315 SOCKBUF_UNLOCK(sb); 2316 error = m_mbuftouio(uio, sb->sb_mb, len); 2317 SOCKBUF_LOCK(sb); 2318 if (error) 2319 goto out; 2320 } 2321 SBLASTRECORDCHK(sb); 2322 SBLASTMBUFCHK(sb); 2323 2324 /* 2325 * Remove the delivered data from the socket buffer unless we 2326 * were only peeking. 2327 */ 2328 if (!(flags & MSG_PEEK)) { 2329 if (len > 0) 2330 sbdrop_locked(sb, len); 2331 2332 /* Notify protocol that we drained some data. */ 2333 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 2334 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 2335 !(flags & MSG_SOCALLBCK))) { 2336 SOCKBUF_UNLOCK(sb); 2337 VNET_SO_ASSERT(so); 2338 (*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags); 2339 SOCKBUF_LOCK(sb); 2340 } 2341 } 2342 2343 /* 2344 * For MSG_WAITALL we may have to loop again and wait for 2345 * more data to come in. 2346 */ 2347 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 2348 goto restart; 2349 out: 2350 SOCKBUF_LOCK_ASSERT(sb); 2351 SBLASTRECORDCHK(sb); 2352 SBLASTMBUFCHK(sb); 2353 SOCKBUF_UNLOCK(sb); 2354 sbunlock(sb); 2355 return (error); 2356 } 2357 2358 /* 2359 * Optimized version of soreceive() for simple datagram cases from userspace. 2360 * Unlike in the stream case, we're able to drop a datagram if copyout() 2361 * fails, and because we handle datagrams atomically, we don't need to use a 2362 * sleep lock to prevent I/O interlacing. 2363 */ 2364 int 2365 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2366 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2367 { 2368 struct mbuf *m, *m2; 2369 int flags, error; 2370 ssize_t len; 2371 struct protosw *pr = so->so_proto; 2372 struct mbuf *nextrecord; 2373 2374 if (psa != NULL) 2375 *psa = NULL; 2376 if (controlp != NULL) 2377 *controlp = NULL; 2378 if (flagsp != NULL) 2379 flags = *flagsp &~ MSG_EOR; 2380 else 2381 flags = 0; 2382 2383 /* 2384 * For any complicated cases, fall back to the full 2385 * soreceive_generic(). 2386 */ 2387 if (mp0 != NULL || (flags & MSG_PEEK) || (flags & MSG_OOB)) 2388 return (soreceive_generic(so, psa, uio, mp0, controlp, 2389 flagsp)); 2390 2391 /* 2392 * Enforce restrictions on use. 2393 */ 2394 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 2395 ("soreceive_dgram: wantrcvd")); 2396 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 2397 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 2398 ("soreceive_dgram: SBS_RCVATMARK")); 2399 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 2400 ("soreceive_dgram: P_CONNREQUIRED")); 2401 2402 /* 2403 * Loop blocking while waiting for a datagram. 2404 */ 2405 SOCKBUF_LOCK(&so->so_rcv); 2406 while ((m = so->so_rcv.sb_mb) == NULL) { 2407 KASSERT(sbavail(&so->so_rcv) == 0, 2408 ("soreceive_dgram: sb_mb NULL but sbavail %u", 2409 sbavail(&so->so_rcv))); 2410 if (so->so_error) { 2411 error = so->so_error; 2412 so->so_error = 0; 2413 SOCKBUF_UNLOCK(&so->so_rcv); 2414 return (error); 2415 } 2416 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2417 uio->uio_resid == 0) { 2418 SOCKBUF_UNLOCK(&so->so_rcv); 2419 return (0); 2420 } 2421 if ((so->so_state & SS_NBIO) || 2422 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2423 SOCKBUF_UNLOCK(&so->so_rcv); 2424 return (EWOULDBLOCK); 2425 } 2426 SBLASTRECORDCHK(&so->so_rcv); 2427 SBLASTMBUFCHK(&so->so_rcv); 2428 error = sbwait(&so->so_rcv); 2429 if (error) { 2430 SOCKBUF_UNLOCK(&so->so_rcv); 2431 return (error); 2432 } 2433 } 2434 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2435 2436 if (uio->uio_td) 2437 uio->uio_td->td_ru.ru_msgrcv++; 2438 SBLASTRECORDCHK(&so->so_rcv); 2439 SBLASTMBUFCHK(&so->so_rcv); 2440 nextrecord = m->m_nextpkt; 2441 if (nextrecord == NULL) { 2442 KASSERT(so->so_rcv.sb_lastrecord == m, 2443 ("soreceive_dgram: lastrecord != m")); 2444 } 2445 2446 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 2447 ("soreceive_dgram: m_nextpkt != nextrecord")); 2448 2449 /* 2450 * Pull 'm' and its chain off the front of the packet queue. 2451 */ 2452 so->so_rcv.sb_mb = NULL; 2453 sockbuf_pushsync(&so->so_rcv, nextrecord); 2454 2455 /* 2456 * Walk 'm's chain and free that many bytes from the socket buffer. 2457 */ 2458 for (m2 = m; m2 != NULL; m2 = m2->m_next) 2459 sbfree(&so->so_rcv, m2); 2460 2461 /* 2462 * Do a few last checks before we let go of the lock. 2463 */ 2464 SBLASTRECORDCHK(&so->so_rcv); 2465 SBLASTMBUFCHK(&so->so_rcv); 2466 SOCKBUF_UNLOCK(&so->so_rcv); 2467 2468 if (pr->pr_flags & PR_ADDR) { 2469 KASSERT(m->m_type == MT_SONAME, 2470 ("m->m_type == %d", m->m_type)); 2471 if (psa != NULL) 2472 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2473 M_NOWAIT); 2474 m = m_free(m); 2475 } 2476 if (m == NULL) { 2477 /* XXXRW: Can this happen? */ 2478 return (0); 2479 } 2480 2481 /* 2482 * Packet to copyout() is now in 'm' and it is disconnected from the 2483 * queue. 2484 * 2485 * Process one or more MT_CONTROL mbufs present before any data mbufs 2486 * in the first mbuf chain on the socket buffer. We call into the 2487 * protocol to perform externalization (or freeing if controlp == 2488 * NULL). In some cases there can be only MT_CONTROL mbufs without 2489 * MT_DATA mbufs. 2490 */ 2491 if (m->m_type == MT_CONTROL) { 2492 struct mbuf *cm = NULL, *cmn; 2493 struct mbuf **cme = &cm; 2494 2495 do { 2496 m2 = m->m_next; 2497 m->m_next = NULL; 2498 *cme = m; 2499 cme = &(*cme)->m_next; 2500 m = m2; 2501 } while (m != NULL && m->m_type == MT_CONTROL); 2502 while (cm != NULL) { 2503 cmn = cm->m_next; 2504 cm->m_next = NULL; 2505 if (pr->pr_domain->dom_externalize != NULL) { 2506 error = (*pr->pr_domain->dom_externalize) 2507 (cm, controlp, flags); 2508 } else if (controlp != NULL) 2509 *controlp = cm; 2510 else 2511 m_freem(cm); 2512 if (controlp != NULL) { 2513 while (*controlp != NULL) 2514 controlp = &(*controlp)->m_next; 2515 } 2516 cm = cmn; 2517 } 2518 } 2519 KASSERT(m == NULL || m->m_type == MT_DATA, 2520 ("soreceive_dgram: !data")); 2521 while (m != NULL && uio->uio_resid > 0) { 2522 len = uio->uio_resid; 2523 if (len > m->m_len) 2524 len = m->m_len; 2525 error = uiomove(mtod(m, char *), (int)len, uio); 2526 if (error) { 2527 m_freem(m); 2528 return (error); 2529 } 2530 if (len == m->m_len) 2531 m = m_free(m); 2532 else { 2533 m->m_data += len; 2534 m->m_len -= len; 2535 } 2536 } 2537 if (m != NULL) { 2538 flags |= MSG_TRUNC; 2539 m_freem(m); 2540 } 2541 if (flagsp != NULL) 2542 *flagsp |= flags; 2543 return (0); 2544 } 2545 2546 int 2547 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 2548 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2549 { 2550 int error; 2551 2552 CURVNET_SET(so->so_vnet); 2553 if (!SOLISTENING(so)) 2554 error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, 2555 mp0, controlp, flagsp)); 2556 else 2557 error = ENOTCONN; 2558 CURVNET_RESTORE(); 2559 return (error); 2560 } 2561 2562 int 2563 soshutdown(struct socket *so, int how) 2564 { 2565 struct protosw *pr = so->so_proto; 2566 int error, soerror_enotconn; 2567 2568 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 2569 return (EINVAL); 2570 2571 soerror_enotconn = 0; 2572 if ((so->so_state & 2573 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2574 /* 2575 * POSIX mandates us to return ENOTCONN when shutdown(2) is 2576 * invoked on a datagram sockets, however historically we would 2577 * actually tear socket down. This is known to be leveraged by 2578 * some applications to unblock process waiting in recvXXX(2) 2579 * by other process that it shares that socket with. Try to meet 2580 * both backward-compatibility and POSIX requirements by forcing 2581 * ENOTCONN but still asking protocol to perform pru_shutdown(). 2582 */ 2583 if (so->so_type != SOCK_DGRAM) 2584 return (ENOTCONN); 2585 soerror_enotconn = 1; 2586 } 2587 2588 CURVNET_SET(so->so_vnet); 2589 if (pr->pr_usrreqs->pru_flush != NULL) 2590 (*pr->pr_usrreqs->pru_flush)(so, how); 2591 if (how != SHUT_WR) 2592 sorflush(so); 2593 if (how != SHUT_RD) { 2594 error = (*pr->pr_usrreqs->pru_shutdown)(so); 2595 wakeup(&so->so_timeo); 2596 CURVNET_RESTORE(); 2597 return ((error == 0 && soerror_enotconn) ? ENOTCONN : error); 2598 } 2599 wakeup(&so->so_timeo); 2600 CURVNET_RESTORE(); 2601 2602 return (soerror_enotconn ? ENOTCONN : 0); 2603 } 2604 2605 void 2606 sorflush(struct socket *so) 2607 { 2608 struct sockbuf *sb = &so->so_rcv; 2609 struct protosw *pr = so->so_proto; 2610 struct socket aso; 2611 2612 VNET_SO_ASSERT(so); 2613 2614 /* 2615 * In order to avoid calling dom_dispose with the socket buffer mutex 2616 * held, and in order to generally avoid holding the lock for a long 2617 * time, we make a copy of the socket buffer and clear the original 2618 * (except locks, state). The new socket buffer copy won't have 2619 * initialized locks so we can only call routines that won't use or 2620 * assert those locks. 2621 * 2622 * Dislodge threads currently blocked in receive and wait to acquire 2623 * a lock against other simultaneous readers before clearing the 2624 * socket buffer. Don't let our acquire be interrupted by a signal 2625 * despite any existing socket disposition on interruptable waiting. 2626 */ 2627 socantrcvmore(so); 2628 (void) sblock(sb, SBL_WAIT | SBL_NOINTR); 2629 2630 /* 2631 * Invalidate/clear most of the sockbuf structure, but leave selinfo 2632 * and mutex data unchanged. 2633 */ 2634 SOCKBUF_LOCK(sb); 2635 bzero(&aso, sizeof(aso)); 2636 aso.so_pcb = so->so_pcb; 2637 bcopy(&sb->sb_startzero, &aso.so_rcv.sb_startzero, 2638 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2639 bzero(&sb->sb_startzero, 2640 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2641 SOCKBUF_UNLOCK(sb); 2642 sbunlock(sb); 2643 2644 /* 2645 * Dispose of special rights and flush the copied socket. Don't call 2646 * any unsafe routines (that rely on locks being initialized) on aso. 2647 */ 2648 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 2649 (*pr->pr_domain->dom_dispose)(&aso); 2650 sbrelease_internal(&aso.so_rcv, so); 2651 } 2652 2653 /* 2654 * Wrapper for Socket established helper hook. 2655 * Parameters: socket, context of the hook point, hook id. 2656 */ 2657 static int inline 2658 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 2659 { 2660 struct socket_hhook_data hhook_data = { 2661 .so = so, 2662 .hctx = hctx, 2663 .m = NULL, 2664 .status = 0 2665 }; 2666 2667 CURVNET_SET(so->so_vnet); 2668 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 2669 CURVNET_RESTORE(); 2670 2671 /* Ugly but needed, since hhooks return void for now */ 2672 return (hhook_data.status); 2673 } 2674 2675 /* 2676 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 2677 * additional variant to handle the case where the option value needs to be 2678 * some kind of integer, but not a specific size. In addition to their use 2679 * here, these functions are also called by the protocol-level pr_ctloutput() 2680 * routines. 2681 */ 2682 int 2683 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 2684 { 2685 size_t valsize; 2686 2687 /* 2688 * If the user gives us more than we wanted, we ignore it, but if we 2689 * don't get the minimum length the caller wants, we return EINVAL. 2690 * On success, sopt->sopt_valsize is set to however much we actually 2691 * retrieved. 2692 */ 2693 if ((valsize = sopt->sopt_valsize) < minlen) 2694 return EINVAL; 2695 if (valsize > len) 2696 sopt->sopt_valsize = valsize = len; 2697 2698 if (sopt->sopt_td != NULL) 2699 return (copyin(sopt->sopt_val, buf, valsize)); 2700 2701 bcopy(sopt->sopt_val, buf, valsize); 2702 return (0); 2703 } 2704 2705 /* 2706 * Kernel version of setsockopt(2). 2707 * 2708 * XXX: optlen is size_t, not socklen_t 2709 */ 2710 int 2711 so_setsockopt(struct socket *so, int level, int optname, void *optval, 2712 size_t optlen) 2713 { 2714 struct sockopt sopt; 2715 2716 sopt.sopt_level = level; 2717 sopt.sopt_name = optname; 2718 sopt.sopt_dir = SOPT_SET; 2719 sopt.sopt_val = optval; 2720 sopt.sopt_valsize = optlen; 2721 sopt.sopt_td = NULL; 2722 return (sosetopt(so, &sopt)); 2723 } 2724 2725 int 2726 sosetopt(struct socket *so, struct sockopt *sopt) 2727 { 2728 int error, optval; 2729 struct linger l; 2730 struct timeval tv; 2731 sbintime_t val; 2732 uint32_t val32; 2733 #ifdef MAC 2734 struct mac extmac; 2735 #endif 2736 2737 CURVNET_SET(so->so_vnet); 2738 error = 0; 2739 if (sopt->sopt_level != SOL_SOCKET) { 2740 if (so->so_proto->pr_ctloutput != NULL) { 2741 error = (*so->so_proto->pr_ctloutput)(so, sopt); 2742 CURVNET_RESTORE(); 2743 return (error); 2744 } 2745 error = ENOPROTOOPT; 2746 } else { 2747 switch (sopt->sopt_name) { 2748 case SO_ACCEPTFILTER: 2749 error = accept_filt_setopt(so, sopt); 2750 if (error) 2751 goto bad; 2752 break; 2753 2754 case SO_LINGER: 2755 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 2756 if (error) 2757 goto bad; 2758 2759 SOCK_LOCK(so); 2760 so->so_linger = l.l_linger; 2761 if (l.l_onoff) 2762 so->so_options |= SO_LINGER; 2763 else 2764 so->so_options &= ~SO_LINGER; 2765 SOCK_UNLOCK(so); 2766 break; 2767 2768 case SO_DEBUG: 2769 case SO_KEEPALIVE: 2770 case SO_DONTROUTE: 2771 case SO_USELOOPBACK: 2772 case SO_BROADCAST: 2773 case SO_REUSEADDR: 2774 case SO_REUSEPORT: 2775 case SO_OOBINLINE: 2776 case SO_TIMESTAMP: 2777 case SO_BINTIME: 2778 case SO_NOSIGPIPE: 2779 case SO_NO_DDP: 2780 case SO_NO_OFFLOAD: 2781 error = sooptcopyin(sopt, &optval, sizeof optval, 2782 sizeof optval); 2783 if (error) 2784 goto bad; 2785 SOCK_LOCK(so); 2786 if (optval) 2787 so->so_options |= sopt->sopt_name; 2788 else 2789 so->so_options &= ~sopt->sopt_name; 2790 SOCK_UNLOCK(so); 2791 break; 2792 2793 case SO_SETFIB: 2794 error = sooptcopyin(sopt, &optval, sizeof optval, 2795 sizeof optval); 2796 if (error) 2797 goto bad; 2798 2799 if (optval < 0 || optval >= rt_numfibs) { 2800 error = EINVAL; 2801 goto bad; 2802 } 2803 if (((so->so_proto->pr_domain->dom_family == PF_INET) || 2804 (so->so_proto->pr_domain->dom_family == PF_INET6) || 2805 (so->so_proto->pr_domain->dom_family == PF_ROUTE))) 2806 so->so_fibnum = optval; 2807 else 2808 so->so_fibnum = 0; 2809 break; 2810 2811 case SO_USER_COOKIE: 2812 error = sooptcopyin(sopt, &val32, sizeof val32, 2813 sizeof val32); 2814 if (error) 2815 goto bad; 2816 so->so_user_cookie = val32; 2817 break; 2818 2819 case SO_SNDBUF: 2820 case SO_RCVBUF: 2821 case SO_SNDLOWAT: 2822 case SO_RCVLOWAT: 2823 error = sooptcopyin(sopt, &optval, sizeof optval, 2824 sizeof optval); 2825 if (error) 2826 goto bad; 2827 2828 /* 2829 * Values < 1 make no sense for any of these options, 2830 * so disallow them. 2831 */ 2832 if (optval < 1) { 2833 error = EINVAL; 2834 goto bad; 2835 } 2836 2837 error = sbsetopt(so, sopt->sopt_name, optval); 2838 break; 2839 2840 case SO_SNDTIMEO: 2841 case SO_RCVTIMEO: 2842 #ifdef COMPAT_FREEBSD32 2843 if (SV_CURPROC_FLAG(SV_ILP32)) { 2844 struct timeval32 tv32; 2845 2846 error = sooptcopyin(sopt, &tv32, sizeof tv32, 2847 sizeof tv32); 2848 CP(tv32, tv, tv_sec); 2849 CP(tv32, tv, tv_usec); 2850 } else 2851 #endif 2852 error = sooptcopyin(sopt, &tv, sizeof tv, 2853 sizeof tv); 2854 if (error) 2855 goto bad; 2856 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 2857 tv.tv_usec >= 1000000) { 2858 error = EDOM; 2859 goto bad; 2860 } 2861 if (tv.tv_sec > INT32_MAX) 2862 val = SBT_MAX; 2863 else 2864 val = tvtosbt(tv); 2865 switch (sopt->sopt_name) { 2866 case SO_SNDTIMEO: 2867 so->so_snd.sb_timeo = val; 2868 break; 2869 case SO_RCVTIMEO: 2870 so->so_rcv.sb_timeo = val; 2871 break; 2872 } 2873 break; 2874 2875 case SO_LABEL: 2876 #ifdef MAC 2877 error = sooptcopyin(sopt, &extmac, sizeof extmac, 2878 sizeof extmac); 2879 if (error) 2880 goto bad; 2881 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 2882 so, &extmac); 2883 #else 2884 error = EOPNOTSUPP; 2885 #endif 2886 break; 2887 2888 case SO_TS_CLOCK: 2889 error = sooptcopyin(sopt, &optval, sizeof optval, 2890 sizeof optval); 2891 if (error) 2892 goto bad; 2893 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 2894 error = EINVAL; 2895 goto bad; 2896 } 2897 so->so_ts_clock = optval; 2898 break; 2899 2900 case SO_MAX_PACING_RATE: 2901 error = sooptcopyin(sopt, &val32, sizeof(val32), 2902 sizeof(val32)); 2903 if (error) 2904 goto bad; 2905 so->so_max_pacing_rate = val32; 2906 break; 2907 2908 default: 2909 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 2910 error = hhook_run_socket(so, sopt, 2911 HHOOK_SOCKET_OPT); 2912 else 2913 error = ENOPROTOOPT; 2914 break; 2915 } 2916 if (error == 0 && so->so_proto->pr_ctloutput != NULL) 2917 (void)(*so->so_proto->pr_ctloutput)(so, sopt); 2918 } 2919 bad: 2920 CURVNET_RESTORE(); 2921 return (error); 2922 } 2923 2924 /* 2925 * Helper routine for getsockopt. 2926 */ 2927 int 2928 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 2929 { 2930 int error; 2931 size_t valsize; 2932 2933 error = 0; 2934 2935 /* 2936 * Documented get behavior is that we always return a value, possibly 2937 * truncated to fit in the user's buffer. Traditional behavior is 2938 * that we always tell the user precisely how much we copied, rather 2939 * than something useful like the total amount we had available for 2940 * her. Note that this interface is not idempotent; the entire 2941 * answer must be generated ahead of time. 2942 */ 2943 valsize = min(len, sopt->sopt_valsize); 2944 sopt->sopt_valsize = valsize; 2945 if (sopt->sopt_val != NULL) { 2946 if (sopt->sopt_td != NULL) 2947 error = copyout(buf, sopt->sopt_val, valsize); 2948 else 2949 bcopy(buf, sopt->sopt_val, valsize); 2950 } 2951 return (error); 2952 } 2953 2954 int 2955 sogetopt(struct socket *so, struct sockopt *sopt) 2956 { 2957 int error, optval; 2958 struct linger l; 2959 struct timeval tv; 2960 #ifdef MAC 2961 struct mac extmac; 2962 #endif 2963 2964 CURVNET_SET(so->so_vnet); 2965 error = 0; 2966 if (sopt->sopt_level != SOL_SOCKET) { 2967 if (so->so_proto->pr_ctloutput != NULL) 2968 error = (*so->so_proto->pr_ctloutput)(so, sopt); 2969 else 2970 error = ENOPROTOOPT; 2971 CURVNET_RESTORE(); 2972 return (error); 2973 } else { 2974 switch (sopt->sopt_name) { 2975 case SO_ACCEPTFILTER: 2976 error = accept_filt_getopt(so, sopt); 2977 break; 2978 2979 case SO_LINGER: 2980 SOCK_LOCK(so); 2981 l.l_onoff = so->so_options & SO_LINGER; 2982 l.l_linger = so->so_linger; 2983 SOCK_UNLOCK(so); 2984 error = sooptcopyout(sopt, &l, sizeof l); 2985 break; 2986 2987 case SO_USELOOPBACK: 2988 case SO_DONTROUTE: 2989 case SO_DEBUG: 2990 case SO_KEEPALIVE: 2991 case SO_REUSEADDR: 2992 case SO_REUSEPORT: 2993 case SO_BROADCAST: 2994 case SO_OOBINLINE: 2995 case SO_ACCEPTCONN: 2996 case SO_TIMESTAMP: 2997 case SO_BINTIME: 2998 case SO_NOSIGPIPE: 2999 optval = so->so_options & sopt->sopt_name; 3000 integer: 3001 error = sooptcopyout(sopt, &optval, sizeof optval); 3002 break; 3003 3004 case SO_TYPE: 3005 optval = so->so_type; 3006 goto integer; 3007 3008 case SO_PROTOCOL: 3009 optval = so->so_proto->pr_protocol; 3010 goto integer; 3011 3012 case SO_ERROR: 3013 SOCK_LOCK(so); 3014 optval = so->so_error; 3015 so->so_error = 0; 3016 SOCK_UNLOCK(so); 3017 goto integer; 3018 3019 case SO_SNDBUF: 3020 optval = so->so_snd.sb_hiwat; 3021 goto integer; 3022 3023 case SO_RCVBUF: 3024 optval = so->so_rcv.sb_hiwat; 3025 goto integer; 3026 3027 case SO_SNDLOWAT: 3028 optval = so->so_snd.sb_lowat; 3029 goto integer; 3030 3031 case SO_RCVLOWAT: 3032 optval = so->so_rcv.sb_lowat; 3033 goto integer; 3034 3035 case SO_SNDTIMEO: 3036 case SO_RCVTIMEO: 3037 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 3038 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 3039 #ifdef COMPAT_FREEBSD32 3040 if (SV_CURPROC_FLAG(SV_ILP32)) { 3041 struct timeval32 tv32; 3042 3043 CP(tv, tv32, tv_sec); 3044 CP(tv, tv32, tv_usec); 3045 error = sooptcopyout(sopt, &tv32, sizeof tv32); 3046 } else 3047 #endif 3048 error = sooptcopyout(sopt, &tv, sizeof tv); 3049 break; 3050 3051 case SO_LABEL: 3052 #ifdef MAC 3053 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3054 sizeof(extmac)); 3055 if (error) 3056 goto bad; 3057 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 3058 so, &extmac); 3059 if (error) 3060 goto bad; 3061 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3062 #else 3063 error = EOPNOTSUPP; 3064 #endif 3065 break; 3066 3067 case SO_PEERLABEL: 3068 #ifdef MAC 3069 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3070 sizeof(extmac)); 3071 if (error) 3072 goto bad; 3073 error = mac_getsockopt_peerlabel( 3074 sopt->sopt_td->td_ucred, so, &extmac); 3075 if (error) 3076 goto bad; 3077 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3078 #else 3079 error = EOPNOTSUPP; 3080 #endif 3081 break; 3082 3083 case SO_LISTENQLIMIT: 3084 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 3085 goto integer; 3086 3087 case SO_LISTENQLEN: 3088 optval = SOLISTENING(so) ? so->sol_qlen : 0; 3089 goto integer; 3090 3091 case SO_LISTENINCQLEN: 3092 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 3093 goto integer; 3094 3095 case SO_TS_CLOCK: 3096 optval = so->so_ts_clock; 3097 goto integer; 3098 3099 case SO_MAX_PACING_RATE: 3100 optval = so->so_max_pacing_rate; 3101 goto integer; 3102 3103 default: 3104 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 3105 error = hhook_run_socket(so, sopt, 3106 HHOOK_SOCKET_OPT); 3107 else 3108 error = ENOPROTOOPT; 3109 break; 3110 } 3111 } 3112 #ifdef MAC 3113 bad: 3114 #endif 3115 CURVNET_RESTORE(); 3116 return (error); 3117 } 3118 3119 int 3120 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 3121 { 3122 struct mbuf *m, *m_prev; 3123 int sopt_size = sopt->sopt_valsize; 3124 3125 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3126 if (m == NULL) 3127 return ENOBUFS; 3128 if (sopt_size > MLEN) { 3129 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 3130 if ((m->m_flags & M_EXT) == 0) { 3131 m_free(m); 3132 return ENOBUFS; 3133 } 3134 m->m_len = min(MCLBYTES, sopt_size); 3135 } else { 3136 m->m_len = min(MLEN, sopt_size); 3137 } 3138 sopt_size -= m->m_len; 3139 *mp = m; 3140 m_prev = m; 3141 3142 while (sopt_size) { 3143 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3144 if (m == NULL) { 3145 m_freem(*mp); 3146 return ENOBUFS; 3147 } 3148 if (sopt_size > MLEN) { 3149 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 3150 M_NOWAIT); 3151 if ((m->m_flags & M_EXT) == 0) { 3152 m_freem(m); 3153 m_freem(*mp); 3154 return ENOBUFS; 3155 } 3156 m->m_len = min(MCLBYTES, sopt_size); 3157 } else { 3158 m->m_len = min(MLEN, sopt_size); 3159 } 3160 sopt_size -= m->m_len; 3161 m_prev->m_next = m; 3162 m_prev = m; 3163 } 3164 return (0); 3165 } 3166 3167 int 3168 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 3169 { 3170 struct mbuf *m0 = m; 3171 3172 if (sopt->sopt_val == NULL) 3173 return (0); 3174 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3175 if (sopt->sopt_td != NULL) { 3176 int error; 3177 3178 error = copyin(sopt->sopt_val, mtod(m, char *), 3179 m->m_len); 3180 if (error != 0) { 3181 m_freem(m0); 3182 return(error); 3183 } 3184 } else 3185 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 3186 sopt->sopt_valsize -= m->m_len; 3187 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3188 m = m->m_next; 3189 } 3190 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 3191 panic("ip6_sooptmcopyin"); 3192 return (0); 3193 } 3194 3195 int 3196 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 3197 { 3198 struct mbuf *m0 = m; 3199 size_t valsize = 0; 3200 3201 if (sopt->sopt_val == NULL) 3202 return (0); 3203 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3204 if (sopt->sopt_td != NULL) { 3205 int error; 3206 3207 error = copyout(mtod(m, char *), sopt->sopt_val, 3208 m->m_len); 3209 if (error != 0) { 3210 m_freem(m0); 3211 return(error); 3212 } 3213 } else 3214 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 3215 sopt->sopt_valsize -= m->m_len; 3216 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3217 valsize += m->m_len; 3218 m = m->m_next; 3219 } 3220 if (m != NULL) { 3221 /* enough soopt buffer should be given from user-land */ 3222 m_freem(m0); 3223 return(EINVAL); 3224 } 3225 sopt->sopt_valsize = valsize; 3226 return (0); 3227 } 3228 3229 /* 3230 * sohasoutofband(): protocol notifies socket layer of the arrival of new 3231 * out-of-band data, which will then notify socket consumers. 3232 */ 3233 void 3234 sohasoutofband(struct socket *so) 3235 { 3236 3237 if (so->so_sigio != NULL) 3238 pgsigio(&so->so_sigio, SIGURG, 0); 3239 selwakeuppri(&so->so_rdsel, PSOCK); 3240 } 3241 3242 int 3243 sopoll(struct socket *so, int events, struct ucred *active_cred, 3244 struct thread *td) 3245 { 3246 3247 /* 3248 * We do not need to set or assert curvnet as long as everyone uses 3249 * sopoll_generic(). 3250 */ 3251 return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred, 3252 td)); 3253 } 3254 3255 int 3256 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, 3257 struct thread *td) 3258 { 3259 int revents; 3260 3261 SOCK_LOCK(so); 3262 if (SOLISTENING(so)) { 3263 if (!(events & (POLLIN | POLLRDNORM))) 3264 revents = 0; 3265 else if (!TAILQ_EMPTY(&so->sol_comp)) 3266 revents = events & (POLLIN | POLLRDNORM); 3267 else { 3268 selrecord(td, &so->so_rdsel); 3269 revents = 0; 3270 } 3271 } else { 3272 revents = 0; 3273 SOCKBUF_LOCK(&so->so_snd); 3274 SOCKBUF_LOCK(&so->so_rcv); 3275 if (events & (POLLIN | POLLRDNORM)) 3276 if (soreadabledata(so)) 3277 revents |= events & (POLLIN | POLLRDNORM); 3278 if (events & (POLLOUT | POLLWRNORM)) 3279 if (sowriteable(so)) 3280 revents |= events & (POLLOUT | POLLWRNORM); 3281 if (events & (POLLPRI | POLLRDBAND)) 3282 if (so->so_oobmark || 3283 (so->so_rcv.sb_state & SBS_RCVATMARK)) 3284 revents |= events & (POLLPRI | POLLRDBAND); 3285 if ((events & POLLINIGNEOF) == 0) { 3286 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3287 revents |= events & (POLLIN | POLLRDNORM); 3288 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 3289 revents |= POLLHUP; 3290 } 3291 } 3292 if (revents == 0) { 3293 if (events & 3294 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) { 3295 selrecord(td, &so->so_rdsel); 3296 so->so_rcv.sb_flags |= SB_SEL; 3297 } 3298 if (events & (POLLOUT | POLLWRNORM)) { 3299 selrecord(td, &so->so_wrsel); 3300 so->so_snd.sb_flags |= SB_SEL; 3301 } 3302 } 3303 SOCKBUF_UNLOCK(&so->so_rcv); 3304 SOCKBUF_UNLOCK(&so->so_snd); 3305 } 3306 SOCK_UNLOCK(so); 3307 return (revents); 3308 } 3309 3310 int 3311 soo_kqfilter(struct file *fp, struct knote *kn) 3312 { 3313 struct socket *so = kn->kn_fp->f_data; 3314 struct sockbuf *sb; 3315 struct knlist *knl; 3316 3317 switch (kn->kn_filter) { 3318 case EVFILT_READ: 3319 kn->kn_fop = &soread_filtops; 3320 knl = &so->so_rdsel.si_note; 3321 sb = &so->so_rcv; 3322 break; 3323 case EVFILT_WRITE: 3324 kn->kn_fop = &sowrite_filtops; 3325 knl = &so->so_wrsel.si_note; 3326 sb = &so->so_snd; 3327 break; 3328 case EVFILT_EMPTY: 3329 kn->kn_fop = &soempty_filtops; 3330 knl = &so->so_wrsel.si_note; 3331 sb = &so->so_snd; 3332 break; 3333 default: 3334 return (EINVAL); 3335 } 3336 3337 SOCK_LOCK(so); 3338 if (SOLISTENING(so)) { 3339 knlist_add(knl, kn, 1); 3340 } else { 3341 SOCKBUF_LOCK(sb); 3342 knlist_add(knl, kn, 1); 3343 sb->sb_flags |= SB_KNOTE; 3344 SOCKBUF_UNLOCK(sb); 3345 } 3346 SOCK_UNLOCK(so); 3347 return (0); 3348 } 3349 3350 /* 3351 * Some routines that return EOPNOTSUPP for entry points that are not 3352 * supported by a protocol. Fill in as needed. 3353 */ 3354 int 3355 pru_accept_notsupp(struct socket *so, struct sockaddr **nam) 3356 { 3357 3358 return EOPNOTSUPP; 3359 } 3360 3361 int 3362 pru_aio_queue_notsupp(struct socket *so, struct kaiocb *job) 3363 { 3364 3365 return EOPNOTSUPP; 3366 } 3367 3368 int 3369 pru_attach_notsupp(struct socket *so, int proto, struct thread *td) 3370 { 3371 3372 return EOPNOTSUPP; 3373 } 3374 3375 int 3376 pru_bind_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3377 { 3378 3379 return EOPNOTSUPP; 3380 } 3381 3382 int 3383 pru_bindat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3384 struct thread *td) 3385 { 3386 3387 return EOPNOTSUPP; 3388 } 3389 3390 int 3391 pru_connect_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3392 { 3393 3394 return EOPNOTSUPP; 3395 } 3396 3397 int 3398 pru_connectat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3399 struct thread *td) 3400 { 3401 3402 return EOPNOTSUPP; 3403 } 3404 3405 int 3406 pru_connect2_notsupp(struct socket *so1, struct socket *so2) 3407 { 3408 3409 return EOPNOTSUPP; 3410 } 3411 3412 int 3413 pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data, 3414 struct ifnet *ifp, struct thread *td) 3415 { 3416 3417 return EOPNOTSUPP; 3418 } 3419 3420 int 3421 pru_disconnect_notsupp(struct socket *so) 3422 { 3423 3424 return EOPNOTSUPP; 3425 } 3426 3427 int 3428 pru_listen_notsupp(struct socket *so, int backlog, struct thread *td) 3429 { 3430 3431 return EOPNOTSUPP; 3432 } 3433 3434 int 3435 pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam) 3436 { 3437 3438 return EOPNOTSUPP; 3439 } 3440 3441 int 3442 pru_rcvd_notsupp(struct socket *so, int flags) 3443 { 3444 3445 return EOPNOTSUPP; 3446 } 3447 3448 int 3449 pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags) 3450 { 3451 3452 return EOPNOTSUPP; 3453 } 3454 3455 int 3456 pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, 3457 struct sockaddr *addr, struct mbuf *control, struct thread *td) 3458 { 3459 3460 return EOPNOTSUPP; 3461 } 3462 3463 int 3464 pru_ready_notsupp(struct socket *so, struct mbuf *m, int count) 3465 { 3466 3467 return (EOPNOTSUPP); 3468 } 3469 3470 /* 3471 * This isn't really a ``null'' operation, but it's the default one and 3472 * doesn't do anything destructive. 3473 */ 3474 int 3475 pru_sense_null(struct socket *so, struct stat *sb) 3476 { 3477 3478 sb->st_blksize = so->so_snd.sb_hiwat; 3479 return 0; 3480 } 3481 3482 int 3483 pru_shutdown_notsupp(struct socket *so) 3484 { 3485 3486 return EOPNOTSUPP; 3487 } 3488 3489 int 3490 pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam) 3491 { 3492 3493 return EOPNOTSUPP; 3494 } 3495 3496 int 3497 pru_sosend_notsupp(struct socket *so, struct sockaddr *addr, struct uio *uio, 3498 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 3499 { 3500 3501 return EOPNOTSUPP; 3502 } 3503 3504 int 3505 pru_soreceive_notsupp(struct socket *so, struct sockaddr **paddr, 3506 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3507 { 3508 3509 return EOPNOTSUPP; 3510 } 3511 3512 int 3513 pru_sopoll_notsupp(struct socket *so, int events, struct ucred *cred, 3514 struct thread *td) 3515 { 3516 3517 return EOPNOTSUPP; 3518 } 3519 3520 static void 3521 filt_sordetach(struct knote *kn) 3522 { 3523 struct socket *so = kn->kn_fp->f_data; 3524 3525 so_rdknl_lock(so); 3526 knlist_remove(&so->so_rdsel.si_note, kn, 1); 3527 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 3528 so->so_rcv.sb_flags &= ~SB_KNOTE; 3529 so_rdknl_unlock(so); 3530 } 3531 3532 /*ARGSUSED*/ 3533 static int 3534 filt_soread(struct knote *kn, long hint) 3535 { 3536 struct socket *so; 3537 3538 so = kn->kn_fp->f_data; 3539 3540 if (SOLISTENING(so)) { 3541 SOCK_LOCK_ASSERT(so); 3542 kn->kn_data = so->sol_qlen; 3543 return (!TAILQ_EMPTY(&so->sol_comp)); 3544 } 3545 3546 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3547 3548 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 3549 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3550 kn->kn_flags |= EV_EOF; 3551 kn->kn_fflags = so->so_error; 3552 return (1); 3553 } else if (so->so_error) /* temporary udp error */ 3554 return (1); 3555 3556 if (kn->kn_sfflags & NOTE_LOWAT) { 3557 if (kn->kn_data >= kn->kn_sdata) 3558 return (1); 3559 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 3560 return (1); 3561 3562 /* This hook returning non-zero indicates an event, not error */ 3563 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 3564 } 3565 3566 static void 3567 filt_sowdetach(struct knote *kn) 3568 { 3569 struct socket *so = kn->kn_fp->f_data; 3570 3571 so_wrknl_lock(so); 3572 knlist_remove(&so->so_wrsel.si_note, kn, 1); 3573 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 3574 so->so_snd.sb_flags &= ~SB_KNOTE; 3575 so_wrknl_unlock(so); 3576 } 3577 3578 /*ARGSUSED*/ 3579 static int 3580 filt_sowrite(struct knote *kn, long hint) 3581 { 3582 struct socket *so; 3583 3584 so = kn->kn_fp->f_data; 3585 3586 if (SOLISTENING(so)) 3587 return (0); 3588 3589 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3590 kn->kn_data = sbspace(&so->so_snd); 3591 3592 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 3593 3594 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 3595 kn->kn_flags |= EV_EOF; 3596 kn->kn_fflags = so->so_error; 3597 return (1); 3598 } else if (so->so_error) /* temporary udp error */ 3599 return (1); 3600 else if (((so->so_state & SS_ISCONNECTED) == 0) && 3601 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 3602 return (0); 3603 else if (kn->kn_sfflags & NOTE_LOWAT) 3604 return (kn->kn_data >= kn->kn_sdata); 3605 else 3606 return (kn->kn_data >= so->so_snd.sb_lowat); 3607 } 3608 3609 static int 3610 filt_soempty(struct knote *kn, long hint) 3611 { 3612 struct socket *so; 3613 3614 so = kn->kn_fp->f_data; 3615 3616 if (SOLISTENING(so)) 3617 return (1); 3618 3619 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3620 kn->kn_data = sbused(&so->so_snd); 3621 3622 if (kn->kn_data == 0) 3623 return (1); 3624 else 3625 return (0); 3626 } 3627 3628 int 3629 socheckuid(struct socket *so, uid_t uid) 3630 { 3631 3632 if (so == NULL) 3633 return (EPERM); 3634 if (so->so_cred->cr_uid != uid) 3635 return (EPERM); 3636 return (0); 3637 } 3638 3639 /* 3640 * These functions are used by protocols to notify the socket layer (and its 3641 * consumers) of state changes in the sockets driven by protocol-side events. 3642 */ 3643 3644 /* 3645 * Procedures to manipulate state flags of socket and do appropriate wakeups. 3646 * 3647 * Normal sequence from the active (originating) side is that 3648 * soisconnecting() is called during processing of connect() call, resulting 3649 * in an eventual call to soisconnected() if/when the connection is 3650 * established. When the connection is torn down soisdisconnecting() is 3651 * called during processing of disconnect() call, and soisdisconnected() is 3652 * called when the connection to the peer is totally severed. The semantics 3653 * of these routines are such that connectionless protocols can call 3654 * soisconnected() and soisdisconnected() only, bypassing the in-progress 3655 * calls when setting up a ``connection'' takes no time. 3656 * 3657 * From the passive side, a socket is created with two queues of sockets: 3658 * so_incomp for connections in progress and so_comp for connections already 3659 * made and awaiting user acceptance. As a protocol is preparing incoming 3660 * connections, it creates a socket structure queued on so_incomp by calling 3661 * sonewconn(). When the connection is established, soisconnected() is 3662 * called, and transfers the socket structure to so_comp, making it available 3663 * to accept(). 3664 * 3665 * If a socket is closed with sockets on either so_incomp or so_comp, these 3666 * sockets are dropped. 3667 * 3668 * If higher-level protocols are implemented in the kernel, the wakeups done 3669 * here will sometimes cause software-interrupt process scheduling. 3670 */ 3671 void 3672 soisconnecting(struct socket *so) 3673 { 3674 3675 SOCK_LOCK(so); 3676 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 3677 so->so_state |= SS_ISCONNECTING; 3678 SOCK_UNLOCK(so); 3679 } 3680 3681 void 3682 soisconnected(struct socket *so) 3683 { 3684 struct socket *head; 3685 int ret; 3686 3687 /* 3688 * XXXGL: this is the only place where we acquire socket locks 3689 * in reverse order: first child, then listening socket. To 3690 * avoid possible LOR, use try semantics. 3691 */ 3692 restart: 3693 SOCK_LOCK(so); 3694 if ((head = so->so_listen) != NULL && 3695 __predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 3696 SOCK_UNLOCK(so); 3697 goto restart; 3698 } 3699 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING); 3700 so->so_state |= SS_ISCONNECTED; 3701 if (head != NULL && (so->so_qstate == SQ_INCOMP)) { 3702 again: 3703 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 3704 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 3705 head->sol_incqlen--; 3706 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 3707 head->sol_qlen++; 3708 so->so_qstate = SQ_COMP; 3709 SOCK_UNLOCK(so); 3710 solisten_wakeup(head); /* unlocks */ 3711 } else { 3712 SOCKBUF_LOCK(&so->so_rcv); 3713 soupcall_set(so, SO_RCV, 3714 head->sol_accept_filter->accf_callback, 3715 head->sol_accept_filter_arg); 3716 so->so_options &= ~SO_ACCEPTFILTER; 3717 ret = head->sol_accept_filter->accf_callback(so, 3718 head->sol_accept_filter_arg, M_NOWAIT); 3719 if (ret == SU_ISCONNECTED) { 3720 soupcall_clear(so, SO_RCV); 3721 SOCKBUF_UNLOCK(&so->so_rcv); 3722 goto again; 3723 } 3724 SOCKBUF_UNLOCK(&so->so_rcv); 3725 SOCK_UNLOCK(so); 3726 SOLISTEN_UNLOCK(head); 3727 } 3728 return; 3729 } 3730 if (head != NULL) 3731 SOLISTEN_UNLOCK(head); 3732 SOCK_UNLOCK(so); 3733 wakeup(&so->so_timeo); 3734 sorwakeup(so); 3735 sowwakeup(so); 3736 } 3737 3738 void 3739 soisdisconnecting(struct socket *so) 3740 { 3741 3742 SOCK_LOCK(so); 3743 so->so_state &= ~SS_ISCONNECTING; 3744 so->so_state |= SS_ISDISCONNECTING; 3745 3746 if (!SOLISTENING(so)) { 3747 SOCKBUF_LOCK(&so->so_rcv); 3748 socantrcvmore_locked(so); 3749 SOCKBUF_LOCK(&so->so_snd); 3750 socantsendmore_locked(so); 3751 } 3752 SOCK_UNLOCK(so); 3753 wakeup(&so->so_timeo); 3754 } 3755 3756 void 3757 soisdisconnected(struct socket *so) 3758 { 3759 3760 SOCK_LOCK(so); 3761 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 3762 so->so_state |= SS_ISDISCONNECTED; 3763 3764 if (!SOLISTENING(so)) { 3765 SOCKBUF_LOCK(&so->so_rcv); 3766 socantrcvmore_locked(so); 3767 SOCKBUF_LOCK(&so->so_snd); 3768 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 3769 socantsendmore_locked(so); 3770 } 3771 SOCK_UNLOCK(so); 3772 wakeup(&so->so_timeo); 3773 } 3774 3775 /* 3776 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 3777 */ 3778 struct sockaddr * 3779 sodupsockaddr(const struct sockaddr *sa, int mflags) 3780 { 3781 struct sockaddr *sa2; 3782 3783 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 3784 if (sa2) 3785 bcopy(sa, sa2, sa->sa_len); 3786 return sa2; 3787 } 3788 3789 /* 3790 * Register per-socket buffer upcalls. 3791 */ 3792 void 3793 soupcall_set(struct socket *so, int which, so_upcall_t func, void *arg) 3794 { 3795 struct sockbuf *sb; 3796 3797 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3798 3799 switch (which) { 3800 case SO_RCV: 3801 sb = &so->so_rcv; 3802 break; 3803 case SO_SND: 3804 sb = &so->so_snd; 3805 break; 3806 default: 3807 panic("soupcall_set: bad which"); 3808 } 3809 SOCKBUF_LOCK_ASSERT(sb); 3810 sb->sb_upcall = func; 3811 sb->sb_upcallarg = arg; 3812 sb->sb_flags |= SB_UPCALL; 3813 } 3814 3815 void 3816 soupcall_clear(struct socket *so, int which) 3817 { 3818 struct sockbuf *sb; 3819 3820 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3821 3822 switch (which) { 3823 case SO_RCV: 3824 sb = &so->so_rcv; 3825 break; 3826 case SO_SND: 3827 sb = &so->so_snd; 3828 break; 3829 default: 3830 panic("soupcall_clear: bad which"); 3831 } 3832 SOCKBUF_LOCK_ASSERT(sb); 3833 KASSERT(sb->sb_upcall != NULL, 3834 ("%s: so %p no upcall to clear", __func__, so)); 3835 sb->sb_upcall = NULL; 3836 sb->sb_upcallarg = NULL; 3837 sb->sb_flags &= ~SB_UPCALL; 3838 } 3839 3840 void 3841 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 3842 { 3843 3844 SOLISTEN_LOCK_ASSERT(so); 3845 so->sol_upcall = func; 3846 so->sol_upcallarg = arg; 3847 } 3848 3849 static void 3850 so_rdknl_lock(void *arg) 3851 { 3852 struct socket *so = arg; 3853 3854 if (SOLISTENING(so)) 3855 SOCK_LOCK(so); 3856 else 3857 SOCKBUF_LOCK(&so->so_rcv); 3858 } 3859 3860 static void 3861 so_rdknl_unlock(void *arg) 3862 { 3863 struct socket *so = arg; 3864 3865 if (SOLISTENING(so)) 3866 SOCK_UNLOCK(so); 3867 else 3868 SOCKBUF_UNLOCK(&so->so_rcv); 3869 } 3870 3871 static void 3872 so_rdknl_assert_locked(void *arg) 3873 { 3874 struct socket *so = arg; 3875 3876 if (SOLISTENING(so)) 3877 SOCK_LOCK_ASSERT(so); 3878 else 3879 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3880 } 3881 3882 static void 3883 so_rdknl_assert_unlocked(void *arg) 3884 { 3885 struct socket *so = arg; 3886 3887 if (SOLISTENING(so)) 3888 SOCK_UNLOCK_ASSERT(so); 3889 else 3890 SOCKBUF_UNLOCK_ASSERT(&so->so_rcv); 3891 } 3892 3893 static void 3894 so_wrknl_lock(void *arg) 3895 { 3896 struct socket *so = arg; 3897 3898 if (SOLISTENING(so)) 3899 SOCK_LOCK(so); 3900 else 3901 SOCKBUF_LOCK(&so->so_snd); 3902 } 3903 3904 static void 3905 so_wrknl_unlock(void *arg) 3906 { 3907 struct socket *so = arg; 3908 3909 if (SOLISTENING(so)) 3910 SOCK_UNLOCK(so); 3911 else 3912 SOCKBUF_UNLOCK(&so->so_snd); 3913 } 3914 3915 static void 3916 so_wrknl_assert_locked(void *arg) 3917 { 3918 struct socket *so = arg; 3919 3920 if (SOLISTENING(so)) 3921 SOCK_LOCK_ASSERT(so); 3922 else 3923 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3924 } 3925 3926 static void 3927 so_wrknl_assert_unlocked(void *arg) 3928 { 3929 struct socket *so = arg; 3930 3931 if (SOLISTENING(so)) 3932 SOCK_UNLOCK_ASSERT(so); 3933 else 3934 SOCKBUF_UNLOCK_ASSERT(&so->so_snd); 3935 } 3936 3937 /* 3938 * Create an external-format (``xsocket'') structure using the information in 3939 * the kernel-format socket structure pointed to by so. This is done to 3940 * reduce the spew of irrelevant information over this interface, to isolate 3941 * user code from changes in the kernel structure, and potentially to provide 3942 * information-hiding if we decide that some of this information should be 3943 * hidden from users. 3944 */ 3945 void 3946 sotoxsocket(struct socket *so, struct xsocket *xso) 3947 { 3948 3949 xso->xso_len = sizeof *xso; 3950 xso->xso_so = so; 3951 xso->so_type = so->so_type; 3952 xso->so_options = so->so_options; 3953 xso->so_linger = so->so_linger; 3954 xso->so_state = so->so_state; 3955 xso->so_pcb = so->so_pcb; 3956 xso->xso_protocol = so->so_proto->pr_protocol; 3957 xso->xso_family = so->so_proto->pr_domain->dom_family; 3958 xso->so_timeo = so->so_timeo; 3959 xso->so_error = so->so_error; 3960 xso->so_uid = so->so_cred->cr_uid; 3961 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 3962 if (SOLISTENING(so)) { 3963 xso->so_qlen = so->sol_qlen; 3964 xso->so_incqlen = so->sol_incqlen; 3965 xso->so_qlimit = so->sol_qlimit; 3966 xso->so_oobmark = 0; 3967 bzero(&xso->so_snd, sizeof(xso->so_snd)); 3968 bzero(&xso->so_rcv, sizeof(xso->so_rcv)); 3969 } else { 3970 xso->so_state |= so->so_qstate; 3971 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 3972 xso->so_oobmark = so->so_oobmark; 3973 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 3974 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 3975 } 3976 } 3977 3978 struct sockbuf * 3979 so_sockbuf_rcv(struct socket *so) 3980 { 3981 3982 return (&so->so_rcv); 3983 } 3984 3985 struct sockbuf * 3986 so_sockbuf_snd(struct socket *so) 3987 { 3988 3989 return (&so->so_snd); 3990 } 3991 3992 int 3993 so_state_get(const struct socket *so) 3994 { 3995 3996 return (so->so_state); 3997 } 3998 3999 void 4000 so_state_set(struct socket *so, int val) 4001 { 4002 4003 so->so_state = val; 4004 } 4005 4006 int 4007 so_options_get(const struct socket *so) 4008 { 4009 4010 return (so->so_options); 4011 } 4012 4013 void 4014 so_options_set(struct socket *so, int val) 4015 { 4016 4017 so->so_options = val; 4018 } 4019 4020 int 4021 so_error_get(const struct socket *so) 4022 { 4023 4024 return (so->so_error); 4025 } 4026 4027 void 4028 so_error_set(struct socket *so, int val) 4029 { 4030 4031 so->so_error = val; 4032 } 4033 4034 int 4035 so_linger_get(const struct socket *so) 4036 { 4037 4038 return (so->so_linger); 4039 } 4040 4041 void 4042 so_linger_set(struct socket *so, int val) 4043 { 4044 4045 so->so_linger = val; 4046 } 4047 4048 struct protosw * 4049 so_protosw_get(const struct socket *so) 4050 { 4051 4052 return (so->so_proto); 4053 } 4054 4055 void 4056 so_protosw_set(struct socket *so, struct protosw *val) 4057 { 4058 4059 so->so_proto = val; 4060 } 4061 4062 void 4063 so_sorwakeup(struct socket *so) 4064 { 4065 4066 sorwakeup(so); 4067 } 4068 4069 void 4070 so_sowwakeup(struct socket *so) 4071 { 4072 4073 sowwakeup(so); 4074 } 4075 4076 void 4077 so_sorwakeup_locked(struct socket *so) 4078 { 4079 4080 sorwakeup_locked(so); 4081 } 4082 4083 void 4084 so_sowwakeup_locked(struct socket *so) 4085 { 4086 4087 sowwakeup_locked(so); 4088 } 4089 4090 void 4091 so_lock(struct socket *so) 4092 { 4093 4094 SOCK_LOCK(so); 4095 } 4096 4097 void 4098 so_unlock(struct socket *so) 4099 { 4100 4101 SOCK_UNLOCK(so); 4102 } 4103