1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. 4 * Copyright (c) 2004 The FreeBSD Foundation 5 * Copyright (c) 2004-2008 Robert N. M. Watson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pru_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pru_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pru_attach() has 50 * been successfully called. If pru_attach() returned an error, 51 * pru_detach() will not be called. Socket layer private. 52 * 53 * pru_abort() and pru_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pru_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 * 96 * NOTE: With regard to VNETs the general rule is that callers do not set 97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 98 * sofree() (and with that sorele(), sotryfree()), as well as sonewconn() 99 * and sorflush(), which are usually called from a pre-set VNET context. 100 * sopoll() currently does not need a VNET context to be set. 101 */ 102 103 #include <sys/cdefs.h> 104 __FBSDID("$FreeBSD$"); 105 106 #include "opt_inet.h" 107 #include "opt_inet6.h" 108 #include "opt_compat.h" 109 #include "opt_sctp.h" 110 111 #include <sys/param.h> 112 #include <sys/systm.h> 113 #include <sys/fcntl.h> 114 #include <sys/limits.h> 115 #include <sys/lock.h> 116 #include <sys/mac.h> 117 #include <sys/malloc.h> 118 #include <sys/mbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/domain.h> 121 #include <sys/file.h> /* for struct knote */ 122 #include <sys/hhook.h> 123 #include <sys/kernel.h> 124 #include <sys/khelp.h> 125 #include <sys/event.h> 126 #include <sys/eventhandler.h> 127 #include <sys/poll.h> 128 #include <sys/proc.h> 129 #include <sys/protosw.h> 130 #include <sys/socket.h> 131 #include <sys/socketvar.h> 132 #include <sys/resourcevar.h> 133 #include <net/route.h> 134 #include <sys/signalvar.h> 135 #include <sys/stat.h> 136 #include <sys/sx.h> 137 #include <sys/sysctl.h> 138 #include <sys/taskqueue.h> 139 #include <sys/uio.h> 140 #include <sys/jail.h> 141 #include <sys/syslog.h> 142 #include <netinet/in.h> 143 144 #include <net/vnet.h> 145 146 #include <security/mac/mac_framework.h> 147 148 #include <vm/uma.h> 149 150 #ifdef COMPAT_FREEBSD32 151 #include <sys/mount.h> 152 #include <sys/sysent.h> 153 #include <compat/freebsd32/freebsd32.h> 154 #endif 155 156 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 157 int flags); 158 static void so_rdknl_lock(void *); 159 static void so_rdknl_unlock(void *); 160 static void so_rdknl_assert_locked(void *); 161 static void so_rdknl_assert_unlocked(void *); 162 static void so_wrknl_lock(void *); 163 static void so_wrknl_unlock(void *); 164 static void so_wrknl_assert_locked(void *); 165 static void so_wrknl_assert_unlocked(void *); 166 167 static void filt_sordetach(struct knote *kn); 168 static int filt_soread(struct knote *kn, long hint); 169 static void filt_sowdetach(struct knote *kn); 170 static int filt_sowrite(struct knote *kn, long hint); 171 static int filt_soempty(struct knote *kn, long hint); 172 static int inline hhook_run_socket(struct socket *so, void *hctx, int32_t h_id); 173 fo_kqfilter_t soo_kqfilter; 174 175 static struct filterops soread_filtops = { 176 .f_isfd = 1, 177 .f_detach = filt_sordetach, 178 .f_event = filt_soread, 179 }; 180 static struct filterops sowrite_filtops = { 181 .f_isfd = 1, 182 .f_detach = filt_sowdetach, 183 .f_event = filt_sowrite, 184 }; 185 static struct filterops soempty_filtops = { 186 .f_isfd = 1, 187 .f_detach = filt_sowdetach, 188 .f_event = filt_soempty, 189 }; 190 191 so_gen_t so_gencnt; /* generation count for sockets */ 192 193 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 194 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 195 196 #define VNET_SO_ASSERT(so) \ 197 VNET_ASSERT(curvnet != NULL, \ 198 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 199 200 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 201 #define V_socket_hhh VNET(socket_hhh) 202 203 /* 204 * Limit on the number of connections in the listen queue waiting 205 * for accept(2). 206 * NB: The original sysctl somaxconn is still available but hidden 207 * to prevent confusion about the actual purpose of this number. 208 */ 209 static u_int somaxconn = SOMAXCONN; 210 211 static int 212 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 213 { 214 int error; 215 int val; 216 217 val = somaxconn; 218 error = sysctl_handle_int(oidp, &val, 0, req); 219 if (error || !req->newptr ) 220 return (error); 221 222 /* 223 * The purpose of the UINT_MAX / 3 limit, is so that the formula 224 * 3 * so_qlimit / 2 225 * below, will not overflow. 226 */ 227 228 if (val < 1 || val > UINT_MAX / 3) 229 return (EINVAL); 230 231 somaxconn = val; 232 return (0); 233 } 234 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, CTLTYPE_UINT | CTLFLAG_RW, 235 0, sizeof(int), sysctl_somaxconn, "I", 236 "Maximum listen socket pending connection accept queue size"); 237 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 238 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP, 239 0, sizeof(int), sysctl_somaxconn, "I", 240 "Maximum listen socket pending connection accept queue size (compat)"); 241 242 static int numopensockets; 243 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 244 &numopensockets, 0, "Number of open sockets"); 245 246 /* 247 * accept_mtx locks down per-socket fields relating to accept queues. See 248 * socketvar.h for an annotation of the protected fields of struct socket. 249 */ 250 struct mtx accept_mtx; 251 MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 252 253 /* 254 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 255 * so_gencnt field. 256 */ 257 static struct mtx so_global_mtx; 258 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 259 260 /* 261 * General IPC sysctl name space, used by sockets and a variety of other IPC 262 * types. 263 */ 264 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC"); 265 266 /* 267 * Initialize the socket subsystem and set up the socket 268 * memory allocator. 269 */ 270 static uma_zone_t socket_zone; 271 int maxsockets; 272 273 static void 274 socket_zone_change(void *tag) 275 { 276 277 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 278 } 279 280 static void 281 socket_hhook_register(int subtype) 282 { 283 284 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 285 &V_socket_hhh[subtype], 286 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 287 printf("%s: WARNING: unable to register hook\n", __func__); 288 } 289 290 static void 291 socket_hhook_deregister(int subtype) 292 { 293 294 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 295 printf("%s: WARNING: unable to deregister hook\n", __func__); 296 } 297 298 static void 299 socket_init(void *tag) 300 { 301 302 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 303 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 304 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 305 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 306 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 307 EVENTHANDLER_PRI_FIRST); 308 } 309 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 310 311 static void 312 socket_vnet_init(const void *unused __unused) 313 { 314 int i; 315 316 /* We expect a contiguous range */ 317 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 318 socket_hhook_register(i); 319 } 320 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 321 socket_vnet_init, NULL); 322 323 static void 324 socket_vnet_uninit(const void *unused __unused) 325 { 326 int i; 327 328 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 329 socket_hhook_deregister(i); 330 } 331 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 332 socket_vnet_uninit, NULL); 333 334 /* 335 * Initialise maxsockets. This SYSINIT must be run after 336 * tunable_mbinit(). 337 */ 338 static void 339 init_maxsockets(void *ignored) 340 { 341 342 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 343 maxsockets = imax(maxsockets, maxfiles); 344 } 345 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 346 347 /* 348 * Sysctl to get and set the maximum global sockets limit. Notify protocols 349 * of the change so that they can update their dependent limits as required. 350 */ 351 static int 352 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 353 { 354 int error, newmaxsockets; 355 356 newmaxsockets = maxsockets; 357 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 358 if (error == 0 && req->newptr) { 359 if (newmaxsockets > maxsockets && 360 newmaxsockets <= maxfiles) { 361 maxsockets = newmaxsockets; 362 EVENTHANDLER_INVOKE(maxsockets_change); 363 } else 364 error = EINVAL; 365 } 366 return (error); 367 } 368 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, CTLTYPE_INT|CTLFLAG_RW, 369 &maxsockets, 0, sysctl_maxsockets, "IU", 370 "Maximum number of sockets available"); 371 372 /* 373 * Socket operation routines. These routines are called by the routines in 374 * sys_socket.c or from a system process, and implement the semantics of 375 * socket operations by switching out to the protocol specific routines. 376 */ 377 378 /* 379 * Get a socket structure from our zone, and initialize it. Note that it 380 * would probably be better to allocate socket and PCB at the same time, but 381 * I'm not convinced that all the protocols can be easily modified to do 382 * this. 383 * 384 * soalloc() returns a socket with a ref count of 0. 385 */ 386 static struct socket * 387 soalloc(struct vnet *vnet) 388 { 389 struct socket *so; 390 391 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 392 if (so == NULL) 393 return (NULL); 394 #ifdef MAC 395 if (mac_socket_init(so, M_NOWAIT) != 0) { 396 uma_zfree(socket_zone, so); 397 return (NULL); 398 } 399 #endif 400 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 401 uma_zfree(socket_zone, so); 402 return (NULL); 403 } 404 405 /* 406 * The socket locking protocol allows to lock 2 sockets at a time, 407 * however, the first one must be a listening socket. WITNESS lacks 408 * a feature to change class of an existing lock, so we use DUPOK. 409 */ 410 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 411 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 412 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 413 so->so_rcv.sb_sel = &so->so_rdsel; 414 so->so_snd.sb_sel = &so->so_wrsel; 415 sx_init(&so->so_snd.sb_sx, "so_snd_sx"); 416 sx_init(&so->so_rcv.sb_sx, "so_rcv_sx"); 417 TAILQ_INIT(&so->so_snd.sb_aiojobq); 418 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 419 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 420 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 421 #ifdef VIMAGE 422 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 423 __func__, __LINE__, so)); 424 so->so_vnet = vnet; 425 #endif 426 /* We shouldn't need the so_global_mtx */ 427 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 428 /* Do we need more comprehensive error returns? */ 429 uma_zfree(socket_zone, so); 430 return (NULL); 431 } 432 mtx_lock(&so_global_mtx); 433 so->so_gencnt = ++so_gencnt; 434 ++numopensockets; 435 #ifdef VIMAGE 436 vnet->vnet_sockcnt++; 437 #endif 438 mtx_unlock(&so_global_mtx); 439 440 return (so); 441 } 442 443 /* 444 * Free the storage associated with a socket at the socket layer, tear down 445 * locks, labels, etc. All protocol state is assumed already to have been 446 * torn down (and possibly never set up) by the caller. 447 */ 448 static void 449 sodealloc(struct socket *so) 450 { 451 452 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 453 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 454 455 mtx_lock(&so_global_mtx); 456 so->so_gencnt = ++so_gencnt; 457 --numopensockets; /* Could be below, but faster here. */ 458 #ifdef VIMAGE 459 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 460 __func__, __LINE__, so)); 461 so->so_vnet->vnet_sockcnt--; 462 #endif 463 mtx_unlock(&so_global_mtx); 464 if (so->so_rcv.sb_hiwat) 465 (void)chgsbsize(so->so_cred->cr_uidinfo, 466 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 467 if (so->so_snd.sb_hiwat) 468 (void)chgsbsize(so->so_cred->cr_uidinfo, 469 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 470 #ifdef MAC 471 mac_socket_destroy(so); 472 #endif 473 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 474 475 crfree(so->so_cred); 476 khelp_destroy_osd(&so->osd); 477 if (SOLISTENING(so)) { 478 if (so->sol_accept_filter != NULL) 479 accept_filt_setopt(so, NULL); 480 } else { 481 sx_destroy(&so->so_snd.sb_sx); 482 sx_destroy(&so->so_rcv.sb_sx); 483 SOCKBUF_LOCK_DESTROY(&so->so_snd); 484 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 485 } 486 mtx_destroy(&so->so_lock); 487 uma_zfree(socket_zone, so); 488 } 489 490 /* 491 * socreate returns a socket with a ref count of 1. The socket should be 492 * closed with soclose(). 493 */ 494 int 495 socreate(int dom, struct socket **aso, int type, int proto, 496 struct ucred *cred, struct thread *td) 497 { 498 struct protosw *prp; 499 struct socket *so; 500 int error; 501 502 if (proto) 503 prp = pffindproto(dom, proto, type); 504 else 505 prp = pffindtype(dom, type); 506 507 if (prp == NULL) { 508 /* No support for domain. */ 509 if (pffinddomain(dom) == NULL) 510 return (EAFNOSUPPORT); 511 /* No support for socket type. */ 512 if (proto == 0 && type != 0) 513 return (EPROTOTYPE); 514 return (EPROTONOSUPPORT); 515 } 516 if (prp->pr_usrreqs->pru_attach == NULL || 517 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 518 return (EPROTONOSUPPORT); 519 520 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 521 return (EPROTONOSUPPORT); 522 523 if (prp->pr_type != type) 524 return (EPROTOTYPE); 525 so = soalloc(CRED_TO_VNET(cred)); 526 if (so == NULL) 527 return (ENOBUFS); 528 529 so->so_type = type; 530 so->so_cred = crhold(cred); 531 if ((prp->pr_domain->dom_family == PF_INET) || 532 (prp->pr_domain->dom_family == PF_INET6) || 533 (prp->pr_domain->dom_family == PF_ROUTE)) 534 so->so_fibnum = td->td_proc->p_fibnum; 535 else 536 so->so_fibnum = 0; 537 so->so_proto = prp; 538 #ifdef MAC 539 mac_socket_create(cred, so); 540 #endif 541 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 542 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 543 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 544 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 545 /* 546 * Auto-sizing of socket buffers is managed by the protocols and 547 * the appropriate flags must be set in the pru_attach function. 548 */ 549 CURVNET_SET(so->so_vnet); 550 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 551 CURVNET_RESTORE(); 552 if (error) { 553 sodealloc(so); 554 return (error); 555 } 556 soref(so); 557 *aso = so; 558 return (0); 559 } 560 561 #ifdef REGRESSION 562 static int regression_sonewconn_earlytest = 1; 563 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 564 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 565 #endif 566 567 /* 568 * When an attempt at a new connection is noted on a socket which accepts 569 * connections, sonewconn is called. If the connection is possible (subject 570 * to space constraints, etc.) then we allocate a new structure, properly 571 * linked into the data structure of the original socket, and return this. 572 * Connstatus may be 0, or SS_ISCONFIRMING, or SS_ISCONNECTED. 573 * 574 * Note: the ref count on the socket is 0 on return. 575 */ 576 struct socket * 577 sonewconn(struct socket *head, int connstatus) 578 { 579 static struct timeval lastover; 580 static struct timeval overinterval = { 60, 0 }; 581 static int overcount; 582 583 struct socket *so; 584 u_int over; 585 586 SOLISTEN_LOCK(head); 587 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 588 SOLISTEN_UNLOCK(head); 589 #ifdef REGRESSION 590 if (regression_sonewconn_earlytest && over) { 591 #else 592 if (over) { 593 #endif 594 overcount++; 595 596 if (ratecheck(&lastover, &overinterval)) { 597 log(LOG_DEBUG, "%s: pcb %p: Listen queue overflow: " 598 "%i already in queue awaiting acceptance " 599 "(%d occurrences)\n", 600 __func__, head->so_pcb, head->sol_qlen, overcount); 601 602 overcount = 0; 603 } 604 605 return (NULL); 606 } 607 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 608 __func__, head)); 609 so = soalloc(head->so_vnet); 610 if (so == NULL) { 611 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 612 "limit reached or out of memory\n", 613 __func__, head->so_pcb); 614 return (NULL); 615 } 616 so->so_listen = head; 617 so->so_type = head->so_type; 618 so->so_linger = head->so_linger; 619 so->so_state = head->so_state | SS_NOFDREF; 620 so->so_fibnum = head->so_fibnum; 621 so->so_proto = head->so_proto; 622 so->so_cred = crhold(head->so_cred); 623 #ifdef MAC 624 mac_socket_newconn(head, so); 625 #endif 626 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 627 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 628 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 629 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 630 VNET_SO_ASSERT(head); 631 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { 632 sodealloc(so); 633 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 634 __func__, head->so_pcb); 635 return (NULL); 636 } 637 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 638 sodealloc(so); 639 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 640 __func__, head->so_pcb); 641 return (NULL); 642 } 643 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 644 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 645 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 646 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 647 so->so_rcv.sb_flags |= head->sol_sbrcv_flags & SB_AUTOSIZE; 648 so->so_snd.sb_flags |= head->sol_sbsnd_flags & SB_AUTOSIZE; 649 650 SOLISTEN_LOCK(head); 651 if (head->sol_accept_filter != NULL) 652 connstatus = 0; 653 so->so_state |= connstatus; 654 so->so_options = head->so_options & ~SO_ACCEPTCONN; 655 soref(head); /* A socket on (in)complete queue refs head. */ 656 if (connstatus) { 657 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 658 so->so_qstate = SQ_COMP; 659 head->sol_qlen++; 660 solisten_wakeup(head); /* unlocks */ 661 } else { 662 /* 663 * Keep removing sockets from the head until there's room for 664 * us to insert on the tail. In pre-locking revisions, this 665 * was a simple if(), but as we could be racing with other 666 * threads and soabort() requires dropping locks, we must 667 * loop waiting for the condition to be true. 668 */ 669 while (head->sol_incqlen > head->sol_qlimit) { 670 struct socket *sp; 671 672 sp = TAILQ_FIRST(&head->sol_incomp); 673 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 674 head->sol_incqlen--; 675 SOCK_LOCK(sp); 676 sp->so_qstate = SQ_NONE; 677 sp->so_listen = NULL; 678 SOCK_UNLOCK(sp); 679 sorele(head); /* does SOLISTEN_UNLOCK, head stays */ 680 soabort(sp); 681 SOLISTEN_LOCK(head); 682 } 683 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 684 so->so_qstate = SQ_INCOMP; 685 head->sol_incqlen++; 686 SOLISTEN_UNLOCK(head); 687 } 688 return (so); 689 } 690 691 #ifdef SCTP 692 /* 693 * Socket part of sctp_peeloff(). Detach a new socket from an 694 * association. The new socket is returned with a reference. 695 */ 696 struct socket * 697 sopeeloff(struct socket *head) 698 { 699 struct socket *so; 700 701 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 702 __func__, __LINE__, head)); 703 so = soalloc(head->so_vnet); 704 if (so == NULL) { 705 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 706 "limit reached or out of memory\n", 707 __func__, head->so_pcb); 708 return (NULL); 709 } 710 so->so_type = head->so_type; 711 so->so_options = head->so_options; 712 so->so_linger = head->so_linger; 713 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 714 so->so_fibnum = head->so_fibnum; 715 so->so_proto = head->so_proto; 716 so->so_cred = crhold(head->so_cred); 717 #ifdef MAC 718 mac_socket_newconn(head, so); 719 #endif 720 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 721 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 722 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 723 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 724 VNET_SO_ASSERT(head); 725 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 726 sodealloc(so); 727 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 728 __func__, head->so_pcb); 729 return (NULL); 730 } 731 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 732 sodealloc(so); 733 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 734 __func__, head->so_pcb); 735 return (NULL); 736 } 737 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 738 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 739 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 740 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 741 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 742 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 743 744 soref(so); 745 746 return (so); 747 } 748 #endif /* SCTP */ 749 750 int 751 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 752 { 753 int error; 754 755 CURVNET_SET(so->so_vnet); 756 error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td); 757 CURVNET_RESTORE(); 758 return (error); 759 } 760 761 int 762 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 763 { 764 int error; 765 766 CURVNET_SET(so->so_vnet); 767 error = (*so->so_proto->pr_usrreqs->pru_bindat)(fd, so, nam, td); 768 CURVNET_RESTORE(); 769 return (error); 770 } 771 772 /* 773 * solisten() transitions a socket from a non-listening state to a listening 774 * state, but can also be used to update the listen queue depth on an 775 * existing listen socket. The protocol will call back into the sockets 776 * layer using solisten_proto_check() and solisten_proto() to check and set 777 * socket-layer listen state. Call backs are used so that the protocol can 778 * acquire both protocol and socket layer locks in whatever order is required 779 * by the protocol. 780 * 781 * Protocol implementors are advised to hold the socket lock across the 782 * socket-layer test and set to avoid races at the socket layer. 783 */ 784 int 785 solisten(struct socket *so, int backlog, struct thread *td) 786 { 787 int error; 788 789 CURVNET_SET(so->so_vnet); 790 error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td); 791 CURVNET_RESTORE(); 792 return (error); 793 } 794 795 int 796 solisten_proto_check(struct socket *so) 797 { 798 799 SOCK_LOCK_ASSERT(so); 800 801 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 802 SS_ISDISCONNECTING)) 803 return (EINVAL); 804 return (0); 805 } 806 807 void 808 solisten_proto(struct socket *so, int backlog) 809 { 810 int sbrcv_lowat, sbsnd_lowat; 811 u_int sbrcv_hiwat, sbsnd_hiwat; 812 short sbrcv_flags, sbsnd_flags; 813 sbintime_t sbrcv_timeo, sbsnd_timeo; 814 815 SOCK_LOCK_ASSERT(so); 816 817 if (SOLISTENING(so)) 818 goto listening; 819 820 /* 821 * Change this socket to listening state. 822 */ 823 sbrcv_lowat = so->so_rcv.sb_lowat; 824 sbsnd_lowat = so->so_snd.sb_lowat; 825 sbrcv_hiwat = so->so_rcv.sb_hiwat; 826 sbsnd_hiwat = so->so_snd.sb_hiwat; 827 sbrcv_flags = so->so_rcv.sb_flags; 828 sbsnd_flags = so->so_snd.sb_flags; 829 sbrcv_timeo = so->so_rcv.sb_timeo; 830 sbsnd_timeo = so->so_snd.sb_timeo; 831 832 sbdestroy(&so->so_snd, so); 833 sbdestroy(&so->so_rcv, so); 834 sx_destroy(&so->so_snd.sb_sx); 835 sx_destroy(&so->so_rcv.sb_sx); 836 SOCKBUF_LOCK_DESTROY(&so->so_snd); 837 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 838 839 #ifdef INVARIANTS 840 bzero(&so->so_rcv, 841 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 842 #endif 843 844 so->sol_sbrcv_lowat = sbrcv_lowat; 845 so->sol_sbsnd_lowat = sbsnd_lowat; 846 so->sol_sbrcv_hiwat = sbrcv_hiwat; 847 so->sol_sbsnd_hiwat = sbsnd_hiwat; 848 so->sol_sbrcv_flags = sbrcv_flags; 849 so->sol_sbsnd_flags = sbsnd_flags; 850 so->sol_sbrcv_timeo = sbrcv_timeo; 851 so->sol_sbsnd_timeo = sbsnd_timeo; 852 853 so->sol_qlen = so->sol_incqlen = 0; 854 TAILQ_INIT(&so->sol_incomp); 855 TAILQ_INIT(&so->sol_comp); 856 857 so->sol_accept_filter = NULL; 858 so->sol_accept_filter_arg = NULL; 859 so->sol_accept_filter_str = NULL; 860 861 so->so_options |= SO_ACCEPTCONN; 862 863 listening: 864 if (backlog < 0 || backlog > somaxconn) 865 backlog = somaxconn; 866 so->sol_qlimit = backlog; 867 } 868 869 /* 870 * Wakeup listeners/subsystems once we have a complete connection. 871 * Enters with lock, returns unlocked. 872 */ 873 void 874 solisten_wakeup(struct socket *sol) 875 { 876 877 if (sol->sol_upcall != NULL) 878 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 879 else { 880 selwakeuppri(&sol->so_rdsel, PSOCK); 881 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 882 } 883 SOLISTEN_UNLOCK(sol); 884 wakeup_one(&sol->sol_comp); 885 } 886 887 /* 888 * Return single connection off a listening socket queue. Main consumer of 889 * the function is kern_accept4(). Some modules, that do their own accept 890 * management also use the function. 891 * 892 * Listening socket must be locked on entry and is returned unlocked on 893 * return. 894 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 895 */ 896 int 897 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 898 { 899 struct socket *so; 900 int error; 901 902 SOLISTEN_LOCK_ASSERT(head); 903 904 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 905 head->so_error == 0) { 906 error = msleep(&head->sol_comp, &head->so_lock, PSOCK | PCATCH, 907 "accept", 0); 908 if (error != 0) { 909 SOLISTEN_UNLOCK(head); 910 return (error); 911 } 912 } 913 if (head->so_error) { 914 error = head->so_error; 915 head->so_error = 0; 916 SOLISTEN_UNLOCK(head); 917 return (error); 918 } 919 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) { 920 SOLISTEN_UNLOCK(head); 921 return (EWOULDBLOCK); 922 } 923 so = TAILQ_FIRST(&head->sol_comp); 924 SOCK_LOCK(so); 925 KASSERT(so->so_qstate == SQ_COMP, 926 ("%s: so %p not SQ_COMP", __func__, so)); 927 soref(so); 928 head->sol_qlen--; 929 so->so_qstate = SQ_NONE; 930 so->so_listen = NULL; 931 TAILQ_REMOVE(&head->sol_comp, so, so_list); 932 if (flags & ACCEPT4_INHERIT) 933 so->so_state |= (head->so_state & SS_NBIO); 934 else 935 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 936 SOCK_UNLOCK(so); 937 sorele(head); 938 939 *ret = so; 940 return (0); 941 } 942 943 /* 944 * Evaluate the reference count and named references on a socket; if no 945 * references remain, free it. This should be called whenever a reference is 946 * released, such as in sorele(), but also when named reference flags are 947 * cleared in socket or protocol code. 948 * 949 * sofree() will free the socket if: 950 * 951 * - There are no outstanding file descriptor references or related consumers 952 * (so_count == 0). 953 * 954 * - The socket has been closed by user space, if ever open (SS_NOFDREF). 955 * 956 * - The protocol does not have an outstanding strong reference on the socket 957 * (SS_PROTOREF). 958 * 959 * - The socket is not in a completed connection queue, so a process has been 960 * notified that it is present. If it is removed, the user process may 961 * block in accept() despite select() saying the socket was ready. 962 */ 963 void 964 sofree(struct socket *so) 965 { 966 struct protosw *pr = so->so_proto; 967 968 SOCK_LOCK_ASSERT(so); 969 970 if ((so->so_state & SS_NOFDREF) == 0 || so->so_count != 0 || 971 (so->so_state & SS_PROTOREF) || (so->so_qstate == SQ_COMP)) { 972 SOCK_UNLOCK(so); 973 return; 974 } 975 976 if (!SOLISTENING(so) && so->so_qstate == SQ_INCOMP) { 977 struct socket *sol; 978 979 sol = so->so_listen; 980 KASSERT(sol, ("%s: so %p on incomp of NULL", __func__, so)); 981 982 /* 983 * To solve race between close of a listening socket and 984 * a socket on its incomplete queue, we need to lock both. 985 * The order is first listening socket, then regular. 986 * Since we don't have SS_NOFDREF neither SS_PROTOREF, this 987 * function and the listening socket are the only pointers 988 * to so. To preserve so and sol, we reference both and then 989 * relock. 990 * After relock the socket may not move to so_comp since it 991 * doesn't have PCB already, but it may be removed from 992 * so_incomp. If that happens, we share responsiblity on 993 * freeing the socket, but soclose() has already removed 994 * it from queue. 995 */ 996 soref(sol); 997 soref(so); 998 SOCK_UNLOCK(so); 999 SOLISTEN_LOCK(sol); 1000 SOCK_LOCK(so); 1001 if (so->so_qstate == SQ_INCOMP) { 1002 KASSERT(so->so_listen == sol, 1003 ("%s: so %p migrated out of sol %p", 1004 __func__, so, sol)); 1005 TAILQ_REMOVE(&sol->sol_incomp, so, so_list); 1006 sol->sol_incqlen--; 1007 /* This is guarenteed not to be the last. */ 1008 refcount_release(&sol->so_count); 1009 so->so_qstate = SQ_NONE; 1010 so->so_listen = NULL; 1011 } else 1012 KASSERT(so->so_listen == NULL, 1013 ("%s: so %p not on (in)comp with so_listen", 1014 __func__, so)); 1015 sorele(sol); 1016 KASSERT(so->so_count == 1, 1017 ("%s: so %p count %u", __func__, so, so->so_count)); 1018 so->so_count = 0; 1019 } 1020 if (SOLISTENING(so)) 1021 so->so_error = ECONNABORTED; 1022 SOCK_UNLOCK(so); 1023 1024 VNET_SO_ASSERT(so); 1025 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1026 (*pr->pr_domain->dom_dispose)(so); 1027 if (pr->pr_usrreqs->pru_detach != NULL) 1028 (*pr->pr_usrreqs->pru_detach)(so); 1029 1030 /* 1031 * From this point on, we assume that no other references to this 1032 * socket exist anywhere else in the stack. Therefore, no locks need 1033 * to be acquired or held. 1034 * 1035 * We used to do a lot of socket buffer and socket locking here, as 1036 * well as invoke sorflush() and perform wakeups. The direct call to 1037 * dom_dispose() and sbrelease_internal() are an inlining of what was 1038 * necessary from sorflush(). 1039 * 1040 * Notice that the socket buffer and kqueue state are torn down 1041 * before calling pru_detach. This means that protocols shold not 1042 * assume they can perform socket wakeups, etc, in their detach code. 1043 */ 1044 if (!SOLISTENING(so)) { 1045 sbdestroy(&so->so_snd, so); 1046 sbdestroy(&so->so_rcv, so); 1047 } 1048 seldrain(&so->so_rdsel); 1049 seldrain(&so->so_wrsel); 1050 knlist_destroy(&so->so_rdsel.si_note); 1051 knlist_destroy(&so->so_wrsel.si_note); 1052 sodealloc(so); 1053 } 1054 1055 /* 1056 * Close a socket on last file table reference removal. Initiate disconnect 1057 * if connected. Free socket when disconnect complete. 1058 * 1059 * This function will sorele() the socket. Note that soclose() may be called 1060 * prior to the ref count reaching zero. The actual socket structure will 1061 * not be freed until the ref count reaches zero. 1062 */ 1063 int 1064 soclose(struct socket *so) 1065 { 1066 struct accept_queue lqueue; 1067 bool listening; 1068 int error = 0; 1069 1070 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 1071 1072 CURVNET_SET(so->so_vnet); 1073 funsetown(&so->so_sigio); 1074 if (so->so_state & SS_ISCONNECTED) { 1075 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1076 error = sodisconnect(so); 1077 if (error) { 1078 if (error == ENOTCONN) 1079 error = 0; 1080 goto drop; 1081 } 1082 } 1083 if (so->so_options & SO_LINGER) { 1084 if ((so->so_state & SS_ISDISCONNECTING) && 1085 (so->so_state & SS_NBIO)) 1086 goto drop; 1087 while (so->so_state & SS_ISCONNECTED) { 1088 error = tsleep(&so->so_timeo, 1089 PSOCK | PCATCH, "soclos", 1090 so->so_linger * hz); 1091 if (error) 1092 break; 1093 } 1094 } 1095 } 1096 1097 drop: 1098 if (so->so_proto->pr_usrreqs->pru_close != NULL) 1099 (*so->so_proto->pr_usrreqs->pru_close)(so); 1100 1101 SOCK_LOCK(so); 1102 if ((listening = (so->so_options & SO_ACCEPTCONN))) { 1103 struct socket *sp; 1104 1105 TAILQ_INIT(&lqueue); 1106 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 1107 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 1108 1109 so->sol_qlen = so->sol_incqlen = 0; 1110 1111 TAILQ_FOREACH(sp, &lqueue, so_list) { 1112 SOCK_LOCK(sp); 1113 sp->so_qstate = SQ_NONE; 1114 sp->so_listen = NULL; 1115 SOCK_UNLOCK(sp); 1116 /* Guaranteed not to be the last. */ 1117 refcount_release(&so->so_count); 1118 } 1119 } 1120 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 1121 so->so_state |= SS_NOFDREF; 1122 sorele(so); 1123 if (listening) { 1124 struct socket *sp; 1125 1126 TAILQ_FOREACH(sp, &lqueue, so_list) { 1127 SOCK_LOCK(sp); 1128 if (sp->so_count == 0) { 1129 SOCK_UNLOCK(sp); 1130 soabort(sp); 1131 } else 1132 /* sp is now in sofree() */ 1133 SOCK_UNLOCK(sp); 1134 } 1135 } 1136 CURVNET_RESTORE(); 1137 return (error); 1138 } 1139 1140 /* 1141 * soabort() is used to abruptly tear down a connection, such as when a 1142 * resource limit is reached (listen queue depth exceeded), or if a listen 1143 * socket is closed while there are sockets waiting to be accepted. 1144 * 1145 * This interface is tricky, because it is called on an unreferenced socket, 1146 * and must be called only by a thread that has actually removed the socket 1147 * from the listen queue it was on, or races with other threads are risked. 1148 * 1149 * This interface will call into the protocol code, so must not be called 1150 * with any socket locks held. Protocols do call it while holding their own 1151 * recursible protocol mutexes, but this is something that should be subject 1152 * to review in the future. 1153 */ 1154 void 1155 soabort(struct socket *so) 1156 { 1157 1158 /* 1159 * In as much as is possible, assert that no references to this 1160 * socket are held. This is not quite the same as asserting that the 1161 * current thread is responsible for arranging for no references, but 1162 * is as close as we can get for now. 1163 */ 1164 KASSERT(so->so_count == 0, ("soabort: so_count")); 1165 KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF")); 1166 KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF")); 1167 KASSERT(so->so_qstate == SQ_NONE, ("soabort: !SQ_NONE")); 1168 VNET_SO_ASSERT(so); 1169 1170 if (so->so_proto->pr_usrreqs->pru_abort != NULL) 1171 (*so->so_proto->pr_usrreqs->pru_abort)(so); 1172 SOCK_LOCK(so); 1173 sofree(so); 1174 } 1175 1176 int 1177 soaccept(struct socket *so, struct sockaddr **nam) 1178 { 1179 int error; 1180 1181 SOCK_LOCK(so); 1182 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 1183 so->so_state &= ~SS_NOFDREF; 1184 SOCK_UNLOCK(so); 1185 1186 CURVNET_SET(so->so_vnet); 1187 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 1188 CURVNET_RESTORE(); 1189 return (error); 1190 } 1191 1192 int 1193 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 1194 { 1195 1196 return (soconnectat(AT_FDCWD, so, nam, td)); 1197 } 1198 1199 int 1200 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1201 { 1202 int error; 1203 1204 if (so->so_options & SO_ACCEPTCONN) 1205 return (EOPNOTSUPP); 1206 1207 CURVNET_SET(so->so_vnet); 1208 /* 1209 * If protocol is connection-based, can only connect once. 1210 * Otherwise, if connected, try to disconnect first. This allows 1211 * user to disconnect by connecting to, e.g., a null address. 1212 */ 1213 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 1214 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 1215 (error = sodisconnect(so)))) { 1216 error = EISCONN; 1217 } else { 1218 /* 1219 * Prevent accumulated error from previous connection from 1220 * biting us. 1221 */ 1222 so->so_error = 0; 1223 if (fd == AT_FDCWD) { 1224 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, 1225 nam, td); 1226 } else { 1227 error = (*so->so_proto->pr_usrreqs->pru_connectat)(fd, 1228 so, nam, td); 1229 } 1230 } 1231 CURVNET_RESTORE(); 1232 1233 return (error); 1234 } 1235 1236 int 1237 soconnect2(struct socket *so1, struct socket *so2) 1238 { 1239 int error; 1240 1241 CURVNET_SET(so1->so_vnet); 1242 error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2); 1243 CURVNET_RESTORE(); 1244 return (error); 1245 } 1246 1247 int 1248 sodisconnect(struct socket *so) 1249 { 1250 int error; 1251 1252 if ((so->so_state & SS_ISCONNECTED) == 0) 1253 return (ENOTCONN); 1254 if (so->so_state & SS_ISDISCONNECTING) 1255 return (EALREADY); 1256 VNET_SO_ASSERT(so); 1257 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 1258 return (error); 1259 } 1260 1261 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) 1262 1263 int 1264 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1265 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1266 { 1267 long space; 1268 ssize_t resid; 1269 int clen = 0, error, dontroute; 1270 1271 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 1272 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 1273 ("sosend_dgram: !PR_ATOMIC")); 1274 1275 if (uio != NULL) 1276 resid = uio->uio_resid; 1277 else 1278 resid = top->m_pkthdr.len; 1279 /* 1280 * In theory resid should be unsigned. However, space must be 1281 * signed, as it might be less than 0 if we over-committed, and we 1282 * must use a signed comparison of space and resid. On the other 1283 * hand, a negative resid causes us to loop sending 0-length 1284 * segments to the protocol. 1285 */ 1286 if (resid < 0) { 1287 error = EINVAL; 1288 goto out; 1289 } 1290 1291 dontroute = 1292 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 1293 if (td != NULL) 1294 td->td_ru.ru_msgsnd++; 1295 if (control != NULL) 1296 clen = control->m_len; 1297 1298 SOCKBUF_LOCK(&so->so_snd); 1299 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1300 SOCKBUF_UNLOCK(&so->so_snd); 1301 error = EPIPE; 1302 goto out; 1303 } 1304 if (so->so_error) { 1305 error = so->so_error; 1306 so->so_error = 0; 1307 SOCKBUF_UNLOCK(&so->so_snd); 1308 goto out; 1309 } 1310 if ((so->so_state & SS_ISCONNECTED) == 0) { 1311 /* 1312 * `sendto' and `sendmsg' is allowed on a connection-based 1313 * socket if it supports implied connect. Return ENOTCONN if 1314 * not connected and no address is supplied. 1315 */ 1316 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1317 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1318 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1319 !(resid == 0 && clen != 0)) { 1320 SOCKBUF_UNLOCK(&so->so_snd); 1321 error = ENOTCONN; 1322 goto out; 1323 } 1324 } else if (addr == NULL) { 1325 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1326 error = ENOTCONN; 1327 else 1328 error = EDESTADDRREQ; 1329 SOCKBUF_UNLOCK(&so->so_snd); 1330 goto out; 1331 } 1332 } 1333 1334 /* 1335 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 1336 * problem and need fixing. 1337 */ 1338 space = sbspace(&so->so_snd); 1339 if (flags & MSG_OOB) 1340 space += 1024; 1341 space -= clen; 1342 SOCKBUF_UNLOCK(&so->so_snd); 1343 if (resid > space) { 1344 error = EMSGSIZE; 1345 goto out; 1346 } 1347 if (uio == NULL) { 1348 resid = 0; 1349 if (flags & MSG_EOR) 1350 top->m_flags |= M_EOR; 1351 } else { 1352 /* 1353 * Copy the data from userland into a mbuf chain. 1354 * If no data is to be copied in, a single empty mbuf 1355 * is returned. 1356 */ 1357 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 1358 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 1359 if (top == NULL) { 1360 error = EFAULT; /* only possible error */ 1361 goto out; 1362 } 1363 space -= resid - uio->uio_resid; 1364 resid = uio->uio_resid; 1365 } 1366 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 1367 /* 1368 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 1369 * than with. 1370 */ 1371 if (dontroute) { 1372 SOCK_LOCK(so); 1373 so->so_options |= SO_DONTROUTE; 1374 SOCK_UNLOCK(so); 1375 } 1376 /* 1377 * XXX all the SBS_CANTSENDMORE checks previously done could be out 1378 * of date. We could have received a reset packet in an interrupt or 1379 * maybe we slept while doing page faults in uiomove() etc. We could 1380 * probably recheck again inside the locking protection here, but 1381 * there are probably other places that this also happens. We must 1382 * rethink this. 1383 */ 1384 VNET_SO_ASSERT(so); 1385 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1386 (flags & MSG_OOB) ? PRUS_OOB : 1387 /* 1388 * If the user set MSG_EOF, the protocol understands this flag and 1389 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 1390 */ 1391 ((flags & MSG_EOF) && 1392 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1393 (resid <= 0)) ? 1394 PRUS_EOF : 1395 /* If there is more to send set PRUS_MORETOCOME */ 1396 (flags & MSG_MORETOCOME) || 1397 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1398 top, addr, control, td); 1399 if (dontroute) { 1400 SOCK_LOCK(so); 1401 so->so_options &= ~SO_DONTROUTE; 1402 SOCK_UNLOCK(so); 1403 } 1404 clen = 0; 1405 control = NULL; 1406 top = NULL; 1407 out: 1408 if (top != NULL) 1409 m_freem(top); 1410 if (control != NULL) 1411 m_freem(control); 1412 return (error); 1413 } 1414 1415 /* 1416 * Send on a socket. If send must go all at once and message is larger than 1417 * send buffering, then hard error. Lock against other senders. If must go 1418 * all at once and not enough room now, then inform user that this would 1419 * block and do nothing. Otherwise, if nonblocking, send as much as 1420 * possible. The data to be sent is described by "uio" if nonzero, otherwise 1421 * by the mbuf chain "top" (which must be null if uio is not). Data provided 1422 * in mbuf chain must be small enough to send all at once. 1423 * 1424 * Returns nonzero on error, timeout or signal; callers must check for short 1425 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 1426 * on return. 1427 */ 1428 int 1429 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 1430 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1431 { 1432 long space; 1433 ssize_t resid; 1434 int clen = 0, error, dontroute; 1435 int atomic = sosendallatonce(so) || top; 1436 1437 if (uio != NULL) 1438 resid = uio->uio_resid; 1439 else 1440 resid = top->m_pkthdr.len; 1441 /* 1442 * In theory resid should be unsigned. However, space must be 1443 * signed, as it might be less than 0 if we over-committed, and we 1444 * must use a signed comparison of space and resid. On the other 1445 * hand, a negative resid causes us to loop sending 0-length 1446 * segments to the protocol. 1447 * 1448 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 1449 * type sockets since that's an error. 1450 */ 1451 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 1452 error = EINVAL; 1453 goto out; 1454 } 1455 1456 dontroute = 1457 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1458 (so->so_proto->pr_flags & PR_ATOMIC); 1459 if (td != NULL) 1460 td->td_ru.ru_msgsnd++; 1461 if (control != NULL) 1462 clen = control->m_len; 1463 1464 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1465 if (error) 1466 goto out; 1467 1468 restart: 1469 do { 1470 SOCKBUF_LOCK(&so->so_snd); 1471 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1472 SOCKBUF_UNLOCK(&so->so_snd); 1473 error = EPIPE; 1474 goto release; 1475 } 1476 if (so->so_error) { 1477 error = so->so_error; 1478 so->so_error = 0; 1479 SOCKBUF_UNLOCK(&so->so_snd); 1480 goto release; 1481 } 1482 if ((so->so_state & SS_ISCONNECTED) == 0) { 1483 /* 1484 * `sendto' and `sendmsg' is allowed on a connection- 1485 * based socket if it supports implied connect. 1486 * Return ENOTCONN if not connected and no address is 1487 * supplied. 1488 */ 1489 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1490 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1491 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1492 !(resid == 0 && clen != 0)) { 1493 SOCKBUF_UNLOCK(&so->so_snd); 1494 error = ENOTCONN; 1495 goto release; 1496 } 1497 } else if (addr == NULL) { 1498 SOCKBUF_UNLOCK(&so->so_snd); 1499 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1500 error = ENOTCONN; 1501 else 1502 error = EDESTADDRREQ; 1503 goto release; 1504 } 1505 } 1506 space = sbspace(&so->so_snd); 1507 if (flags & MSG_OOB) 1508 space += 1024; 1509 if ((atomic && resid > so->so_snd.sb_hiwat) || 1510 clen > so->so_snd.sb_hiwat) { 1511 SOCKBUF_UNLOCK(&so->so_snd); 1512 error = EMSGSIZE; 1513 goto release; 1514 } 1515 if (space < resid + clen && 1516 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1517 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) { 1518 SOCKBUF_UNLOCK(&so->so_snd); 1519 error = EWOULDBLOCK; 1520 goto release; 1521 } 1522 error = sbwait(&so->so_snd); 1523 SOCKBUF_UNLOCK(&so->so_snd); 1524 if (error) 1525 goto release; 1526 goto restart; 1527 } 1528 SOCKBUF_UNLOCK(&so->so_snd); 1529 space -= clen; 1530 do { 1531 if (uio == NULL) { 1532 resid = 0; 1533 if (flags & MSG_EOR) 1534 top->m_flags |= M_EOR; 1535 } else { 1536 /* 1537 * Copy the data from userland into a mbuf 1538 * chain. If resid is 0, which can happen 1539 * only if we have control to send, then 1540 * a single empty mbuf is returned. This 1541 * is a workaround to prevent protocol send 1542 * methods to panic. 1543 */ 1544 top = m_uiotombuf(uio, M_WAITOK, space, 1545 (atomic ? max_hdr : 0), 1546 (atomic ? M_PKTHDR : 0) | 1547 ((flags & MSG_EOR) ? M_EOR : 0)); 1548 if (top == NULL) { 1549 error = EFAULT; /* only possible error */ 1550 goto release; 1551 } 1552 space -= resid - uio->uio_resid; 1553 resid = uio->uio_resid; 1554 } 1555 if (dontroute) { 1556 SOCK_LOCK(so); 1557 so->so_options |= SO_DONTROUTE; 1558 SOCK_UNLOCK(so); 1559 } 1560 /* 1561 * XXX all the SBS_CANTSENDMORE checks previously 1562 * done could be out of date. We could have received 1563 * a reset packet in an interrupt or maybe we slept 1564 * while doing page faults in uiomove() etc. We 1565 * could probably recheck again inside the locking 1566 * protection here, but there are probably other 1567 * places that this also happens. We must rethink 1568 * this. 1569 */ 1570 VNET_SO_ASSERT(so); 1571 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1572 (flags & MSG_OOB) ? PRUS_OOB : 1573 /* 1574 * If the user set MSG_EOF, the protocol understands 1575 * this flag and nothing left to send then use 1576 * PRU_SEND_EOF instead of PRU_SEND. 1577 */ 1578 ((flags & MSG_EOF) && 1579 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1580 (resid <= 0)) ? 1581 PRUS_EOF : 1582 /* If there is more to send set PRUS_MORETOCOME. */ 1583 (flags & MSG_MORETOCOME) || 1584 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1585 top, addr, control, td); 1586 if (dontroute) { 1587 SOCK_LOCK(so); 1588 so->so_options &= ~SO_DONTROUTE; 1589 SOCK_UNLOCK(so); 1590 } 1591 clen = 0; 1592 control = NULL; 1593 top = NULL; 1594 if (error) 1595 goto release; 1596 } while (resid && space > 0); 1597 } while (resid); 1598 1599 release: 1600 sbunlock(&so->so_snd); 1601 out: 1602 if (top != NULL) 1603 m_freem(top); 1604 if (control != NULL) 1605 m_freem(control); 1606 return (error); 1607 } 1608 1609 int 1610 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 1611 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1612 { 1613 int error; 1614 1615 CURVNET_SET(so->so_vnet); 1616 error = so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, top, 1617 control, flags, td); 1618 CURVNET_RESTORE(); 1619 return (error); 1620 } 1621 1622 /* 1623 * The part of soreceive() that implements reading non-inline out-of-band 1624 * data from a socket. For more complete comments, see soreceive(), from 1625 * which this code originated. 1626 * 1627 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1628 * unable to return an mbuf chain to the caller. 1629 */ 1630 static int 1631 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 1632 { 1633 struct protosw *pr = so->so_proto; 1634 struct mbuf *m; 1635 int error; 1636 1637 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1638 VNET_SO_ASSERT(so); 1639 1640 m = m_get(M_WAITOK, MT_DATA); 1641 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1642 if (error) 1643 goto bad; 1644 do { 1645 error = uiomove(mtod(m, void *), 1646 (int) min(uio->uio_resid, m->m_len), uio); 1647 m = m_free(m); 1648 } while (uio->uio_resid && error == 0 && m); 1649 bad: 1650 if (m != NULL) 1651 m_freem(m); 1652 return (error); 1653 } 1654 1655 /* 1656 * Following replacement or removal of the first mbuf on the first mbuf chain 1657 * of a socket buffer, push necessary state changes back into the socket 1658 * buffer so that other consumers see the values consistently. 'nextrecord' 1659 * is the callers locally stored value of the original value of 1660 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1661 * NOTE: 'nextrecord' may be NULL. 1662 */ 1663 static __inline void 1664 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1665 { 1666 1667 SOCKBUF_LOCK_ASSERT(sb); 1668 /* 1669 * First, update for the new value of nextrecord. If necessary, make 1670 * it the first record. 1671 */ 1672 if (sb->sb_mb != NULL) 1673 sb->sb_mb->m_nextpkt = nextrecord; 1674 else 1675 sb->sb_mb = nextrecord; 1676 1677 /* 1678 * Now update any dependent socket buffer fields to reflect the new 1679 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1680 * addition of a second clause that takes care of the case where 1681 * sb_mb has been updated, but remains the last record. 1682 */ 1683 if (sb->sb_mb == NULL) { 1684 sb->sb_mbtail = NULL; 1685 sb->sb_lastrecord = NULL; 1686 } else if (sb->sb_mb->m_nextpkt == NULL) 1687 sb->sb_lastrecord = sb->sb_mb; 1688 } 1689 1690 /* 1691 * Implement receive operations on a socket. We depend on the way that 1692 * records are added to the sockbuf by sbappend. In particular, each record 1693 * (mbufs linked through m_next) must begin with an address if the protocol 1694 * so specifies, followed by an optional mbuf or mbufs containing ancillary 1695 * data, and then zero or more mbufs of data. In order to allow parallelism 1696 * between network receive and copying to user space, as well as avoid 1697 * sleeping with a mutex held, we release the socket buffer mutex during the 1698 * user space copy. Although the sockbuf is locked, new data may still be 1699 * appended, and thus we must maintain consistency of the sockbuf during that 1700 * time. 1701 * 1702 * The caller may receive the data as a single mbuf chain by supplying an 1703 * mbuf **mp0 for use in returning the chain. The uio is then used only for 1704 * the count in uio_resid. 1705 */ 1706 int 1707 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 1708 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1709 { 1710 struct mbuf *m, **mp; 1711 int flags, error, offset; 1712 ssize_t len; 1713 struct protosw *pr = so->so_proto; 1714 struct mbuf *nextrecord; 1715 int moff, type = 0; 1716 ssize_t orig_resid = uio->uio_resid; 1717 1718 mp = mp0; 1719 if (psa != NULL) 1720 *psa = NULL; 1721 if (controlp != NULL) 1722 *controlp = NULL; 1723 if (flagsp != NULL) 1724 flags = *flagsp &~ MSG_EOR; 1725 else 1726 flags = 0; 1727 if (flags & MSG_OOB) 1728 return (soreceive_rcvoob(so, uio, flags)); 1729 if (mp != NULL) 1730 *mp = NULL; 1731 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1732 && uio->uio_resid) { 1733 VNET_SO_ASSERT(so); 1734 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1735 } 1736 1737 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1738 if (error) 1739 return (error); 1740 1741 restart: 1742 SOCKBUF_LOCK(&so->so_rcv); 1743 m = so->so_rcv.sb_mb; 1744 /* 1745 * If we have less data than requested, block awaiting more (subject 1746 * to any timeout) if: 1747 * 1. the current count is less than the low water mark, or 1748 * 2. MSG_DONTWAIT is not set 1749 */ 1750 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1751 sbavail(&so->so_rcv) < uio->uio_resid) && 1752 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 1753 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1754 KASSERT(m != NULL || !sbavail(&so->so_rcv), 1755 ("receive: m == %p sbavail == %u", 1756 m, sbavail(&so->so_rcv))); 1757 if (so->so_error) { 1758 if (m != NULL) 1759 goto dontblock; 1760 error = so->so_error; 1761 if ((flags & MSG_PEEK) == 0) 1762 so->so_error = 0; 1763 SOCKBUF_UNLOCK(&so->so_rcv); 1764 goto release; 1765 } 1766 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1767 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1768 if (m == NULL) { 1769 SOCKBUF_UNLOCK(&so->so_rcv); 1770 goto release; 1771 } else 1772 goto dontblock; 1773 } 1774 for (; m != NULL; m = m->m_next) 1775 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1776 m = so->so_rcv.sb_mb; 1777 goto dontblock; 1778 } 1779 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1780 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1781 SOCKBUF_UNLOCK(&so->so_rcv); 1782 error = ENOTCONN; 1783 goto release; 1784 } 1785 if (uio->uio_resid == 0) { 1786 SOCKBUF_UNLOCK(&so->so_rcv); 1787 goto release; 1788 } 1789 if ((so->so_state & SS_NBIO) || 1790 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1791 SOCKBUF_UNLOCK(&so->so_rcv); 1792 error = EWOULDBLOCK; 1793 goto release; 1794 } 1795 SBLASTRECORDCHK(&so->so_rcv); 1796 SBLASTMBUFCHK(&so->so_rcv); 1797 error = sbwait(&so->so_rcv); 1798 SOCKBUF_UNLOCK(&so->so_rcv); 1799 if (error) 1800 goto release; 1801 goto restart; 1802 } 1803 dontblock: 1804 /* 1805 * From this point onward, we maintain 'nextrecord' as a cache of the 1806 * pointer to the next record in the socket buffer. We must keep the 1807 * various socket buffer pointers and local stack versions of the 1808 * pointers in sync, pushing out modifications before dropping the 1809 * socket buffer mutex, and re-reading them when picking it up. 1810 * 1811 * Otherwise, we will race with the network stack appending new data 1812 * or records onto the socket buffer by using inconsistent/stale 1813 * versions of the field, possibly resulting in socket buffer 1814 * corruption. 1815 * 1816 * By holding the high-level sblock(), we prevent simultaneous 1817 * readers from pulling off the front of the socket buffer. 1818 */ 1819 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1820 if (uio->uio_td) 1821 uio->uio_td->td_ru.ru_msgrcv++; 1822 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1823 SBLASTRECORDCHK(&so->so_rcv); 1824 SBLASTMBUFCHK(&so->so_rcv); 1825 nextrecord = m->m_nextpkt; 1826 if (pr->pr_flags & PR_ADDR) { 1827 KASSERT(m->m_type == MT_SONAME, 1828 ("m->m_type == %d", m->m_type)); 1829 orig_resid = 0; 1830 if (psa != NULL) 1831 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1832 M_NOWAIT); 1833 if (flags & MSG_PEEK) { 1834 m = m->m_next; 1835 } else { 1836 sbfree(&so->so_rcv, m); 1837 so->so_rcv.sb_mb = m_free(m); 1838 m = so->so_rcv.sb_mb; 1839 sockbuf_pushsync(&so->so_rcv, nextrecord); 1840 } 1841 } 1842 1843 /* 1844 * Process one or more MT_CONTROL mbufs present before any data mbufs 1845 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1846 * just copy the data; if !MSG_PEEK, we call into the protocol to 1847 * perform externalization (or freeing if controlp == NULL). 1848 */ 1849 if (m != NULL && m->m_type == MT_CONTROL) { 1850 struct mbuf *cm = NULL, *cmn; 1851 struct mbuf **cme = &cm; 1852 1853 do { 1854 if (flags & MSG_PEEK) { 1855 if (controlp != NULL) { 1856 *controlp = m_copym(m, 0, m->m_len, 1857 M_NOWAIT); 1858 controlp = &(*controlp)->m_next; 1859 } 1860 m = m->m_next; 1861 } else { 1862 sbfree(&so->so_rcv, m); 1863 so->so_rcv.sb_mb = m->m_next; 1864 m->m_next = NULL; 1865 *cme = m; 1866 cme = &(*cme)->m_next; 1867 m = so->so_rcv.sb_mb; 1868 } 1869 } while (m != NULL && m->m_type == MT_CONTROL); 1870 if ((flags & MSG_PEEK) == 0) 1871 sockbuf_pushsync(&so->so_rcv, nextrecord); 1872 while (cm != NULL) { 1873 cmn = cm->m_next; 1874 cm->m_next = NULL; 1875 if (pr->pr_domain->dom_externalize != NULL) { 1876 SOCKBUF_UNLOCK(&so->so_rcv); 1877 VNET_SO_ASSERT(so); 1878 error = (*pr->pr_domain->dom_externalize) 1879 (cm, controlp, flags); 1880 SOCKBUF_LOCK(&so->so_rcv); 1881 } else if (controlp != NULL) 1882 *controlp = cm; 1883 else 1884 m_freem(cm); 1885 if (controlp != NULL) { 1886 orig_resid = 0; 1887 while (*controlp != NULL) 1888 controlp = &(*controlp)->m_next; 1889 } 1890 cm = cmn; 1891 } 1892 if (m != NULL) 1893 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1894 else 1895 nextrecord = so->so_rcv.sb_mb; 1896 orig_resid = 0; 1897 } 1898 if (m != NULL) { 1899 if ((flags & MSG_PEEK) == 0) { 1900 KASSERT(m->m_nextpkt == nextrecord, 1901 ("soreceive: post-control, nextrecord !sync")); 1902 if (nextrecord == NULL) { 1903 KASSERT(so->so_rcv.sb_mb == m, 1904 ("soreceive: post-control, sb_mb!=m")); 1905 KASSERT(so->so_rcv.sb_lastrecord == m, 1906 ("soreceive: post-control, lastrecord!=m")); 1907 } 1908 } 1909 type = m->m_type; 1910 if (type == MT_OOBDATA) 1911 flags |= MSG_OOB; 1912 } else { 1913 if ((flags & MSG_PEEK) == 0) { 1914 KASSERT(so->so_rcv.sb_mb == nextrecord, 1915 ("soreceive: sb_mb != nextrecord")); 1916 if (so->so_rcv.sb_mb == NULL) { 1917 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1918 ("soreceive: sb_lastercord != NULL")); 1919 } 1920 } 1921 } 1922 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1923 SBLASTRECORDCHK(&so->so_rcv); 1924 SBLASTMBUFCHK(&so->so_rcv); 1925 1926 /* 1927 * Now continue to read any data mbufs off of the head of the socket 1928 * buffer until the read request is satisfied. Note that 'type' is 1929 * used to store the type of any mbuf reads that have happened so far 1930 * such that soreceive() can stop reading if the type changes, which 1931 * causes soreceive() to return only one of regular data and inline 1932 * out-of-band data in a single socket receive operation. 1933 */ 1934 moff = 0; 1935 offset = 0; 1936 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0 1937 && error == 0) { 1938 /* 1939 * If the type of mbuf has changed since the last mbuf 1940 * examined ('type'), end the receive operation. 1941 */ 1942 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1943 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 1944 if (type != m->m_type) 1945 break; 1946 } else if (type == MT_OOBDATA) 1947 break; 1948 else 1949 KASSERT(m->m_type == MT_DATA, 1950 ("m->m_type == %d", m->m_type)); 1951 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1952 len = uio->uio_resid; 1953 if (so->so_oobmark && len > so->so_oobmark - offset) 1954 len = so->so_oobmark - offset; 1955 if (len > m->m_len - moff) 1956 len = m->m_len - moff; 1957 /* 1958 * If mp is set, just pass back the mbufs. Otherwise copy 1959 * them out via the uio, then free. Sockbuf must be 1960 * consistent here (points to current mbuf, it points to next 1961 * record) when we drop priority; we must note any additions 1962 * to the sockbuf when we block interrupts again. 1963 */ 1964 if (mp == NULL) { 1965 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1966 SBLASTRECORDCHK(&so->so_rcv); 1967 SBLASTMBUFCHK(&so->so_rcv); 1968 SOCKBUF_UNLOCK(&so->so_rcv); 1969 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1970 SOCKBUF_LOCK(&so->so_rcv); 1971 if (error) { 1972 /* 1973 * The MT_SONAME mbuf has already been removed 1974 * from the record, so it is necessary to 1975 * remove the data mbufs, if any, to preserve 1976 * the invariant in the case of PR_ADDR that 1977 * requires MT_SONAME mbufs at the head of 1978 * each record. 1979 */ 1980 if (pr->pr_flags & PR_ATOMIC && 1981 ((flags & MSG_PEEK) == 0)) 1982 (void)sbdroprecord_locked(&so->so_rcv); 1983 SOCKBUF_UNLOCK(&so->so_rcv); 1984 goto release; 1985 } 1986 } else 1987 uio->uio_resid -= len; 1988 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1989 if (len == m->m_len - moff) { 1990 if (m->m_flags & M_EOR) 1991 flags |= MSG_EOR; 1992 if (flags & MSG_PEEK) { 1993 m = m->m_next; 1994 moff = 0; 1995 } else { 1996 nextrecord = m->m_nextpkt; 1997 sbfree(&so->so_rcv, m); 1998 if (mp != NULL) { 1999 m->m_nextpkt = NULL; 2000 *mp = m; 2001 mp = &m->m_next; 2002 so->so_rcv.sb_mb = m = m->m_next; 2003 *mp = NULL; 2004 } else { 2005 so->so_rcv.sb_mb = m_free(m); 2006 m = so->so_rcv.sb_mb; 2007 } 2008 sockbuf_pushsync(&so->so_rcv, nextrecord); 2009 SBLASTRECORDCHK(&so->so_rcv); 2010 SBLASTMBUFCHK(&so->so_rcv); 2011 } 2012 } else { 2013 if (flags & MSG_PEEK) 2014 moff += len; 2015 else { 2016 if (mp != NULL) { 2017 if (flags & MSG_DONTWAIT) { 2018 *mp = m_copym(m, 0, len, 2019 M_NOWAIT); 2020 if (*mp == NULL) { 2021 /* 2022 * m_copym() couldn't 2023 * allocate an mbuf. 2024 * Adjust uio_resid back 2025 * (it was adjusted 2026 * down by len bytes, 2027 * which we didn't end 2028 * up "copying" over). 2029 */ 2030 uio->uio_resid += len; 2031 break; 2032 } 2033 } else { 2034 SOCKBUF_UNLOCK(&so->so_rcv); 2035 *mp = m_copym(m, 0, len, 2036 M_WAITOK); 2037 SOCKBUF_LOCK(&so->so_rcv); 2038 } 2039 } 2040 sbcut_locked(&so->so_rcv, len); 2041 } 2042 } 2043 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2044 if (so->so_oobmark) { 2045 if ((flags & MSG_PEEK) == 0) { 2046 so->so_oobmark -= len; 2047 if (so->so_oobmark == 0) { 2048 so->so_rcv.sb_state |= SBS_RCVATMARK; 2049 break; 2050 } 2051 } else { 2052 offset += len; 2053 if (offset == so->so_oobmark) 2054 break; 2055 } 2056 } 2057 if (flags & MSG_EOR) 2058 break; 2059 /* 2060 * If the MSG_WAITALL flag is set (for non-atomic socket), we 2061 * must not quit until "uio->uio_resid == 0" or an error 2062 * termination. If a signal/timeout occurs, return with a 2063 * short count but without error. Keep sockbuf locked 2064 * against other readers. 2065 */ 2066 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 2067 !sosendallatonce(so) && nextrecord == NULL) { 2068 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2069 if (so->so_error || 2070 so->so_rcv.sb_state & SBS_CANTRCVMORE) 2071 break; 2072 /* 2073 * Notify the protocol that some data has been 2074 * drained before blocking. 2075 */ 2076 if (pr->pr_flags & PR_WANTRCVD) { 2077 SOCKBUF_UNLOCK(&so->so_rcv); 2078 VNET_SO_ASSERT(so); 2079 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2080 SOCKBUF_LOCK(&so->so_rcv); 2081 } 2082 SBLASTRECORDCHK(&so->so_rcv); 2083 SBLASTMBUFCHK(&so->so_rcv); 2084 /* 2085 * We could receive some data while was notifying 2086 * the protocol. Skip blocking in this case. 2087 */ 2088 if (so->so_rcv.sb_mb == NULL) { 2089 error = sbwait(&so->so_rcv); 2090 if (error) { 2091 SOCKBUF_UNLOCK(&so->so_rcv); 2092 goto release; 2093 } 2094 } 2095 m = so->so_rcv.sb_mb; 2096 if (m != NULL) 2097 nextrecord = m->m_nextpkt; 2098 } 2099 } 2100 2101 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2102 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 2103 flags |= MSG_TRUNC; 2104 if ((flags & MSG_PEEK) == 0) 2105 (void) sbdroprecord_locked(&so->so_rcv); 2106 } 2107 if ((flags & MSG_PEEK) == 0) { 2108 if (m == NULL) { 2109 /* 2110 * First part is an inline SB_EMPTY_FIXUP(). Second 2111 * part makes sure sb_lastrecord is up-to-date if 2112 * there is still data in the socket buffer. 2113 */ 2114 so->so_rcv.sb_mb = nextrecord; 2115 if (so->so_rcv.sb_mb == NULL) { 2116 so->so_rcv.sb_mbtail = NULL; 2117 so->so_rcv.sb_lastrecord = NULL; 2118 } else if (nextrecord->m_nextpkt == NULL) 2119 so->so_rcv.sb_lastrecord = nextrecord; 2120 } 2121 SBLASTRECORDCHK(&so->so_rcv); 2122 SBLASTMBUFCHK(&so->so_rcv); 2123 /* 2124 * If soreceive() is being done from the socket callback, 2125 * then don't need to generate ACK to peer to update window, 2126 * since ACK will be generated on return to TCP. 2127 */ 2128 if (!(flags & MSG_SOCALLBCK) && 2129 (pr->pr_flags & PR_WANTRCVD)) { 2130 SOCKBUF_UNLOCK(&so->so_rcv); 2131 VNET_SO_ASSERT(so); 2132 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2133 SOCKBUF_LOCK(&so->so_rcv); 2134 } 2135 } 2136 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2137 if (orig_resid == uio->uio_resid && orig_resid && 2138 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 2139 SOCKBUF_UNLOCK(&so->so_rcv); 2140 goto restart; 2141 } 2142 SOCKBUF_UNLOCK(&so->so_rcv); 2143 2144 if (flagsp != NULL) 2145 *flagsp |= flags; 2146 release: 2147 sbunlock(&so->so_rcv); 2148 return (error); 2149 } 2150 2151 /* 2152 * Optimized version of soreceive() for stream (TCP) sockets. 2153 * XXXAO: (MSG_WAITALL | MSG_PEEK) isn't properly handled. 2154 */ 2155 int 2156 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 2157 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2158 { 2159 int len = 0, error = 0, flags, oresid; 2160 struct sockbuf *sb; 2161 struct mbuf *m, *n = NULL; 2162 2163 /* We only do stream sockets. */ 2164 if (so->so_type != SOCK_STREAM) 2165 return (EINVAL); 2166 if (psa != NULL) 2167 *psa = NULL; 2168 if (controlp != NULL) 2169 return (EINVAL); 2170 if (flagsp != NULL) 2171 flags = *flagsp &~ MSG_EOR; 2172 else 2173 flags = 0; 2174 if (flags & MSG_OOB) 2175 return (soreceive_rcvoob(so, uio, flags)); 2176 if (mp0 != NULL) 2177 *mp0 = NULL; 2178 2179 sb = &so->so_rcv; 2180 2181 /* Prevent other readers from entering the socket. */ 2182 error = sblock(sb, SBLOCKWAIT(flags)); 2183 if (error) 2184 goto out; 2185 SOCKBUF_LOCK(sb); 2186 2187 /* Easy one, no space to copyout anything. */ 2188 if (uio->uio_resid == 0) { 2189 error = EINVAL; 2190 goto out; 2191 } 2192 oresid = uio->uio_resid; 2193 2194 /* We will never ever get anything unless we are or were connected. */ 2195 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 2196 error = ENOTCONN; 2197 goto out; 2198 } 2199 2200 restart: 2201 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2202 2203 /* Abort if socket has reported problems. */ 2204 if (so->so_error) { 2205 if (sbavail(sb) > 0) 2206 goto deliver; 2207 if (oresid > uio->uio_resid) 2208 goto out; 2209 error = so->so_error; 2210 if (!(flags & MSG_PEEK)) 2211 so->so_error = 0; 2212 goto out; 2213 } 2214 2215 /* Door is closed. Deliver what is left, if any. */ 2216 if (sb->sb_state & SBS_CANTRCVMORE) { 2217 if (sbavail(sb) > 0) 2218 goto deliver; 2219 else 2220 goto out; 2221 } 2222 2223 /* Socket buffer is empty and we shall not block. */ 2224 if (sbavail(sb) == 0 && 2225 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 2226 error = EAGAIN; 2227 goto out; 2228 } 2229 2230 /* Socket buffer got some data that we shall deliver now. */ 2231 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 2232 ((so->so_state & SS_NBIO) || 2233 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 2234 sbavail(sb) >= sb->sb_lowat || 2235 sbavail(sb) >= uio->uio_resid || 2236 sbavail(sb) >= sb->sb_hiwat) ) { 2237 goto deliver; 2238 } 2239 2240 /* On MSG_WAITALL we must wait until all data or error arrives. */ 2241 if ((flags & MSG_WAITALL) && 2242 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 2243 goto deliver; 2244 2245 /* 2246 * Wait and block until (more) data comes in. 2247 * NB: Drops the sockbuf lock during wait. 2248 */ 2249 error = sbwait(sb); 2250 if (error) 2251 goto out; 2252 goto restart; 2253 2254 deliver: 2255 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2256 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 2257 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 2258 2259 /* Statistics. */ 2260 if (uio->uio_td) 2261 uio->uio_td->td_ru.ru_msgrcv++; 2262 2263 /* Fill uio until full or current end of socket buffer is reached. */ 2264 len = min(uio->uio_resid, sbavail(sb)); 2265 if (mp0 != NULL) { 2266 /* Dequeue as many mbufs as possible. */ 2267 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 2268 if (*mp0 == NULL) 2269 *mp0 = sb->sb_mb; 2270 else 2271 m_cat(*mp0, sb->sb_mb); 2272 for (m = sb->sb_mb; 2273 m != NULL && m->m_len <= len; 2274 m = m->m_next) { 2275 KASSERT(!(m->m_flags & M_NOTAVAIL), 2276 ("%s: m %p not available", __func__, m)); 2277 len -= m->m_len; 2278 uio->uio_resid -= m->m_len; 2279 sbfree(sb, m); 2280 n = m; 2281 } 2282 n->m_next = NULL; 2283 sb->sb_mb = m; 2284 sb->sb_lastrecord = sb->sb_mb; 2285 if (sb->sb_mb == NULL) 2286 SB_EMPTY_FIXUP(sb); 2287 } 2288 /* Copy the remainder. */ 2289 if (len > 0) { 2290 KASSERT(sb->sb_mb != NULL, 2291 ("%s: len > 0 && sb->sb_mb empty", __func__)); 2292 2293 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 2294 if (m == NULL) 2295 len = 0; /* Don't flush data from sockbuf. */ 2296 else 2297 uio->uio_resid -= len; 2298 if (*mp0 != NULL) 2299 m_cat(*mp0, m); 2300 else 2301 *mp0 = m; 2302 if (*mp0 == NULL) { 2303 error = ENOBUFS; 2304 goto out; 2305 } 2306 } 2307 } else { 2308 /* NB: Must unlock socket buffer as uiomove may sleep. */ 2309 SOCKBUF_UNLOCK(sb); 2310 error = m_mbuftouio(uio, sb->sb_mb, len); 2311 SOCKBUF_LOCK(sb); 2312 if (error) 2313 goto out; 2314 } 2315 SBLASTRECORDCHK(sb); 2316 SBLASTMBUFCHK(sb); 2317 2318 /* 2319 * Remove the delivered data from the socket buffer unless we 2320 * were only peeking. 2321 */ 2322 if (!(flags & MSG_PEEK)) { 2323 if (len > 0) 2324 sbdrop_locked(sb, len); 2325 2326 /* Notify protocol that we drained some data. */ 2327 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 2328 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 2329 !(flags & MSG_SOCALLBCK))) { 2330 SOCKBUF_UNLOCK(sb); 2331 VNET_SO_ASSERT(so); 2332 (*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags); 2333 SOCKBUF_LOCK(sb); 2334 } 2335 } 2336 2337 /* 2338 * For MSG_WAITALL we may have to loop again and wait for 2339 * more data to come in. 2340 */ 2341 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 2342 goto restart; 2343 out: 2344 SOCKBUF_LOCK_ASSERT(sb); 2345 SBLASTRECORDCHK(sb); 2346 SBLASTMBUFCHK(sb); 2347 SOCKBUF_UNLOCK(sb); 2348 sbunlock(sb); 2349 return (error); 2350 } 2351 2352 /* 2353 * Optimized version of soreceive() for simple datagram cases from userspace. 2354 * Unlike in the stream case, we're able to drop a datagram if copyout() 2355 * fails, and because we handle datagrams atomically, we don't need to use a 2356 * sleep lock to prevent I/O interlacing. 2357 */ 2358 int 2359 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2360 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2361 { 2362 struct mbuf *m, *m2; 2363 int flags, error; 2364 ssize_t len; 2365 struct protosw *pr = so->so_proto; 2366 struct mbuf *nextrecord; 2367 2368 if (psa != NULL) 2369 *psa = NULL; 2370 if (controlp != NULL) 2371 *controlp = NULL; 2372 if (flagsp != NULL) 2373 flags = *flagsp &~ MSG_EOR; 2374 else 2375 flags = 0; 2376 2377 /* 2378 * For any complicated cases, fall back to the full 2379 * soreceive_generic(). 2380 */ 2381 if (mp0 != NULL || (flags & MSG_PEEK) || (flags & MSG_OOB)) 2382 return (soreceive_generic(so, psa, uio, mp0, controlp, 2383 flagsp)); 2384 2385 /* 2386 * Enforce restrictions on use. 2387 */ 2388 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 2389 ("soreceive_dgram: wantrcvd")); 2390 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 2391 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 2392 ("soreceive_dgram: SBS_RCVATMARK")); 2393 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 2394 ("soreceive_dgram: P_CONNREQUIRED")); 2395 2396 /* 2397 * Loop blocking while waiting for a datagram. 2398 */ 2399 SOCKBUF_LOCK(&so->so_rcv); 2400 while ((m = so->so_rcv.sb_mb) == NULL) { 2401 KASSERT(sbavail(&so->so_rcv) == 0, 2402 ("soreceive_dgram: sb_mb NULL but sbavail %u", 2403 sbavail(&so->so_rcv))); 2404 if (so->so_error) { 2405 error = so->so_error; 2406 so->so_error = 0; 2407 SOCKBUF_UNLOCK(&so->so_rcv); 2408 return (error); 2409 } 2410 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2411 uio->uio_resid == 0) { 2412 SOCKBUF_UNLOCK(&so->so_rcv); 2413 return (0); 2414 } 2415 if ((so->so_state & SS_NBIO) || 2416 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2417 SOCKBUF_UNLOCK(&so->so_rcv); 2418 return (EWOULDBLOCK); 2419 } 2420 SBLASTRECORDCHK(&so->so_rcv); 2421 SBLASTMBUFCHK(&so->so_rcv); 2422 error = sbwait(&so->so_rcv); 2423 if (error) { 2424 SOCKBUF_UNLOCK(&so->so_rcv); 2425 return (error); 2426 } 2427 } 2428 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2429 2430 if (uio->uio_td) 2431 uio->uio_td->td_ru.ru_msgrcv++; 2432 SBLASTRECORDCHK(&so->so_rcv); 2433 SBLASTMBUFCHK(&so->so_rcv); 2434 nextrecord = m->m_nextpkt; 2435 if (nextrecord == NULL) { 2436 KASSERT(so->so_rcv.sb_lastrecord == m, 2437 ("soreceive_dgram: lastrecord != m")); 2438 } 2439 2440 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 2441 ("soreceive_dgram: m_nextpkt != nextrecord")); 2442 2443 /* 2444 * Pull 'm' and its chain off the front of the packet queue. 2445 */ 2446 so->so_rcv.sb_mb = NULL; 2447 sockbuf_pushsync(&so->so_rcv, nextrecord); 2448 2449 /* 2450 * Walk 'm's chain and free that many bytes from the socket buffer. 2451 */ 2452 for (m2 = m; m2 != NULL; m2 = m2->m_next) 2453 sbfree(&so->so_rcv, m2); 2454 2455 /* 2456 * Do a few last checks before we let go of the lock. 2457 */ 2458 SBLASTRECORDCHK(&so->so_rcv); 2459 SBLASTMBUFCHK(&so->so_rcv); 2460 SOCKBUF_UNLOCK(&so->so_rcv); 2461 2462 if (pr->pr_flags & PR_ADDR) { 2463 KASSERT(m->m_type == MT_SONAME, 2464 ("m->m_type == %d", m->m_type)); 2465 if (psa != NULL) 2466 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2467 M_NOWAIT); 2468 m = m_free(m); 2469 } 2470 if (m == NULL) { 2471 /* XXXRW: Can this happen? */ 2472 return (0); 2473 } 2474 2475 /* 2476 * Packet to copyout() is now in 'm' and it is disconnected from the 2477 * queue. 2478 * 2479 * Process one or more MT_CONTROL mbufs present before any data mbufs 2480 * in the first mbuf chain on the socket buffer. We call into the 2481 * protocol to perform externalization (or freeing if controlp == 2482 * NULL). In some cases there can be only MT_CONTROL mbufs without 2483 * MT_DATA mbufs. 2484 */ 2485 if (m->m_type == MT_CONTROL) { 2486 struct mbuf *cm = NULL, *cmn; 2487 struct mbuf **cme = &cm; 2488 2489 do { 2490 m2 = m->m_next; 2491 m->m_next = NULL; 2492 *cme = m; 2493 cme = &(*cme)->m_next; 2494 m = m2; 2495 } while (m != NULL && m->m_type == MT_CONTROL); 2496 while (cm != NULL) { 2497 cmn = cm->m_next; 2498 cm->m_next = NULL; 2499 if (pr->pr_domain->dom_externalize != NULL) { 2500 error = (*pr->pr_domain->dom_externalize) 2501 (cm, controlp, flags); 2502 } else if (controlp != NULL) 2503 *controlp = cm; 2504 else 2505 m_freem(cm); 2506 if (controlp != NULL) { 2507 while (*controlp != NULL) 2508 controlp = &(*controlp)->m_next; 2509 } 2510 cm = cmn; 2511 } 2512 } 2513 KASSERT(m == NULL || m->m_type == MT_DATA, 2514 ("soreceive_dgram: !data")); 2515 while (m != NULL && uio->uio_resid > 0) { 2516 len = uio->uio_resid; 2517 if (len > m->m_len) 2518 len = m->m_len; 2519 error = uiomove(mtod(m, char *), (int)len, uio); 2520 if (error) { 2521 m_freem(m); 2522 return (error); 2523 } 2524 if (len == m->m_len) 2525 m = m_free(m); 2526 else { 2527 m->m_data += len; 2528 m->m_len -= len; 2529 } 2530 } 2531 if (m != NULL) { 2532 flags |= MSG_TRUNC; 2533 m_freem(m); 2534 } 2535 if (flagsp != NULL) 2536 *flagsp |= flags; 2537 return (0); 2538 } 2539 2540 int 2541 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 2542 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2543 { 2544 int error; 2545 2546 CURVNET_SET(so->so_vnet); 2547 error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, mp0, 2548 controlp, flagsp)); 2549 CURVNET_RESTORE(); 2550 return (error); 2551 } 2552 2553 int 2554 soshutdown(struct socket *so, int how) 2555 { 2556 struct protosw *pr = so->so_proto; 2557 int error, soerror_enotconn; 2558 2559 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 2560 return (EINVAL); 2561 2562 soerror_enotconn = 0; 2563 if ((so->so_state & 2564 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2565 /* 2566 * POSIX mandates us to return ENOTCONN when shutdown(2) is 2567 * invoked on a datagram sockets, however historically we would 2568 * actually tear socket down. This is known to be leveraged by 2569 * some applications to unblock process waiting in recvXXX(2) 2570 * by other process that it shares that socket with. Try to meet 2571 * both backward-compatibility and POSIX requirements by forcing 2572 * ENOTCONN but still asking protocol to perform pru_shutdown(). 2573 */ 2574 if (so->so_type != SOCK_DGRAM) 2575 return (ENOTCONN); 2576 soerror_enotconn = 1; 2577 } 2578 2579 CURVNET_SET(so->so_vnet); 2580 if (pr->pr_usrreqs->pru_flush != NULL) 2581 (*pr->pr_usrreqs->pru_flush)(so, how); 2582 if (how != SHUT_WR) 2583 sorflush(so); 2584 if (how != SHUT_RD) { 2585 error = (*pr->pr_usrreqs->pru_shutdown)(so); 2586 wakeup(&so->so_timeo); 2587 CURVNET_RESTORE(); 2588 return ((error == 0 && soerror_enotconn) ? ENOTCONN : error); 2589 } 2590 wakeup(&so->so_timeo); 2591 CURVNET_RESTORE(); 2592 2593 return (soerror_enotconn ? ENOTCONN : 0); 2594 } 2595 2596 void 2597 sorflush(struct socket *so) 2598 { 2599 struct sockbuf *sb = &so->so_rcv; 2600 struct protosw *pr = so->so_proto; 2601 struct socket aso; 2602 2603 VNET_SO_ASSERT(so); 2604 2605 /* 2606 * In order to avoid calling dom_dispose with the socket buffer mutex 2607 * held, and in order to generally avoid holding the lock for a long 2608 * time, we make a copy of the socket buffer and clear the original 2609 * (except locks, state). The new socket buffer copy won't have 2610 * initialized locks so we can only call routines that won't use or 2611 * assert those locks. 2612 * 2613 * Dislodge threads currently blocked in receive and wait to acquire 2614 * a lock against other simultaneous readers before clearing the 2615 * socket buffer. Don't let our acquire be interrupted by a signal 2616 * despite any existing socket disposition on interruptable waiting. 2617 */ 2618 socantrcvmore(so); 2619 (void) sblock(sb, SBL_WAIT | SBL_NOINTR); 2620 2621 /* 2622 * Invalidate/clear most of the sockbuf structure, but leave selinfo 2623 * and mutex data unchanged. 2624 */ 2625 SOCKBUF_LOCK(sb); 2626 bzero(&aso, sizeof(aso)); 2627 aso.so_pcb = so->so_pcb; 2628 bcopy(&sb->sb_startzero, &aso.so_rcv.sb_startzero, 2629 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2630 bzero(&sb->sb_startzero, 2631 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2632 SOCKBUF_UNLOCK(sb); 2633 sbunlock(sb); 2634 2635 /* 2636 * Dispose of special rights and flush the copied socket. Don't call 2637 * any unsafe routines (that rely on locks being initialized) on aso. 2638 */ 2639 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 2640 (*pr->pr_domain->dom_dispose)(&aso); 2641 sbrelease_internal(&aso.so_rcv, so); 2642 } 2643 2644 /* 2645 * Wrapper for Socket established helper hook. 2646 * Parameters: socket, context of the hook point, hook id. 2647 */ 2648 static int inline 2649 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 2650 { 2651 struct socket_hhook_data hhook_data = { 2652 .so = so, 2653 .hctx = hctx, 2654 .m = NULL, 2655 .status = 0 2656 }; 2657 2658 CURVNET_SET(so->so_vnet); 2659 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 2660 CURVNET_RESTORE(); 2661 2662 /* Ugly but needed, since hhooks return void for now */ 2663 return (hhook_data.status); 2664 } 2665 2666 /* 2667 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 2668 * additional variant to handle the case where the option value needs to be 2669 * some kind of integer, but not a specific size. In addition to their use 2670 * here, these functions are also called by the protocol-level pr_ctloutput() 2671 * routines. 2672 */ 2673 int 2674 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 2675 { 2676 size_t valsize; 2677 2678 /* 2679 * If the user gives us more than we wanted, we ignore it, but if we 2680 * don't get the minimum length the caller wants, we return EINVAL. 2681 * On success, sopt->sopt_valsize is set to however much we actually 2682 * retrieved. 2683 */ 2684 if ((valsize = sopt->sopt_valsize) < minlen) 2685 return EINVAL; 2686 if (valsize > len) 2687 sopt->sopt_valsize = valsize = len; 2688 2689 if (sopt->sopt_td != NULL) 2690 return (copyin(sopt->sopt_val, buf, valsize)); 2691 2692 bcopy(sopt->sopt_val, buf, valsize); 2693 return (0); 2694 } 2695 2696 /* 2697 * Kernel version of setsockopt(2). 2698 * 2699 * XXX: optlen is size_t, not socklen_t 2700 */ 2701 int 2702 so_setsockopt(struct socket *so, int level, int optname, void *optval, 2703 size_t optlen) 2704 { 2705 struct sockopt sopt; 2706 2707 sopt.sopt_level = level; 2708 sopt.sopt_name = optname; 2709 sopt.sopt_dir = SOPT_SET; 2710 sopt.sopt_val = optval; 2711 sopt.sopt_valsize = optlen; 2712 sopt.sopt_td = NULL; 2713 return (sosetopt(so, &sopt)); 2714 } 2715 2716 int 2717 sosetopt(struct socket *so, struct sockopt *sopt) 2718 { 2719 int error, optval; 2720 struct linger l; 2721 struct timeval tv; 2722 sbintime_t val; 2723 uint32_t val32; 2724 #ifdef MAC 2725 struct mac extmac; 2726 #endif 2727 2728 CURVNET_SET(so->so_vnet); 2729 error = 0; 2730 if (sopt->sopt_level != SOL_SOCKET) { 2731 if (so->so_proto->pr_ctloutput != NULL) { 2732 error = (*so->so_proto->pr_ctloutput)(so, sopt); 2733 CURVNET_RESTORE(); 2734 return (error); 2735 } 2736 error = ENOPROTOOPT; 2737 } else { 2738 switch (sopt->sopt_name) { 2739 case SO_ACCEPTFILTER: 2740 error = accept_filt_setopt(so, sopt); 2741 if (error) 2742 goto bad; 2743 break; 2744 2745 case SO_LINGER: 2746 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 2747 if (error) 2748 goto bad; 2749 2750 SOCK_LOCK(so); 2751 so->so_linger = l.l_linger; 2752 if (l.l_onoff) 2753 so->so_options |= SO_LINGER; 2754 else 2755 so->so_options &= ~SO_LINGER; 2756 SOCK_UNLOCK(so); 2757 break; 2758 2759 case SO_DEBUG: 2760 case SO_KEEPALIVE: 2761 case SO_DONTROUTE: 2762 case SO_USELOOPBACK: 2763 case SO_BROADCAST: 2764 case SO_REUSEADDR: 2765 case SO_REUSEPORT: 2766 case SO_OOBINLINE: 2767 case SO_TIMESTAMP: 2768 case SO_BINTIME: 2769 case SO_NOSIGPIPE: 2770 case SO_NO_DDP: 2771 case SO_NO_OFFLOAD: 2772 error = sooptcopyin(sopt, &optval, sizeof optval, 2773 sizeof optval); 2774 if (error) 2775 goto bad; 2776 SOCK_LOCK(so); 2777 if (optval) 2778 so->so_options |= sopt->sopt_name; 2779 else 2780 so->so_options &= ~sopt->sopt_name; 2781 SOCK_UNLOCK(so); 2782 break; 2783 2784 case SO_SETFIB: 2785 error = sooptcopyin(sopt, &optval, sizeof optval, 2786 sizeof optval); 2787 if (error) 2788 goto bad; 2789 2790 if (optval < 0 || optval >= rt_numfibs) { 2791 error = EINVAL; 2792 goto bad; 2793 } 2794 if (((so->so_proto->pr_domain->dom_family == PF_INET) || 2795 (so->so_proto->pr_domain->dom_family == PF_INET6) || 2796 (so->so_proto->pr_domain->dom_family == PF_ROUTE))) 2797 so->so_fibnum = optval; 2798 else 2799 so->so_fibnum = 0; 2800 break; 2801 2802 case SO_USER_COOKIE: 2803 error = sooptcopyin(sopt, &val32, sizeof val32, 2804 sizeof val32); 2805 if (error) 2806 goto bad; 2807 so->so_user_cookie = val32; 2808 break; 2809 2810 case SO_SNDBUF: 2811 case SO_RCVBUF: 2812 case SO_SNDLOWAT: 2813 case SO_RCVLOWAT: 2814 error = sooptcopyin(sopt, &optval, sizeof optval, 2815 sizeof optval); 2816 if (error) 2817 goto bad; 2818 2819 /* 2820 * Values < 1 make no sense for any of these options, 2821 * so disallow them. 2822 */ 2823 if (optval < 1) { 2824 error = EINVAL; 2825 goto bad; 2826 } 2827 2828 switch (sopt->sopt_name) { 2829 case SO_SNDBUF: 2830 case SO_RCVBUF: 2831 if (sbreserve(sopt->sopt_name == SO_SNDBUF ? 2832 &so->so_snd : &so->so_rcv, (u_long)optval, 2833 so, curthread) == 0) { 2834 error = ENOBUFS; 2835 goto bad; 2836 } 2837 (sopt->sopt_name == SO_SNDBUF ? &so->so_snd : 2838 &so->so_rcv)->sb_flags &= ~SB_AUTOSIZE; 2839 break; 2840 2841 /* 2842 * Make sure the low-water is never greater than the 2843 * high-water. 2844 */ 2845 case SO_SNDLOWAT: 2846 SOCKBUF_LOCK(&so->so_snd); 2847 so->so_snd.sb_lowat = 2848 (optval > so->so_snd.sb_hiwat) ? 2849 so->so_snd.sb_hiwat : optval; 2850 SOCKBUF_UNLOCK(&so->so_snd); 2851 break; 2852 case SO_RCVLOWAT: 2853 SOCKBUF_LOCK(&so->so_rcv); 2854 so->so_rcv.sb_lowat = 2855 (optval > so->so_rcv.sb_hiwat) ? 2856 so->so_rcv.sb_hiwat : optval; 2857 SOCKBUF_UNLOCK(&so->so_rcv); 2858 break; 2859 } 2860 break; 2861 2862 case SO_SNDTIMEO: 2863 case SO_RCVTIMEO: 2864 #ifdef COMPAT_FREEBSD32 2865 if (SV_CURPROC_FLAG(SV_ILP32)) { 2866 struct timeval32 tv32; 2867 2868 error = sooptcopyin(sopt, &tv32, sizeof tv32, 2869 sizeof tv32); 2870 CP(tv32, tv, tv_sec); 2871 CP(tv32, tv, tv_usec); 2872 } else 2873 #endif 2874 error = sooptcopyin(sopt, &tv, sizeof tv, 2875 sizeof tv); 2876 if (error) 2877 goto bad; 2878 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 2879 tv.tv_usec >= 1000000) { 2880 error = EDOM; 2881 goto bad; 2882 } 2883 if (tv.tv_sec > INT32_MAX) 2884 val = SBT_MAX; 2885 else 2886 val = tvtosbt(tv); 2887 switch (sopt->sopt_name) { 2888 case SO_SNDTIMEO: 2889 so->so_snd.sb_timeo = val; 2890 break; 2891 case SO_RCVTIMEO: 2892 so->so_rcv.sb_timeo = val; 2893 break; 2894 } 2895 break; 2896 2897 case SO_LABEL: 2898 #ifdef MAC 2899 error = sooptcopyin(sopt, &extmac, sizeof extmac, 2900 sizeof extmac); 2901 if (error) 2902 goto bad; 2903 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 2904 so, &extmac); 2905 #else 2906 error = EOPNOTSUPP; 2907 #endif 2908 break; 2909 2910 case SO_TS_CLOCK: 2911 error = sooptcopyin(sopt, &optval, sizeof optval, 2912 sizeof optval); 2913 if (error) 2914 goto bad; 2915 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 2916 error = EINVAL; 2917 goto bad; 2918 } 2919 so->so_ts_clock = optval; 2920 break; 2921 2922 case SO_MAX_PACING_RATE: 2923 error = sooptcopyin(sopt, &val32, sizeof(val32), 2924 sizeof(val32)); 2925 if (error) 2926 goto bad; 2927 so->so_max_pacing_rate = val32; 2928 break; 2929 2930 default: 2931 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 2932 error = hhook_run_socket(so, sopt, 2933 HHOOK_SOCKET_OPT); 2934 else 2935 error = ENOPROTOOPT; 2936 break; 2937 } 2938 if (error == 0 && so->so_proto->pr_ctloutput != NULL) 2939 (void)(*so->so_proto->pr_ctloutput)(so, sopt); 2940 } 2941 bad: 2942 CURVNET_RESTORE(); 2943 return (error); 2944 } 2945 2946 /* 2947 * Helper routine for getsockopt. 2948 */ 2949 int 2950 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 2951 { 2952 int error; 2953 size_t valsize; 2954 2955 error = 0; 2956 2957 /* 2958 * Documented get behavior is that we always return a value, possibly 2959 * truncated to fit in the user's buffer. Traditional behavior is 2960 * that we always tell the user precisely how much we copied, rather 2961 * than something useful like the total amount we had available for 2962 * her. Note that this interface is not idempotent; the entire 2963 * answer must be generated ahead of time. 2964 */ 2965 valsize = min(len, sopt->sopt_valsize); 2966 sopt->sopt_valsize = valsize; 2967 if (sopt->sopt_val != NULL) { 2968 if (sopt->sopt_td != NULL) 2969 error = copyout(buf, sopt->sopt_val, valsize); 2970 else 2971 bcopy(buf, sopt->sopt_val, valsize); 2972 } 2973 return (error); 2974 } 2975 2976 int 2977 sogetopt(struct socket *so, struct sockopt *sopt) 2978 { 2979 int error, optval; 2980 struct linger l; 2981 struct timeval tv; 2982 #ifdef MAC 2983 struct mac extmac; 2984 #endif 2985 2986 CURVNET_SET(so->so_vnet); 2987 error = 0; 2988 if (sopt->sopt_level != SOL_SOCKET) { 2989 if (so->so_proto->pr_ctloutput != NULL) 2990 error = (*so->so_proto->pr_ctloutput)(so, sopt); 2991 else 2992 error = ENOPROTOOPT; 2993 CURVNET_RESTORE(); 2994 return (error); 2995 } else { 2996 switch (sopt->sopt_name) { 2997 case SO_ACCEPTFILTER: 2998 error = accept_filt_getopt(so, sopt); 2999 break; 3000 3001 case SO_LINGER: 3002 SOCK_LOCK(so); 3003 l.l_onoff = so->so_options & SO_LINGER; 3004 l.l_linger = so->so_linger; 3005 SOCK_UNLOCK(so); 3006 error = sooptcopyout(sopt, &l, sizeof l); 3007 break; 3008 3009 case SO_USELOOPBACK: 3010 case SO_DONTROUTE: 3011 case SO_DEBUG: 3012 case SO_KEEPALIVE: 3013 case SO_REUSEADDR: 3014 case SO_REUSEPORT: 3015 case SO_BROADCAST: 3016 case SO_OOBINLINE: 3017 case SO_ACCEPTCONN: 3018 case SO_TIMESTAMP: 3019 case SO_BINTIME: 3020 case SO_NOSIGPIPE: 3021 optval = so->so_options & sopt->sopt_name; 3022 integer: 3023 error = sooptcopyout(sopt, &optval, sizeof optval); 3024 break; 3025 3026 case SO_TYPE: 3027 optval = so->so_type; 3028 goto integer; 3029 3030 case SO_PROTOCOL: 3031 optval = so->so_proto->pr_protocol; 3032 goto integer; 3033 3034 case SO_ERROR: 3035 SOCK_LOCK(so); 3036 optval = so->so_error; 3037 so->so_error = 0; 3038 SOCK_UNLOCK(so); 3039 goto integer; 3040 3041 case SO_SNDBUF: 3042 optval = so->so_snd.sb_hiwat; 3043 goto integer; 3044 3045 case SO_RCVBUF: 3046 optval = so->so_rcv.sb_hiwat; 3047 goto integer; 3048 3049 case SO_SNDLOWAT: 3050 optval = so->so_snd.sb_lowat; 3051 goto integer; 3052 3053 case SO_RCVLOWAT: 3054 optval = so->so_rcv.sb_lowat; 3055 goto integer; 3056 3057 case SO_SNDTIMEO: 3058 case SO_RCVTIMEO: 3059 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 3060 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 3061 #ifdef COMPAT_FREEBSD32 3062 if (SV_CURPROC_FLAG(SV_ILP32)) { 3063 struct timeval32 tv32; 3064 3065 CP(tv, tv32, tv_sec); 3066 CP(tv, tv32, tv_usec); 3067 error = sooptcopyout(sopt, &tv32, sizeof tv32); 3068 } else 3069 #endif 3070 error = sooptcopyout(sopt, &tv, sizeof tv); 3071 break; 3072 3073 case SO_LABEL: 3074 #ifdef MAC 3075 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3076 sizeof(extmac)); 3077 if (error) 3078 goto bad; 3079 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 3080 so, &extmac); 3081 if (error) 3082 goto bad; 3083 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3084 #else 3085 error = EOPNOTSUPP; 3086 #endif 3087 break; 3088 3089 case SO_PEERLABEL: 3090 #ifdef MAC 3091 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3092 sizeof(extmac)); 3093 if (error) 3094 goto bad; 3095 error = mac_getsockopt_peerlabel( 3096 sopt->sopt_td->td_ucred, so, &extmac); 3097 if (error) 3098 goto bad; 3099 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3100 #else 3101 error = EOPNOTSUPP; 3102 #endif 3103 break; 3104 3105 case SO_LISTENQLIMIT: 3106 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 3107 goto integer; 3108 3109 case SO_LISTENQLEN: 3110 optval = SOLISTENING(so) ? so->sol_qlen : 0; 3111 goto integer; 3112 3113 case SO_LISTENINCQLEN: 3114 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 3115 goto integer; 3116 3117 case SO_TS_CLOCK: 3118 optval = so->so_ts_clock; 3119 goto integer; 3120 3121 case SO_MAX_PACING_RATE: 3122 optval = so->so_max_pacing_rate; 3123 goto integer; 3124 3125 default: 3126 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 3127 error = hhook_run_socket(so, sopt, 3128 HHOOK_SOCKET_OPT); 3129 else 3130 error = ENOPROTOOPT; 3131 break; 3132 } 3133 } 3134 #ifdef MAC 3135 bad: 3136 #endif 3137 CURVNET_RESTORE(); 3138 return (error); 3139 } 3140 3141 int 3142 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 3143 { 3144 struct mbuf *m, *m_prev; 3145 int sopt_size = sopt->sopt_valsize; 3146 3147 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3148 if (m == NULL) 3149 return ENOBUFS; 3150 if (sopt_size > MLEN) { 3151 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 3152 if ((m->m_flags & M_EXT) == 0) { 3153 m_free(m); 3154 return ENOBUFS; 3155 } 3156 m->m_len = min(MCLBYTES, sopt_size); 3157 } else { 3158 m->m_len = min(MLEN, sopt_size); 3159 } 3160 sopt_size -= m->m_len; 3161 *mp = m; 3162 m_prev = m; 3163 3164 while (sopt_size) { 3165 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3166 if (m == NULL) { 3167 m_freem(*mp); 3168 return ENOBUFS; 3169 } 3170 if (sopt_size > MLEN) { 3171 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 3172 M_NOWAIT); 3173 if ((m->m_flags & M_EXT) == 0) { 3174 m_freem(m); 3175 m_freem(*mp); 3176 return ENOBUFS; 3177 } 3178 m->m_len = min(MCLBYTES, sopt_size); 3179 } else { 3180 m->m_len = min(MLEN, sopt_size); 3181 } 3182 sopt_size -= m->m_len; 3183 m_prev->m_next = m; 3184 m_prev = m; 3185 } 3186 return (0); 3187 } 3188 3189 int 3190 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 3191 { 3192 struct mbuf *m0 = m; 3193 3194 if (sopt->sopt_val == NULL) 3195 return (0); 3196 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3197 if (sopt->sopt_td != NULL) { 3198 int error; 3199 3200 error = copyin(sopt->sopt_val, mtod(m, char *), 3201 m->m_len); 3202 if (error != 0) { 3203 m_freem(m0); 3204 return(error); 3205 } 3206 } else 3207 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 3208 sopt->sopt_valsize -= m->m_len; 3209 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3210 m = m->m_next; 3211 } 3212 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 3213 panic("ip6_sooptmcopyin"); 3214 return (0); 3215 } 3216 3217 int 3218 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 3219 { 3220 struct mbuf *m0 = m; 3221 size_t valsize = 0; 3222 3223 if (sopt->sopt_val == NULL) 3224 return (0); 3225 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3226 if (sopt->sopt_td != NULL) { 3227 int error; 3228 3229 error = copyout(mtod(m, char *), sopt->sopt_val, 3230 m->m_len); 3231 if (error != 0) { 3232 m_freem(m0); 3233 return(error); 3234 } 3235 } else 3236 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 3237 sopt->sopt_valsize -= m->m_len; 3238 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3239 valsize += m->m_len; 3240 m = m->m_next; 3241 } 3242 if (m != NULL) { 3243 /* enough soopt buffer should be given from user-land */ 3244 m_freem(m0); 3245 return(EINVAL); 3246 } 3247 sopt->sopt_valsize = valsize; 3248 return (0); 3249 } 3250 3251 /* 3252 * sohasoutofband(): protocol notifies socket layer of the arrival of new 3253 * out-of-band data, which will then notify socket consumers. 3254 */ 3255 void 3256 sohasoutofband(struct socket *so) 3257 { 3258 3259 if (so->so_sigio != NULL) 3260 pgsigio(&so->so_sigio, SIGURG, 0); 3261 selwakeuppri(&so->so_rdsel, PSOCK); 3262 } 3263 3264 int 3265 sopoll(struct socket *so, int events, struct ucred *active_cred, 3266 struct thread *td) 3267 { 3268 3269 /* 3270 * We do not need to set or assert curvnet as long as everyone uses 3271 * sopoll_generic(). 3272 */ 3273 return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred, 3274 td)); 3275 } 3276 3277 int 3278 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, 3279 struct thread *td) 3280 { 3281 int revents; 3282 3283 SOCK_LOCK(so); 3284 if (SOLISTENING(so)) { 3285 if (!(events & (POLLIN | POLLRDNORM))) 3286 revents = 0; 3287 else if (!TAILQ_EMPTY(&so->sol_comp)) 3288 revents = events & (POLLIN | POLLRDNORM); 3289 else { 3290 selrecord(td, &so->so_rdsel); 3291 revents = 0; 3292 } 3293 } else { 3294 revents = 0; 3295 SOCKBUF_LOCK(&so->so_snd); 3296 SOCKBUF_LOCK(&so->so_rcv); 3297 if (events & (POLLIN | POLLRDNORM)) 3298 if (soreadabledata(so)) 3299 revents |= events & (POLLIN | POLLRDNORM); 3300 if (events & (POLLOUT | POLLWRNORM)) 3301 if (sowriteable(so)) 3302 revents |= events & (POLLOUT | POLLWRNORM); 3303 if (events & (POLLPRI | POLLRDBAND)) 3304 if (so->so_oobmark || 3305 (so->so_rcv.sb_state & SBS_RCVATMARK)) 3306 revents |= events & (POLLPRI | POLLRDBAND); 3307 if ((events & POLLINIGNEOF) == 0) { 3308 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3309 revents |= events & (POLLIN | POLLRDNORM); 3310 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 3311 revents |= POLLHUP; 3312 } 3313 } 3314 if (revents == 0) { 3315 if (events & 3316 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) { 3317 selrecord(td, &so->so_rdsel); 3318 so->so_rcv.sb_flags |= SB_SEL; 3319 } 3320 if (events & (POLLOUT | POLLWRNORM)) { 3321 selrecord(td, &so->so_wrsel); 3322 so->so_snd.sb_flags |= SB_SEL; 3323 } 3324 } 3325 SOCKBUF_UNLOCK(&so->so_rcv); 3326 SOCKBUF_UNLOCK(&so->so_snd); 3327 } 3328 SOCK_UNLOCK(so); 3329 return (revents); 3330 } 3331 3332 int 3333 soo_kqfilter(struct file *fp, struct knote *kn) 3334 { 3335 struct socket *so = kn->kn_fp->f_data; 3336 struct sockbuf *sb; 3337 struct knlist *knl; 3338 3339 switch (kn->kn_filter) { 3340 case EVFILT_READ: 3341 kn->kn_fop = &soread_filtops; 3342 knl = &so->so_rdsel.si_note; 3343 sb = &so->so_rcv; 3344 break; 3345 case EVFILT_WRITE: 3346 kn->kn_fop = &sowrite_filtops; 3347 knl = &so->so_wrsel.si_note; 3348 sb = &so->so_snd; 3349 break; 3350 case EVFILT_EMPTY: 3351 kn->kn_fop = &soempty_filtops; 3352 knl = &so->so_wrsel.si_note; 3353 sb = &so->so_snd; 3354 break; 3355 default: 3356 return (EINVAL); 3357 } 3358 3359 SOCK_LOCK(so); 3360 if (SOLISTENING(so)) { 3361 knlist_add(knl, kn, 1); 3362 } else { 3363 SOCKBUF_LOCK(sb); 3364 knlist_add(knl, kn, 1); 3365 sb->sb_flags |= SB_KNOTE; 3366 SOCKBUF_UNLOCK(sb); 3367 } 3368 SOCK_UNLOCK(so); 3369 return (0); 3370 } 3371 3372 /* 3373 * Some routines that return EOPNOTSUPP for entry points that are not 3374 * supported by a protocol. Fill in as needed. 3375 */ 3376 int 3377 pru_accept_notsupp(struct socket *so, struct sockaddr **nam) 3378 { 3379 3380 return EOPNOTSUPP; 3381 } 3382 3383 int 3384 pru_aio_queue_notsupp(struct socket *so, struct kaiocb *job) 3385 { 3386 3387 return EOPNOTSUPP; 3388 } 3389 3390 int 3391 pru_attach_notsupp(struct socket *so, int proto, struct thread *td) 3392 { 3393 3394 return EOPNOTSUPP; 3395 } 3396 3397 int 3398 pru_bind_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3399 { 3400 3401 return EOPNOTSUPP; 3402 } 3403 3404 int 3405 pru_bindat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3406 struct thread *td) 3407 { 3408 3409 return EOPNOTSUPP; 3410 } 3411 3412 int 3413 pru_connect_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3414 { 3415 3416 return EOPNOTSUPP; 3417 } 3418 3419 int 3420 pru_connectat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3421 struct thread *td) 3422 { 3423 3424 return EOPNOTSUPP; 3425 } 3426 3427 int 3428 pru_connect2_notsupp(struct socket *so1, struct socket *so2) 3429 { 3430 3431 return EOPNOTSUPP; 3432 } 3433 3434 int 3435 pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data, 3436 struct ifnet *ifp, struct thread *td) 3437 { 3438 3439 return EOPNOTSUPP; 3440 } 3441 3442 int 3443 pru_disconnect_notsupp(struct socket *so) 3444 { 3445 3446 return EOPNOTSUPP; 3447 } 3448 3449 int 3450 pru_listen_notsupp(struct socket *so, int backlog, struct thread *td) 3451 { 3452 3453 return EOPNOTSUPP; 3454 } 3455 3456 int 3457 pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam) 3458 { 3459 3460 return EOPNOTSUPP; 3461 } 3462 3463 int 3464 pru_rcvd_notsupp(struct socket *so, int flags) 3465 { 3466 3467 return EOPNOTSUPP; 3468 } 3469 3470 int 3471 pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags) 3472 { 3473 3474 return EOPNOTSUPP; 3475 } 3476 3477 int 3478 pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, 3479 struct sockaddr *addr, struct mbuf *control, struct thread *td) 3480 { 3481 3482 return EOPNOTSUPP; 3483 } 3484 3485 int 3486 pru_ready_notsupp(struct socket *so, struct mbuf *m, int count) 3487 { 3488 3489 return (EOPNOTSUPP); 3490 } 3491 3492 /* 3493 * This isn't really a ``null'' operation, but it's the default one and 3494 * doesn't do anything destructive. 3495 */ 3496 int 3497 pru_sense_null(struct socket *so, struct stat *sb) 3498 { 3499 3500 sb->st_blksize = so->so_snd.sb_hiwat; 3501 return 0; 3502 } 3503 3504 int 3505 pru_shutdown_notsupp(struct socket *so) 3506 { 3507 3508 return EOPNOTSUPP; 3509 } 3510 3511 int 3512 pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam) 3513 { 3514 3515 return EOPNOTSUPP; 3516 } 3517 3518 int 3519 pru_sosend_notsupp(struct socket *so, struct sockaddr *addr, struct uio *uio, 3520 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 3521 { 3522 3523 return EOPNOTSUPP; 3524 } 3525 3526 int 3527 pru_soreceive_notsupp(struct socket *so, struct sockaddr **paddr, 3528 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3529 { 3530 3531 return EOPNOTSUPP; 3532 } 3533 3534 int 3535 pru_sopoll_notsupp(struct socket *so, int events, struct ucred *cred, 3536 struct thread *td) 3537 { 3538 3539 return EOPNOTSUPP; 3540 } 3541 3542 static void 3543 filt_sordetach(struct knote *kn) 3544 { 3545 struct socket *so = kn->kn_fp->f_data; 3546 3547 so_rdknl_lock(so); 3548 knlist_remove(&so->so_rdsel.si_note, kn, 1); 3549 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 3550 so->so_rcv.sb_flags &= ~SB_KNOTE; 3551 so_rdknl_unlock(so); 3552 } 3553 3554 /*ARGSUSED*/ 3555 static int 3556 filt_soread(struct knote *kn, long hint) 3557 { 3558 struct socket *so; 3559 3560 so = kn->kn_fp->f_data; 3561 3562 if (SOLISTENING(so)) { 3563 SOCK_LOCK_ASSERT(so); 3564 kn->kn_data = so->sol_qlen; 3565 return (!TAILQ_EMPTY(&so->sol_comp)); 3566 } 3567 3568 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3569 3570 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 3571 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3572 kn->kn_flags |= EV_EOF; 3573 kn->kn_fflags = so->so_error; 3574 return (1); 3575 } else if (so->so_error) /* temporary udp error */ 3576 return (1); 3577 3578 if (kn->kn_sfflags & NOTE_LOWAT) { 3579 if (kn->kn_data >= kn->kn_sdata) 3580 return (1); 3581 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 3582 return (1); 3583 3584 /* This hook returning non-zero indicates an event, not error */ 3585 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 3586 } 3587 3588 static void 3589 filt_sowdetach(struct knote *kn) 3590 { 3591 struct socket *so = kn->kn_fp->f_data; 3592 3593 so_wrknl_lock(so); 3594 knlist_remove(&so->so_wrsel.si_note, kn, 1); 3595 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 3596 so->so_snd.sb_flags &= ~SB_KNOTE; 3597 so_wrknl_unlock(so); 3598 } 3599 3600 /*ARGSUSED*/ 3601 static int 3602 filt_sowrite(struct knote *kn, long hint) 3603 { 3604 struct socket *so; 3605 3606 so = kn->kn_fp->f_data; 3607 3608 if (SOLISTENING(so)) 3609 return (0); 3610 3611 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3612 kn->kn_data = sbspace(&so->so_snd); 3613 3614 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 3615 3616 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 3617 kn->kn_flags |= EV_EOF; 3618 kn->kn_fflags = so->so_error; 3619 return (1); 3620 } else if (so->so_error) /* temporary udp error */ 3621 return (1); 3622 else if (((so->so_state & SS_ISCONNECTED) == 0) && 3623 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 3624 return (0); 3625 else if (kn->kn_sfflags & NOTE_LOWAT) 3626 return (kn->kn_data >= kn->kn_sdata); 3627 else 3628 return (kn->kn_data >= so->so_snd.sb_lowat); 3629 } 3630 3631 static int 3632 filt_soempty(struct knote *kn, long hint) 3633 { 3634 struct socket *so; 3635 3636 so = kn->kn_fp->f_data; 3637 3638 if (SOLISTENING(so)) 3639 return (1); 3640 3641 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3642 kn->kn_data = sbused(&so->so_snd); 3643 3644 if (kn->kn_data == 0) 3645 return (1); 3646 else 3647 return (0); 3648 } 3649 3650 int 3651 socheckuid(struct socket *so, uid_t uid) 3652 { 3653 3654 if (so == NULL) 3655 return (EPERM); 3656 if (so->so_cred->cr_uid != uid) 3657 return (EPERM); 3658 return (0); 3659 } 3660 3661 /* 3662 * These functions are used by protocols to notify the socket layer (and its 3663 * consumers) of state changes in the sockets driven by protocol-side events. 3664 */ 3665 3666 /* 3667 * Procedures to manipulate state flags of socket and do appropriate wakeups. 3668 * 3669 * Normal sequence from the active (originating) side is that 3670 * soisconnecting() is called during processing of connect() call, resulting 3671 * in an eventual call to soisconnected() if/when the connection is 3672 * established. When the connection is torn down soisdisconnecting() is 3673 * called during processing of disconnect() call, and soisdisconnected() is 3674 * called when the connection to the peer is totally severed. The semantics 3675 * of these routines are such that connectionless protocols can call 3676 * soisconnected() and soisdisconnected() only, bypassing the in-progress 3677 * calls when setting up a ``connection'' takes no time. 3678 * 3679 * From the passive side, a socket is created with two queues of sockets: 3680 * so_incomp for connections in progress and so_comp for connections already 3681 * made and awaiting user acceptance. As a protocol is preparing incoming 3682 * connections, it creates a socket structure queued on so_incomp by calling 3683 * sonewconn(). When the connection is established, soisconnected() is 3684 * called, and transfers the socket structure to so_comp, making it available 3685 * to accept(). 3686 * 3687 * If a socket is closed with sockets on either so_incomp or so_comp, these 3688 * sockets are dropped. 3689 * 3690 * If higher-level protocols are implemented in the kernel, the wakeups done 3691 * here will sometimes cause software-interrupt process scheduling. 3692 */ 3693 void 3694 soisconnecting(struct socket *so) 3695 { 3696 3697 SOCK_LOCK(so); 3698 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 3699 so->so_state |= SS_ISCONNECTING; 3700 SOCK_UNLOCK(so); 3701 } 3702 3703 void 3704 soisconnected(struct socket *so) 3705 { 3706 struct socket *head; 3707 int ret; 3708 3709 /* 3710 * XXXGL: this is the only place where we acquire socket locks 3711 * in reverse order: first child, then listening socket. To 3712 * avoid possible LOR, use try semantics. 3713 */ 3714 restart: 3715 SOCK_LOCK(so); 3716 if ((head = so->so_listen) != NULL && 3717 __predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 3718 SOCK_UNLOCK(so); 3719 goto restart; 3720 } 3721 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING); 3722 so->so_state |= SS_ISCONNECTED; 3723 if (head != NULL && (so->so_qstate == SQ_INCOMP)) { 3724 again: 3725 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 3726 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 3727 head->sol_incqlen--; 3728 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 3729 head->sol_qlen++; 3730 so->so_qstate = SQ_COMP; 3731 SOCK_UNLOCK(so); 3732 solisten_wakeup(head); /* unlocks */ 3733 } else { 3734 SOCKBUF_LOCK(&so->so_rcv); 3735 soupcall_set(so, SO_RCV, 3736 head->sol_accept_filter->accf_callback, 3737 head->sol_accept_filter_arg); 3738 so->so_options &= ~SO_ACCEPTFILTER; 3739 ret = head->sol_accept_filter->accf_callback(so, 3740 head->sol_accept_filter_arg, M_NOWAIT); 3741 if (ret == SU_ISCONNECTED) { 3742 soupcall_clear(so, SO_RCV); 3743 SOCKBUF_UNLOCK(&so->so_rcv); 3744 goto again; 3745 } 3746 SOCKBUF_UNLOCK(&so->so_rcv); 3747 SOCK_UNLOCK(so); 3748 SOLISTEN_UNLOCK(head); 3749 } 3750 return; 3751 } 3752 if (head != NULL) 3753 SOLISTEN_UNLOCK(head); 3754 SOCK_UNLOCK(so); 3755 wakeup(&so->so_timeo); 3756 sorwakeup(so); 3757 sowwakeup(so); 3758 } 3759 3760 void 3761 soisdisconnecting(struct socket *so) 3762 { 3763 3764 SOCK_LOCK(so); 3765 so->so_state &= ~SS_ISCONNECTING; 3766 so->so_state |= SS_ISDISCONNECTING; 3767 3768 if (!SOLISTENING(so)) { 3769 SOCKBUF_LOCK(&so->so_rcv); 3770 socantrcvmore_locked(so); 3771 SOCKBUF_LOCK(&so->so_snd); 3772 socantsendmore_locked(so); 3773 } 3774 SOCK_UNLOCK(so); 3775 wakeup(&so->so_timeo); 3776 } 3777 3778 void 3779 soisdisconnected(struct socket *so) 3780 { 3781 3782 SOCK_LOCK(so); 3783 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 3784 so->so_state |= SS_ISDISCONNECTED; 3785 3786 if (!SOLISTENING(so)) { 3787 SOCKBUF_LOCK(&so->so_rcv); 3788 socantrcvmore_locked(so); 3789 SOCKBUF_LOCK(&so->so_snd); 3790 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 3791 socantsendmore_locked(so); 3792 } 3793 SOCK_UNLOCK(so); 3794 wakeup(&so->so_timeo); 3795 } 3796 3797 /* 3798 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 3799 */ 3800 struct sockaddr * 3801 sodupsockaddr(const struct sockaddr *sa, int mflags) 3802 { 3803 struct sockaddr *sa2; 3804 3805 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 3806 if (sa2) 3807 bcopy(sa, sa2, sa->sa_len); 3808 return sa2; 3809 } 3810 3811 /* 3812 * Register per-socket buffer upcalls. 3813 */ 3814 void 3815 soupcall_set(struct socket *so, int which, so_upcall_t func, void *arg) 3816 { 3817 struct sockbuf *sb; 3818 3819 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3820 3821 switch (which) { 3822 case SO_RCV: 3823 sb = &so->so_rcv; 3824 break; 3825 case SO_SND: 3826 sb = &so->so_snd; 3827 break; 3828 default: 3829 panic("soupcall_set: bad which"); 3830 } 3831 SOCKBUF_LOCK_ASSERT(sb); 3832 sb->sb_upcall = func; 3833 sb->sb_upcallarg = arg; 3834 sb->sb_flags |= SB_UPCALL; 3835 } 3836 3837 void 3838 soupcall_clear(struct socket *so, int which) 3839 { 3840 struct sockbuf *sb; 3841 3842 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3843 3844 switch (which) { 3845 case SO_RCV: 3846 sb = &so->so_rcv; 3847 break; 3848 case SO_SND: 3849 sb = &so->so_snd; 3850 break; 3851 default: 3852 panic("soupcall_clear: bad which"); 3853 } 3854 SOCKBUF_LOCK_ASSERT(sb); 3855 KASSERT(sb->sb_upcall != NULL, 3856 ("%s: so %p no upcall to clear", __func__, so)); 3857 sb->sb_upcall = NULL; 3858 sb->sb_upcallarg = NULL; 3859 sb->sb_flags &= ~SB_UPCALL; 3860 } 3861 3862 void 3863 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 3864 { 3865 3866 SOLISTEN_LOCK_ASSERT(so); 3867 so->sol_upcall = func; 3868 so->sol_upcallarg = arg; 3869 } 3870 3871 static void 3872 so_rdknl_lock(void *arg) 3873 { 3874 struct socket *so = arg; 3875 3876 if (SOLISTENING(so)) 3877 SOCK_LOCK(so); 3878 else 3879 SOCKBUF_LOCK(&so->so_rcv); 3880 } 3881 3882 static void 3883 so_rdknl_unlock(void *arg) 3884 { 3885 struct socket *so = arg; 3886 3887 if (SOLISTENING(so)) 3888 SOCK_UNLOCK(so); 3889 else 3890 SOCKBUF_UNLOCK(&so->so_rcv); 3891 } 3892 3893 static void 3894 so_rdknl_assert_locked(void *arg) 3895 { 3896 struct socket *so = arg; 3897 3898 if (SOLISTENING(so)) 3899 SOCK_LOCK_ASSERT(so); 3900 else 3901 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3902 } 3903 3904 static void 3905 so_rdknl_assert_unlocked(void *arg) 3906 { 3907 struct socket *so = arg; 3908 3909 if (SOLISTENING(so)) 3910 SOCK_UNLOCK_ASSERT(so); 3911 else 3912 SOCKBUF_UNLOCK_ASSERT(&so->so_rcv); 3913 } 3914 3915 static void 3916 so_wrknl_lock(void *arg) 3917 { 3918 struct socket *so = arg; 3919 3920 if (SOLISTENING(so)) 3921 SOCK_LOCK(so); 3922 else 3923 SOCKBUF_LOCK(&so->so_snd); 3924 } 3925 3926 static void 3927 so_wrknl_unlock(void *arg) 3928 { 3929 struct socket *so = arg; 3930 3931 if (SOLISTENING(so)) 3932 SOCK_UNLOCK(so); 3933 else 3934 SOCKBUF_UNLOCK(&so->so_snd); 3935 } 3936 3937 static void 3938 so_wrknl_assert_locked(void *arg) 3939 { 3940 struct socket *so = arg; 3941 3942 if (SOLISTENING(so)) 3943 SOCK_LOCK_ASSERT(so); 3944 else 3945 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3946 } 3947 3948 static void 3949 so_wrknl_assert_unlocked(void *arg) 3950 { 3951 struct socket *so = arg; 3952 3953 if (SOLISTENING(so)) 3954 SOCK_UNLOCK_ASSERT(so); 3955 else 3956 SOCKBUF_UNLOCK_ASSERT(&so->so_snd); 3957 } 3958 3959 /* 3960 * Create an external-format (``xsocket'') structure using the information in 3961 * the kernel-format socket structure pointed to by so. This is done to 3962 * reduce the spew of irrelevant information over this interface, to isolate 3963 * user code from changes in the kernel structure, and potentially to provide 3964 * information-hiding if we decide that some of this information should be 3965 * hidden from users. 3966 */ 3967 void 3968 sotoxsocket(struct socket *so, struct xsocket *xso) 3969 { 3970 3971 xso->xso_len = sizeof *xso; 3972 xso->xso_so = so; 3973 xso->so_type = so->so_type; 3974 xso->so_options = so->so_options; 3975 xso->so_linger = so->so_linger; 3976 xso->so_state = so->so_state; 3977 xso->so_pcb = so->so_pcb; 3978 xso->xso_protocol = so->so_proto->pr_protocol; 3979 xso->xso_family = so->so_proto->pr_domain->dom_family; 3980 xso->so_timeo = so->so_timeo; 3981 xso->so_error = so->so_error; 3982 xso->so_uid = so->so_cred->cr_uid; 3983 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 3984 if (SOLISTENING(so)) { 3985 xso->so_qlen = so->sol_qlen; 3986 xso->so_incqlen = so->sol_incqlen; 3987 xso->so_qlimit = so->sol_qlimit; 3988 xso->so_oobmark = 0; 3989 bzero(&xso->so_snd, sizeof(xso->so_snd)); 3990 bzero(&xso->so_rcv, sizeof(xso->so_rcv)); 3991 } else { 3992 xso->so_state |= so->so_qstate; 3993 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 3994 xso->so_oobmark = so->so_oobmark; 3995 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 3996 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 3997 } 3998 } 3999 4000 struct sockbuf * 4001 so_sockbuf_rcv(struct socket *so) 4002 { 4003 4004 return (&so->so_rcv); 4005 } 4006 4007 struct sockbuf * 4008 so_sockbuf_snd(struct socket *so) 4009 { 4010 4011 return (&so->so_snd); 4012 } 4013 4014 int 4015 so_state_get(const struct socket *so) 4016 { 4017 4018 return (so->so_state); 4019 } 4020 4021 void 4022 so_state_set(struct socket *so, int val) 4023 { 4024 4025 so->so_state = val; 4026 } 4027 4028 int 4029 so_options_get(const struct socket *so) 4030 { 4031 4032 return (so->so_options); 4033 } 4034 4035 void 4036 so_options_set(struct socket *so, int val) 4037 { 4038 4039 so->so_options = val; 4040 } 4041 4042 int 4043 so_error_get(const struct socket *so) 4044 { 4045 4046 return (so->so_error); 4047 } 4048 4049 void 4050 so_error_set(struct socket *so, int val) 4051 { 4052 4053 so->so_error = val; 4054 } 4055 4056 int 4057 so_linger_get(const struct socket *so) 4058 { 4059 4060 return (so->so_linger); 4061 } 4062 4063 void 4064 so_linger_set(struct socket *so, int val) 4065 { 4066 4067 so->so_linger = val; 4068 } 4069 4070 struct protosw * 4071 so_protosw_get(const struct socket *so) 4072 { 4073 4074 return (so->so_proto); 4075 } 4076 4077 void 4078 so_protosw_set(struct socket *so, struct protosw *val) 4079 { 4080 4081 so->so_proto = val; 4082 } 4083 4084 void 4085 so_sorwakeup(struct socket *so) 4086 { 4087 4088 sorwakeup(so); 4089 } 4090 4091 void 4092 so_sowwakeup(struct socket *so) 4093 { 4094 4095 sowwakeup(so); 4096 } 4097 4098 void 4099 so_sorwakeup_locked(struct socket *so) 4100 { 4101 4102 sorwakeup_locked(so); 4103 } 4104 4105 void 4106 so_sowwakeup_locked(struct socket *so) 4107 { 4108 4109 sowwakeup_locked(so); 4110 } 4111 4112 void 4113 so_lock(struct socket *so) 4114 { 4115 4116 SOCK_LOCK(so); 4117 } 4118 4119 void 4120 so_unlock(struct socket *so) 4121 { 4122 4123 SOCK_UNLOCK(so); 4124 } 4125