1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. 4 * Copyright (c) 2004 The FreeBSD Foundation 5 * Copyright (c) 2004-2008 Robert N. M. Watson 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. Neither the name of the University nor the names of its contributors 17 * may be used to endorse or promote products derived from this software 18 * without specific prior written permission. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 * 32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94 33 */ 34 35 /* 36 * Comments on the socket life cycle: 37 * 38 * soalloc() sets of socket layer state for a socket, called only by 39 * socreate() and sonewconn(). Socket layer private. 40 * 41 * sodealloc() tears down socket layer state for a socket, called only by 42 * sofree() and sonewconn(). Socket layer private. 43 * 44 * pru_attach() associates protocol layer state with an allocated socket; 45 * called only once, may fail, aborting socket allocation. This is called 46 * from socreate() and sonewconn(). Socket layer private. 47 * 48 * pru_detach() disassociates protocol layer state from an attached socket, 49 * and will be called exactly once for sockets in which pru_attach() has 50 * been successfully called. If pru_attach() returned an error, 51 * pru_detach() will not be called. Socket layer private. 52 * 53 * pru_abort() and pru_close() notify the protocol layer that the last 54 * consumer of a socket is starting to tear down the socket, and that the 55 * protocol should terminate the connection. Historically, pru_abort() also 56 * detached protocol state from the socket state, but this is no longer the 57 * case. 58 * 59 * socreate() creates a socket and attaches protocol state. This is a public 60 * interface that may be used by socket layer consumers to create new 61 * sockets. 62 * 63 * sonewconn() creates a socket and attaches protocol state. This is a 64 * public interface that may be used by protocols to create new sockets when 65 * a new connection is received and will be available for accept() on a 66 * listen socket. 67 * 68 * soclose() destroys a socket after possibly waiting for it to disconnect. 69 * This is a public interface that socket consumers should use to close and 70 * release a socket when done with it. 71 * 72 * soabort() destroys a socket without waiting for it to disconnect (used 73 * only for incoming connections that are already partially or fully 74 * connected). This is used internally by the socket layer when clearing 75 * listen socket queues (due to overflow or close on the listen socket), but 76 * is also a public interface protocols may use to abort connections in 77 * their incomplete listen queues should they no longer be required. Sockets 78 * placed in completed connection listen queues should not be aborted for 79 * reasons described in the comment above the soclose() implementation. This 80 * is not a general purpose close routine, and except in the specific 81 * circumstances described here, should not be used. 82 * 83 * sofree() will free a socket and its protocol state if all references on 84 * the socket have been released, and is the public interface to attempt to 85 * free a socket when a reference is removed. This is a socket layer private 86 * interface. 87 * 88 * NOTE: In addition to socreate() and soclose(), which provide a single 89 * socket reference to the consumer to be managed as required, there are two 90 * calls to explicitly manage socket references, soref(), and sorele(). 91 * Currently, these are generally required only when transitioning a socket 92 * from a listen queue to a file descriptor, in order to prevent garbage 93 * collection of the socket at an untimely moment. For a number of reasons, 94 * these interfaces are not preferred, and should be avoided. 95 * 96 * NOTE: With regard to VNETs the general rule is that callers do not set 97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(), 98 * sofree() (and with that sorele(), sotryfree()), as well as sonewconn() 99 * and sorflush(), which are usually called from a pre-set VNET context. 100 * sopoll() currently does not need a VNET context to be set. 101 */ 102 103 #include <sys/cdefs.h> 104 __FBSDID("$FreeBSD$"); 105 106 #include "opt_inet.h" 107 #include "opt_inet6.h" 108 #include "opt_compat.h" 109 #include "opt_sctp.h" 110 111 #include <sys/param.h> 112 #include <sys/systm.h> 113 #include <sys/fcntl.h> 114 #include <sys/limits.h> 115 #include <sys/lock.h> 116 #include <sys/mac.h> 117 #include <sys/malloc.h> 118 #include <sys/mbuf.h> 119 #include <sys/mutex.h> 120 #include <sys/domain.h> 121 #include <sys/file.h> /* for struct knote */ 122 #include <sys/hhook.h> 123 #include <sys/kernel.h> 124 #include <sys/khelp.h> 125 #include <sys/event.h> 126 #include <sys/eventhandler.h> 127 #include <sys/poll.h> 128 #include <sys/proc.h> 129 #include <sys/protosw.h> 130 #include <sys/socket.h> 131 #include <sys/socketvar.h> 132 #include <sys/resourcevar.h> 133 #include <net/route.h> 134 #include <sys/signalvar.h> 135 #include <sys/stat.h> 136 #include <sys/sx.h> 137 #include <sys/sysctl.h> 138 #include <sys/taskqueue.h> 139 #include <sys/uio.h> 140 #include <sys/jail.h> 141 #include <sys/syslog.h> 142 #include <netinet/in.h> 143 144 #include <net/vnet.h> 145 146 #include <security/mac/mac_framework.h> 147 148 #include <vm/uma.h> 149 150 #ifdef COMPAT_FREEBSD32 151 #include <sys/mount.h> 152 #include <sys/sysent.h> 153 #include <compat/freebsd32/freebsd32.h> 154 #endif 155 156 static int soreceive_rcvoob(struct socket *so, struct uio *uio, 157 int flags); 158 static void so_rdknl_lock(void *); 159 static void so_rdknl_unlock(void *); 160 static void so_rdknl_assert_locked(void *); 161 static void so_rdknl_assert_unlocked(void *); 162 static void so_wrknl_lock(void *); 163 static void so_wrknl_unlock(void *); 164 static void so_wrknl_assert_locked(void *); 165 static void so_wrknl_assert_unlocked(void *); 166 167 static void filt_sordetach(struct knote *kn); 168 static int filt_soread(struct knote *kn, long hint); 169 static void filt_sowdetach(struct knote *kn); 170 static int filt_sowrite(struct knote *kn, long hint); 171 static int filt_soempty(struct knote *kn, long hint); 172 static int inline hhook_run_socket(struct socket *so, void *hctx, int32_t h_id); 173 fo_kqfilter_t soo_kqfilter; 174 175 static struct filterops soread_filtops = { 176 .f_isfd = 1, 177 .f_detach = filt_sordetach, 178 .f_event = filt_soread, 179 }; 180 static struct filterops sowrite_filtops = { 181 .f_isfd = 1, 182 .f_detach = filt_sowdetach, 183 .f_event = filt_sowrite, 184 }; 185 static struct filterops soempty_filtops = { 186 .f_isfd = 1, 187 .f_detach = filt_sowdetach, 188 .f_event = filt_soempty, 189 }; 190 191 so_gen_t so_gencnt; /* generation count for sockets */ 192 193 MALLOC_DEFINE(M_SONAME, "soname", "socket name"); 194 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block"); 195 196 #define VNET_SO_ASSERT(so) \ 197 VNET_ASSERT(curvnet != NULL, \ 198 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so))); 199 200 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]); 201 #define V_socket_hhh VNET(socket_hhh) 202 203 /* 204 * Limit on the number of connections in the listen queue waiting 205 * for accept(2). 206 * NB: The original sysctl somaxconn is still available but hidden 207 * to prevent confusion about the actual purpose of this number. 208 */ 209 static u_int somaxconn = SOMAXCONN; 210 211 static int 212 sysctl_somaxconn(SYSCTL_HANDLER_ARGS) 213 { 214 int error; 215 int val; 216 217 val = somaxconn; 218 error = sysctl_handle_int(oidp, &val, 0, req); 219 if (error || !req->newptr ) 220 return (error); 221 222 /* 223 * The purpose of the UINT_MAX / 3 limit, is so that the formula 224 * 3 * so_qlimit / 2 225 * below, will not overflow. 226 */ 227 228 if (val < 1 || val > UINT_MAX / 3) 229 return (EINVAL); 230 231 somaxconn = val; 232 return (0); 233 } 234 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue, CTLTYPE_UINT | CTLFLAG_RW, 235 0, sizeof(int), sysctl_somaxconn, "I", 236 "Maximum listen socket pending connection accept queue size"); 237 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn, 238 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP, 239 0, sizeof(int), sysctl_somaxconn, "I", 240 "Maximum listen socket pending connection accept queue size (compat)"); 241 242 static int numopensockets; 243 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD, 244 &numopensockets, 0, "Number of open sockets"); 245 246 /* 247 * accept_mtx locks down per-socket fields relating to accept queues. See 248 * socketvar.h for an annotation of the protected fields of struct socket. 249 */ 250 struct mtx accept_mtx; 251 MTX_SYSINIT(accept_mtx, &accept_mtx, "accept", MTX_DEF); 252 253 /* 254 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket 255 * so_gencnt field. 256 */ 257 static struct mtx so_global_mtx; 258 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF); 259 260 /* 261 * General IPC sysctl name space, used by sockets and a variety of other IPC 262 * types. 263 */ 264 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW, 0, "IPC"); 265 266 /* 267 * Initialize the socket subsystem and set up the socket 268 * memory allocator. 269 */ 270 static uma_zone_t socket_zone; 271 int maxsockets; 272 273 static void 274 socket_zone_change(void *tag) 275 { 276 277 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 278 } 279 280 static void 281 socket_hhook_register(int subtype) 282 { 283 284 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype, 285 &V_socket_hhh[subtype], 286 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0) 287 printf("%s: WARNING: unable to register hook\n", __func__); 288 } 289 290 static void 291 socket_hhook_deregister(int subtype) 292 { 293 294 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0) 295 printf("%s: WARNING: unable to deregister hook\n", __func__); 296 } 297 298 static void 299 socket_init(void *tag) 300 { 301 302 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL, 303 NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_NOFREE); 304 maxsockets = uma_zone_set_max(socket_zone, maxsockets); 305 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached"); 306 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL, 307 EVENTHANDLER_PRI_FIRST); 308 } 309 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL); 310 311 static void 312 socket_vnet_init(const void *unused __unused) 313 { 314 int i; 315 316 /* We expect a contiguous range */ 317 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 318 socket_hhook_register(i); 319 } 320 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 321 socket_vnet_init, NULL); 322 323 static void 324 socket_vnet_uninit(const void *unused __unused) 325 { 326 int i; 327 328 for (i = 0; i <= HHOOK_SOCKET_LAST; i++) 329 socket_hhook_deregister(i); 330 } 331 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, 332 socket_vnet_uninit, NULL); 333 334 /* 335 * Initialise maxsockets. This SYSINIT must be run after 336 * tunable_mbinit(). 337 */ 338 static void 339 init_maxsockets(void *ignored) 340 { 341 342 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets); 343 maxsockets = imax(maxsockets, maxfiles); 344 } 345 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL); 346 347 /* 348 * Sysctl to get and set the maximum global sockets limit. Notify protocols 349 * of the change so that they can update their dependent limits as required. 350 */ 351 static int 352 sysctl_maxsockets(SYSCTL_HANDLER_ARGS) 353 { 354 int error, newmaxsockets; 355 356 newmaxsockets = maxsockets; 357 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req); 358 if (error == 0 && req->newptr) { 359 if (newmaxsockets > maxsockets && 360 newmaxsockets <= maxfiles) { 361 maxsockets = newmaxsockets; 362 EVENTHANDLER_INVOKE(maxsockets_change); 363 } else 364 error = EINVAL; 365 } 366 return (error); 367 } 368 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets, CTLTYPE_INT|CTLFLAG_RW, 369 &maxsockets, 0, sysctl_maxsockets, "IU", 370 "Maximum number of sockets available"); 371 372 /* 373 * Socket operation routines. These routines are called by the routines in 374 * sys_socket.c or from a system process, and implement the semantics of 375 * socket operations by switching out to the protocol specific routines. 376 */ 377 378 /* 379 * Get a socket structure from our zone, and initialize it. Note that it 380 * would probably be better to allocate socket and PCB at the same time, but 381 * I'm not convinced that all the protocols can be easily modified to do 382 * this. 383 * 384 * soalloc() returns a socket with a ref count of 0. 385 */ 386 static struct socket * 387 soalloc(struct vnet *vnet) 388 { 389 struct socket *so; 390 391 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO); 392 if (so == NULL) 393 return (NULL); 394 #ifdef MAC 395 if (mac_socket_init(so, M_NOWAIT) != 0) { 396 uma_zfree(socket_zone, so); 397 return (NULL); 398 } 399 #endif 400 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) { 401 uma_zfree(socket_zone, so); 402 return (NULL); 403 } 404 405 /* 406 * The socket locking protocol allows to lock 2 sockets at a time, 407 * however, the first one must be a listening socket. WITNESS lacks 408 * a feature to change class of an existing lock, so we use DUPOK. 409 */ 410 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK); 411 SOCKBUF_LOCK_INIT(&so->so_snd, "so_snd"); 412 SOCKBUF_LOCK_INIT(&so->so_rcv, "so_rcv"); 413 so->so_rcv.sb_sel = &so->so_rdsel; 414 so->so_snd.sb_sel = &so->so_wrsel; 415 sx_init(&so->so_snd.sb_sx, "so_snd_sx"); 416 sx_init(&so->so_rcv.sb_sx, "so_rcv_sx"); 417 TAILQ_INIT(&so->so_snd.sb_aiojobq); 418 TAILQ_INIT(&so->so_rcv.sb_aiojobq); 419 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so); 420 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so); 421 #ifdef VIMAGE 422 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p", 423 __func__, __LINE__, so)); 424 so->so_vnet = vnet; 425 #endif 426 /* We shouldn't need the so_global_mtx */ 427 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) { 428 /* Do we need more comprehensive error returns? */ 429 uma_zfree(socket_zone, so); 430 return (NULL); 431 } 432 mtx_lock(&so_global_mtx); 433 so->so_gencnt = ++so_gencnt; 434 ++numopensockets; 435 #ifdef VIMAGE 436 vnet->vnet_sockcnt++; 437 #endif 438 mtx_unlock(&so_global_mtx); 439 440 return (so); 441 } 442 443 /* 444 * Free the storage associated with a socket at the socket layer, tear down 445 * locks, labels, etc. All protocol state is assumed already to have been 446 * torn down (and possibly never set up) by the caller. 447 */ 448 static void 449 sodealloc(struct socket *so) 450 { 451 452 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count)); 453 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL")); 454 455 mtx_lock(&so_global_mtx); 456 so->so_gencnt = ++so_gencnt; 457 --numopensockets; /* Could be below, but faster here. */ 458 #ifdef VIMAGE 459 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p", 460 __func__, __LINE__, so)); 461 so->so_vnet->vnet_sockcnt--; 462 #endif 463 mtx_unlock(&so_global_mtx); 464 if (so->so_rcv.sb_hiwat) 465 (void)chgsbsize(so->so_cred->cr_uidinfo, 466 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY); 467 if (so->so_snd.sb_hiwat) 468 (void)chgsbsize(so->so_cred->cr_uidinfo, 469 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY); 470 #ifdef MAC 471 mac_socket_destroy(so); 472 #endif 473 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE); 474 475 crfree(so->so_cred); 476 khelp_destroy_osd(&so->osd); 477 if (SOLISTENING(so)) { 478 if (so->sol_accept_filter != NULL) 479 accept_filt_setopt(so, NULL); 480 } else { 481 sx_destroy(&so->so_snd.sb_sx); 482 sx_destroy(&so->so_rcv.sb_sx); 483 SOCKBUF_LOCK_DESTROY(&so->so_snd); 484 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 485 } 486 mtx_destroy(&so->so_lock); 487 uma_zfree(socket_zone, so); 488 } 489 490 /* 491 * socreate returns a socket with a ref count of 1. The socket should be 492 * closed with soclose(). 493 */ 494 int 495 socreate(int dom, struct socket **aso, int type, int proto, 496 struct ucred *cred, struct thread *td) 497 { 498 struct protosw *prp; 499 struct socket *so; 500 int error; 501 502 if (proto) 503 prp = pffindproto(dom, proto, type); 504 else 505 prp = pffindtype(dom, type); 506 507 if (prp == NULL) { 508 /* No support for domain. */ 509 if (pffinddomain(dom) == NULL) 510 return (EAFNOSUPPORT); 511 /* No support for socket type. */ 512 if (proto == 0 && type != 0) 513 return (EPROTOTYPE); 514 return (EPROTONOSUPPORT); 515 } 516 if (prp->pr_usrreqs->pru_attach == NULL || 517 prp->pr_usrreqs->pru_attach == pru_attach_notsupp) 518 return (EPROTONOSUPPORT); 519 520 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0) 521 return (EPROTONOSUPPORT); 522 523 if (prp->pr_type != type) 524 return (EPROTOTYPE); 525 so = soalloc(CRED_TO_VNET(cred)); 526 if (so == NULL) 527 return (ENOBUFS); 528 529 so->so_type = type; 530 so->so_cred = crhold(cred); 531 if ((prp->pr_domain->dom_family == PF_INET) || 532 (prp->pr_domain->dom_family == PF_INET6) || 533 (prp->pr_domain->dom_family == PF_ROUTE)) 534 so->so_fibnum = td->td_proc->p_fibnum; 535 else 536 so->so_fibnum = 0; 537 so->so_proto = prp; 538 #ifdef MAC 539 mac_socket_create(cred, so); 540 #endif 541 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 542 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 543 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 544 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 545 /* 546 * Auto-sizing of socket buffers is managed by the protocols and 547 * the appropriate flags must be set in the pru_attach function. 548 */ 549 CURVNET_SET(so->so_vnet); 550 error = (*prp->pr_usrreqs->pru_attach)(so, proto, td); 551 CURVNET_RESTORE(); 552 if (error) { 553 sodealloc(so); 554 return (error); 555 } 556 soref(so); 557 *aso = so; 558 return (0); 559 } 560 561 #ifdef REGRESSION 562 static int regression_sonewconn_earlytest = 1; 563 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW, 564 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test"); 565 #endif 566 567 /* 568 * When an attempt at a new connection is noted on a socket which accepts 569 * connections, sonewconn is called. If the connection is possible (subject 570 * to space constraints, etc.) then we allocate a new structure, properly 571 * linked into the data structure of the original socket, and return this. 572 * Connstatus may be 0, or SS_ISCONFIRMING, or SS_ISCONNECTED. 573 * 574 * Note: the ref count on the socket is 0 on return. 575 */ 576 struct socket * 577 sonewconn(struct socket *head, int connstatus) 578 { 579 static struct timeval lastover; 580 static struct timeval overinterval = { 60, 0 }; 581 static int overcount; 582 583 struct socket *so; 584 u_int over; 585 586 SOLISTEN_LOCK(head); 587 over = (head->sol_qlen > 3 * head->sol_qlimit / 2); 588 SOLISTEN_UNLOCK(head); 589 #ifdef REGRESSION 590 if (regression_sonewconn_earlytest && over) { 591 #else 592 if (over) { 593 #endif 594 overcount++; 595 596 if (ratecheck(&lastover, &overinterval)) { 597 log(LOG_DEBUG, "%s: pcb %p: Listen queue overflow: " 598 "%i already in queue awaiting acceptance " 599 "(%d occurrences)\n", 600 __func__, head->so_pcb, head->sol_qlen, overcount); 601 602 overcount = 0; 603 } 604 605 return (NULL); 606 } 607 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL", 608 __func__, head)); 609 so = soalloc(head->so_vnet); 610 if (so == NULL) { 611 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 612 "limit reached or out of memory\n", 613 __func__, head->so_pcb); 614 return (NULL); 615 } 616 so->so_listen = head; 617 so->so_type = head->so_type; 618 so->so_linger = head->so_linger; 619 so->so_state = head->so_state | SS_NOFDREF; 620 so->so_fibnum = head->so_fibnum; 621 so->so_proto = head->so_proto; 622 so->so_cred = crhold(head->so_cred); 623 #ifdef MAC 624 mac_socket_newconn(head, so); 625 #endif 626 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 627 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 628 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 629 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 630 VNET_SO_ASSERT(head); 631 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) { 632 sodealloc(so); 633 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 634 __func__, head->so_pcb); 635 return (NULL); 636 } 637 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 638 sodealloc(so); 639 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 640 __func__, head->so_pcb); 641 return (NULL); 642 } 643 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat; 644 so->so_snd.sb_lowat = head->sol_sbsnd_lowat; 645 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo; 646 so->so_snd.sb_timeo = head->sol_sbsnd_timeo; 647 so->so_rcv.sb_flags |= head->sol_sbrcv_flags & SB_AUTOSIZE; 648 so->so_snd.sb_flags |= head->sol_sbsnd_flags & SB_AUTOSIZE; 649 650 SOLISTEN_LOCK(head); 651 if (head->sol_accept_filter != NULL) 652 connstatus = 0; 653 so->so_state |= connstatus; 654 so->so_options = head->so_options & ~SO_ACCEPTCONN; 655 soref(head); /* A socket on (in)complete queue refs head. */ 656 if (connstatus) { 657 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 658 so->so_qstate = SQ_COMP; 659 head->sol_qlen++; 660 solisten_wakeup(head); /* unlocks */ 661 } else { 662 /* 663 * Keep removing sockets from the head until there's room for 664 * us to insert on the tail. In pre-locking revisions, this 665 * was a simple if(), but as we could be racing with other 666 * threads and soabort() requires dropping locks, we must 667 * loop waiting for the condition to be true. 668 */ 669 while (head->sol_incqlen > head->sol_qlimit) { 670 struct socket *sp; 671 672 sp = TAILQ_FIRST(&head->sol_incomp); 673 TAILQ_REMOVE(&head->sol_incomp, sp, so_list); 674 head->sol_incqlen--; 675 SOCK_LOCK(sp); 676 sp->so_qstate = SQ_NONE; 677 sp->so_listen = NULL; 678 SOCK_UNLOCK(sp); 679 sorele(head); /* does SOLISTEN_UNLOCK, head stays */ 680 soabort(sp); 681 SOLISTEN_LOCK(head); 682 } 683 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list); 684 so->so_qstate = SQ_INCOMP; 685 head->sol_incqlen++; 686 SOLISTEN_UNLOCK(head); 687 } 688 return (so); 689 } 690 691 #ifdef SCTP 692 /* 693 * Socket part of sctp_peeloff(). Detach a new socket from an 694 * association. The new socket is returned with a reference. 695 */ 696 struct socket * 697 sopeeloff(struct socket *head) 698 { 699 struct socket *so; 700 701 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p", 702 __func__, __LINE__, head)); 703 so = soalloc(head->so_vnet); 704 if (so == NULL) { 705 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: " 706 "limit reached or out of memory\n", 707 __func__, head->so_pcb); 708 return (NULL); 709 } 710 so->so_type = head->so_type; 711 so->so_options = head->so_options; 712 so->so_linger = head->so_linger; 713 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED; 714 so->so_fibnum = head->so_fibnum; 715 so->so_proto = head->so_proto; 716 so->so_cred = crhold(head->so_cred); 717 #ifdef MAC 718 mac_socket_newconn(head, so); 719 #endif 720 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock, 721 so_rdknl_assert_locked, so_rdknl_assert_unlocked); 722 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock, 723 so_wrknl_assert_locked, so_wrknl_assert_unlocked); 724 VNET_SO_ASSERT(head); 725 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) { 726 sodealloc(so); 727 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n", 728 __func__, head->so_pcb); 729 return (NULL); 730 } 731 if ((*so->so_proto->pr_usrreqs->pru_attach)(so, 0, NULL)) { 732 sodealloc(so); 733 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n", 734 __func__, head->so_pcb); 735 return (NULL); 736 } 737 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat; 738 so->so_snd.sb_lowat = head->so_snd.sb_lowat; 739 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo; 740 so->so_snd.sb_timeo = head->so_snd.sb_timeo; 741 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE; 742 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE; 743 744 soref(so); 745 746 return (so); 747 } 748 #endif /* SCTP */ 749 750 int 751 sobind(struct socket *so, struct sockaddr *nam, struct thread *td) 752 { 753 int error; 754 755 CURVNET_SET(so->so_vnet); 756 error = (*so->so_proto->pr_usrreqs->pru_bind)(so, nam, td); 757 CURVNET_RESTORE(); 758 return (error); 759 } 760 761 int 762 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 763 { 764 int error; 765 766 CURVNET_SET(so->so_vnet); 767 error = (*so->so_proto->pr_usrreqs->pru_bindat)(fd, so, nam, td); 768 CURVNET_RESTORE(); 769 return (error); 770 } 771 772 /* 773 * solisten() transitions a socket from a non-listening state to a listening 774 * state, but can also be used to update the listen queue depth on an 775 * existing listen socket. The protocol will call back into the sockets 776 * layer using solisten_proto_check() and solisten_proto() to check and set 777 * socket-layer listen state. Call backs are used so that the protocol can 778 * acquire both protocol and socket layer locks in whatever order is required 779 * by the protocol. 780 * 781 * Protocol implementors are advised to hold the socket lock across the 782 * socket-layer test and set to avoid races at the socket layer. 783 */ 784 int 785 solisten(struct socket *so, int backlog, struct thread *td) 786 { 787 int error; 788 789 CURVNET_SET(so->so_vnet); 790 error = (*so->so_proto->pr_usrreqs->pru_listen)(so, backlog, td); 791 CURVNET_RESTORE(); 792 return (error); 793 } 794 795 int 796 solisten_proto_check(struct socket *so) 797 { 798 799 SOCK_LOCK_ASSERT(so); 800 801 if (so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING | 802 SS_ISDISCONNECTING)) 803 return (EINVAL); 804 return (0); 805 } 806 807 void 808 solisten_proto(struct socket *so, int backlog) 809 { 810 int sbrcv_lowat, sbsnd_lowat; 811 u_int sbrcv_hiwat, sbsnd_hiwat; 812 short sbrcv_flags, sbsnd_flags; 813 sbintime_t sbrcv_timeo, sbsnd_timeo; 814 815 SOCK_LOCK_ASSERT(so); 816 817 if (SOLISTENING(so)) 818 goto listening; 819 820 /* 821 * Change this socket to listening state. 822 */ 823 sbrcv_lowat = so->so_rcv.sb_lowat; 824 sbsnd_lowat = so->so_snd.sb_lowat; 825 sbrcv_hiwat = so->so_rcv.sb_hiwat; 826 sbsnd_hiwat = so->so_snd.sb_hiwat; 827 sbrcv_flags = so->so_rcv.sb_flags; 828 sbsnd_flags = so->so_snd.sb_flags; 829 sbrcv_timeo = so->so_rcv.sb_timeo; 830 sbsnd_timeo = so->so_snd.sb_timeo; 831 832 sbdestroy(&so->so_snd, so); 833 sbdestroy(&so->so_rcv, so); 834 sx_destroy(&so->so_snd.sb_sx); 835 sx_destroy(&so->so_rcv.sb_sx); 836 SOCKBUF_LOCK_DESTROY(&so->so_snd); 837 SOCKBUF_LOCK_DESTROY(&so->so_rcv); 838 839 #ifdef INVARIANTS 840 bzero(&so->so_rcv, 841 sizeof(struct socket) - offsetof(struct socket, so_rcv)); 842 #endif 843 844 so->sol_sbrcv_lowat = sbrcv_lowat; 845 so->sol_sbsnd_lowat = sbsnd_lowat; 846 so->sol_sbrcv_hiwat = sbrcv_hiwat; 847 so->sol_sbsnd_hiwat = sbsnd_hiwat; 848 so->sol_sbrcv_flags = sbrcv_flags; 849 so->sol_sbsnd_flags = sbsnd_flags; 850 so->sol_sbrcv_timeo = sbrcv_timeo; 851 so->sol_sbsnd_timeo = sbsnd_timeo; 852 853 so->sol_qlen = so->sol_incqlen = 0; 854 TAILQ_INIT(&so->sol_incomp); 855 TAILQ_INIT(&so->sol_comp); 856 857 so->sol_accept_filter = NULL; 858 so->sol_accept_filter_arg = NULL; 859 so->sol_accept_filter_str = NULL; 860 861 so->so_options |= SO_ACCEPTCONN; 862 863 listening: 864 if (backlog < 0 || backlog > somaxconn) 865 backlog = somaxconn; 866 so->sol_qlimit = backlog; 867 } 868 869 /* 870 * Wakeup listeners/subsystems once we have a complete connection. 871 * Enters with lock, returns unlocked. 872 */ 873 void 874 solisten_wakeup(struct socket *sol) 875 { 876 877 if (sol->sol_upcall != NULL) 878 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT); 879 else { 880 selwakeuppri(&sol->so_rdsel, PSOCK); 881 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0); 882 } 883 SOLISTEN_UNLOCK(sol); 884 wakeup_one(&sol->sol_comp); 885 } 886 887 /* 888 * Return single connection off a listening socket queue. Main consumer of 889 * the function is kern_accept4(). Some modules, that do their own accept 890 * management also use the function. 891 * 892 * Listening socket must be locked on entry and is returned unlocked on 893 * return. 894 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT. 895 */ 896 int 897 solisten_dequeue(struct socket *head, struct socket **ret, int flags) 898 { 899 struct socket *so; 900 int error; 901 902 SOLISTEN_LOCK_ASSERT(head); 903 904 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) && 905 head->so_error == 0) { 906 error = msleep(&head->sol_comp, &head->so_lock, PSOCK | PCATCH, 907 "accept", 0); 908 if (error != 0) { 909 SOLISTEN_UNLOCK(head); 910 return (error); 911 } 912 } 913 if (head->so_error) { 914 error = head->so_error; 915 head->so_error = 0; 916 SOLISTEN_UNLOCK(head); 917 return (error); 918 } 919 if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp)) { 920 SOLISTEN_UNLOCK(head); 921 return (EWOULDBLOCK); 922 } 923 so = TAILQ_FIRST(&head->sol_comp); 924 SOCK_LOCK(so); 925 KASSERT(so->so_qstate == SQ_COMP, 926 ("%s: so %p not SQ_COMP", __func__, so)); 927 soref(so); 928 head->sol_qlen--; 929 so->so_qstate = SQ_NONE; 930 so->so_listen = NULL; 931 TAILQ_REMOVE(&head->sol_comp, so, so_list); 932 if (flags & ACCEPT4_INHERIT) 933 so->so_state |= (head->so_state & SS_NBIO); 934 else 935 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0; 936 SOCK_UNLOCK(so); 937 sorele(head); 938 939 *ret = so; 940 return (0); 941 } 942 943 /* 944 * Evaluate the reference count and named references on a socket; if no 945 * references remain, free it. This should be called whenever a reference is 946 * released, such as in sorele(), but also when named reference flags are 947 * cleared in socket or protocol code. 948 * 949 * sofree() will free the socket if: 950 * 951 * - There are no outstanding file descriptor references or related consumers 952 * (so_count == 0). 953 * 954 * - The socket has been closed by user space, if ever open (SS_NOFDREF). 955 * 956 * - The protocol does not have an outstanding strong reference on the socket 957 * (SS_PROTOREF). 958 * 959 * - The socket is not in a completed connection queue, so a process has been 960 * notified that it is present. If it is removed, the user process may 961 * block in accept() despite select() saying the socket was ready. 962 */ 963 void 964 sofree(struct socket *so) 965 { 966 struct protosw *pr = so->so_proto; 967 968 SOCK_LOCK_ASSERT(so); 969 970 if ((so->so_state & SS_NOFDREF) == 0 || so->so_count != 0 || 971 (so->so_state & SS_PROTOREF) || (so->so_qstate == SQ_COMP)) { 972 SOCK_UNLOCK(so); 973 return; 974 } 975 976 if (!SOLISTENING(so) && so->so_qstate == SQ_INCOMP) { 977 struct socket *sol; 978 979 sol = so->so_listen; 980 KASSERT(sol, ("%s: so %p on incomp of NULL", __func__, so)); 981 982 /* 983 * To solve race between close of a listening socket and 984 * a socket on its incomplete queue, we need to lock both. 985 * The order is first listening socket, then regular. 986 * Since we don't have SS_NOFDREF neither SS_PROTOREF, this 987 * function and the listening socket are the only pointers 988 * to so. To preserve so and sol, we reference both and then 989 * relock. 990 * After relock the socket may not move to so_comp since it 991 * doesn't have PCB already, but it may be removed from 992 * so_incomp. If that happens, we share responsiblity on 993 * freeing the socket, but soclose() has already removed 994 * it from queue. 995 */ 996 soref(sol); 997 soref(so); 998 SOCK_UNLOCK(so); 999 SOLISTEN_LOCK(sol); 1000 SOCK_LOCK(so); 1001 if (so->so_qstate == SQ_INCOMP) { 1002 KASSERT(so->so_listen == sol, 1003 ("%s: so %p migrated out of sol %p", 1004 __func__, so, sol)); 1005 TAILQ_REMOVE(&sol->sol_incomp, so, so_list); 1006 sol->sol_incqlen--; 1007 /* This is guarenteed not to be the last. */ 1008 refcount_release(&sol->so_count); 1009 so->so_qstate = SQ_NONE; 1010 so->so_listen = NULL; 1011 } else 1012 KASSERT(so->so_listen == NULL, 1013 ("%s: so %p not on (in)comp with so_listen", 1014 __func__, so)); 1015 sorele(sol); 1016 KASSERT(so->so_count == 1, 1017 ("%s: so %p count %u", __func__, so, so->so_count)); 1018 so->so_count = 0; 1019 } 1020 if (SOLISTENING(so)) 1021 so->so_error = ECONNABORTED; 1022 SOCK_UNLOCK(so); 1023 1024 VNET_SO_ASSERT(so); 1025 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 1026 (*pr->pr_domain->dom_dispose)(so); 1027 if (pr->pr_usrreqs->pru_detach != NULL) 1028 (*pr->pr_usrreqs->pru_detach)(so); 1029 1030 /* 1031 * From this point on, we assume that no other references to this 1032 * socket exist anywhere else in the stack. Therefore, no locks need 1033 * to be acquired or held. 1034 * 1035 * We used to do a lot of socket buffer and socket locking here, as 1036 * well as invoke sorflush() and perform wakeups. The direct call to 1037 * dom_dispose() and sbrelease_internal() are an inlining of what was 1038 * necessary from sorflush(). 1039 * 1040 * Notice that the socket buffer and kqueue state are torn down 1041 * before calling pru_detach. This means that protocols shold not 1042 * assume they can perform socket wakeups, etc, in their detach code. 1043 */ 1044 if (!SOLISTENING(so)) { 1045 sbdestroy(&so->so_snd, so); 1046 sbdestroy(&so->so_rcv, so); 1047 } 1048 seldrain(&so->so_rdsel); 1049 seldrain(&so->so_wrsel); 1050 knlist_destroy(&so->so_rdsel.si_note); 1051 knlist_destroy(&so->so_wrsel.si_note); 1052 sodealloc(so); 1053 } 1054 1055 /* 1056 * Close a socket on last file table reference removal. Initiate disconnect 1057 * if connected. Free socket when disconnect complete. 1058 * 1059 * This function will sorele() the socket. Note that soclose() may be called 1060 * prior to the ref count reaching zero. The actual socket structure will 1061 * not be freed until the ref count reaches zero. 1062 */ 1063 int 1064 soclose(struct socket *so) 1065 { 1066 struct accept_queue lqueue; 1067 bool listening; 1068 int error = 0; 1069 1070 KASSERT(!(so->so_state & SS_NOFDREF), ("soclose: SS_NOFDREF on enter")); 1071 1072 CURVNET_SET(so->so_vnet); 1073 funsetown(&so->so_sigio); 1074 if (so->so_state & SS_ISCONNECTED) { 1075 if ((so->so_state & SS_ISDISCONNECTING) == 0) { 1076 error = sodisconnect(so); 1077 if (error) { 1078 if (error == ENOTCONN) 1079 error = 0; 1080 goto drop; 1081 } 1082 } 1083 if (so->so_options & SO_LINGER) { 1084 if ((so->so_state & SS_ISDISCONNECTING) && 1085 (so->so_state & SS_NBIO)) 1086 goto drop; 1087 while (so->so_state & SS_ISCONNECTED) { 1088 error = tsleep(&so->so_timeo, 1089 PSOCK | PCATCH, "soclos", 1090 so->so_linger * hz); 1091 if (error) 1092 break; 1093 } 1094 } 1095 } 1096 1097 drop: 1098 if (so->so_proto->pr_usrreqs->pru_close != NULL) 1099 (*so->so_proto->pr_usrreqs->pru_close)(so); 1100 1101 SOCK_LOCK(so); 1102 if ((listening = (so->so_options & SO_ACCEPTCONN))) { 1103 struct socket *sp; 1104 1105 TAILQ_INIT(&lqueue); 1106 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list); 1107 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list); 1108 1109 so->sol_qlen = so->sol_incqlen = 0; 1110 1111 TAILQ_FOREACH(sp, &lqueue, so_list) { 1112 SOCK_LOCK(sp); 1113 sp->so_qstate = SQ_NONE; 1114 sp->so_listen = NULL; 1115 SOCK_UNLOCK(sp); 1116 /* Guaranteed not to be the last. */ 1117 refcount_release(&so->so_count); 1118 } 1119 } 1120 KASSERT((so->so_state & SS_NOFDREF) == 0, ("soclose: NOFDREF")); 1121 so->so_state |= SS_NOFDREF; 1122 sorele(so); 1123 if (listening) { 1124 struct socket *sp; 1125 1126 TAILQ_FOREACH(sp, &lqueue, so_list) { 1127 SOCK_LOCK(sp); 1128 if (sp->so_count == 0) { 1129 SOCK_UNLOCK(sp); 1130 soabort(sp); 1131 } else 1132 /* sp is now in sofree() */ 1133 SOCK_UNLOCK(sp); 1134 } 1135 } 1136 CURVNET_RESTORE(); 1137 return (error); 1138 } 1139 1140 /* 1141 * soabort() is used to abruptly tear down a connection, such as when a 1142 * resource limit is reached (listen queue depth exceeded), or if a listen 1143 * socket is closed while there are sockets waiting to be accepted. 1144 * 1145 * This interface is tricky, because it is called on an unreferenced socket, 1146 * and must be called only by a thread that has actually removed the socket 1147 * from the listen queue it was on, or races with other threads are risked. 1148 * 1149 * This interface will call into the protocol code, so must not be called 1150 * with any socket locks held. Protocols do call it while holding their own 1151 * recursible protocol mutexes, but this is something that should be subject 1152 * to review in the future. 1153 */ 1154 void 1155 soabort(struct socket *so) 1156 { 1157 1158 /* 1159 * In as much as is possible, assert that no references to this 1160 * socket are held. This is not quite the same as asserting that the 1161 * current thread is responsible for arranging for no references, but 1162 * is as close as we can get for now. 1163 */ 1164 KASSERT(so->so_count == 0, ("soabort: so_count")); 1165 KASSERT((so->so_state & SS_PROTOREF) == 0, ("soabort: SS_PROTOREF")); 1166 KASSERT(so->so_state & SS_NOFDREF, ("soabort: !SS_NOFDREF")); 1167 KASSERT(so->so_qstate == SQ_NONE, ("soabort: !SQ_NONE")); 1168 VNET_SO_ASSERT(so); 1169 1170 if (so->so_proto->pr_usrreqs->pru_abort != NULL) 1171 (*so->so_proto->pr_usrreqs->pru_abort)(so); 1172 SOCK_LOCK(so); 1173 sofree(so); 1174 } 1175 1176 int 1177 soaccept(struct socket *so, struct sockaddr **nam) 1178 { 1179 int error; 1180 1181 SOCK_LOCK(so); 1182 KASSERT((so->so_state & SS_NOFDREF) != 0, ("soaccept: !NOFDREF")); 1183 so->so_state &= ~SS_NOFDREF; 1184 SOCK_UNLOCK(so); 1185 1186 CURVNET_SET(so->so_vnet); 1187 error = (*so->so_proto->pr_usrreqs->pru_accept)(so, nam); 1188 CURVNET_RESTORE(); 1189 return (error); 1190 } 1191 1192 int 1193 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td) 1194 { 1195 1196 return (soconnectat(AT_FDCWD, so, nam, td)); 1197 } 1198 1199 int 1200 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td) 1201 { 1202 int error; 1203 1204 if (so->so_options & SO_ACCEPTCONN) 1205 return (EOPNOTSUPP); 1206 1207 CURVNET_SET(so->so_vnet); 1208 /* 1209 * If protocol is connection-based, can only connect once. 1210 * Otherwise, if connected, try to disconnect first. This allows 1211 * user to disconnect by connecting to, e.g., a null address. 1212 */ 1213 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) && 1214 ((so->so_proto->pr_flags & PR_CONNREQUIRED) || 1215 (error = sodisconnect(so)))) { 1216 error = EISCONN; 1217 } else { 1218 /* 1219 * Prevent accumulated error from previous connection from 1220 * biting us. 1221 */ 1222 so->so_error = 0; 1223 if (fd == AT_FDCWD) { 1224 error = (*so->so_proto->pr_usrreqs->pru_connect)(so, 1225 nam, td); 1226 } else { 1227 error = (*so->so_proto->pr_usrreqs->pru_connectat)(fd, 1228 so, nam, td); 1229 } 1230 } 1231 CURVNET_RESTORE(); 1232 1233 return (error); 1234 } 1235 1236 int 1237 soconnect2(struct socket *so1, struct socket *so2) 1238 { 1239 int error; 1240 1241 CURVNET_SET(so1->so_vnet); 1242 error = (*so1->so_proto->pr_usrreqs->pru_connect2)(so1, so2); 1243 CURVNET_RESTORE(); 1244 return (error); 1245 } 1246 1247 int 1248 sodisconnect(struct socket *so) 1249 { 1250 int error; 1251 1252 if ((so->so_state & SS_ISCONNECTED) == 0) 1253 return (ENOTCONN); 1254 if (so->so_state & SS_ISDISCONNECTING) 1255 return (EALREADY); 1256 VNET_SO_ASSERT(so); 1257 error = (*so->so_proto->pr_usrreqs->pru_disconnect)(so); 1258 return (error); 1259 } 1260 1261 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT) 1262 1263 int 1264 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio, 1265 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1266 { 1267 long space; 1268 ssize_t resid; 1269 int clen = 0, error, dontroute; 1270 1271 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM")); 1272 KASSERT(so->so_proto->pr_flags & PR_ATOMIC, 1273 ("sosend_dgram: !PR_ATOMIC")); 1274 1275 if (uio != NULL) 1276 resid = uio->uio_resid; 1277 else 1278 resid = top->m_pkthdr.len; 1279 /* 1280 * In theory resid should be unsigned. However, space must be 1281 * signed, as it might be less than 0 if we over-committed, and we 1282 * must use a signed comparison of space and resid. On the other 1283 * hand, a negative resid causes us to loop sending 0-length 1284 * segments to the protocol. 1285 */ 1286 if (resid < 0) { 1287 error = EINVAL; 1288 goto out; 1289 } 1290 1291 dontroute = 1292 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0; 1293 if (td != NULL) 1294 td->td_ru.ru_msgsnd++; 1295 if (control != NULL) 1296 clen = control->m_len; 1297 1298 SOCKBUF_LOCK(&so->so_snd); 1299 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1300 SOCKBUF_UNLOCK(&so->so_snd); 1301 error = EPIPE; 1302 goto out; 1303 } 1304 if (so->so_error) { 1305 error = so->so_error; 1306 so->so_error = 0; 1307 SOCKBUF_UNLOCK(&so->so_snd); 1308 goto out; 1309 } 1310 if ((so->so_state & SS_ISCONNECTED) == 0) { 1311 /* 1312 * `sendto' and `sendmsg' is allowed on a connection-based 1313 * socket if it supports implied connect. Return ENOTCONN if 1314 * not connected and no address is supplied. 1315 */ 1316 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1317 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1318 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1319 !(resid == 0 && clen != 0)) { 1320 SOCKBUF_UNLOCK(&so->so_snd); 1321 error = ENOTCONN; 1322 goto out; 1323 } 1324 } else if (addr == NULL) { 1325 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1326 error = ENOTCONN; 1327 else 1328 error = EDESTADDRREQ; 1329 SOCKBUF_UNLOCK(&so->so_snd); 1330 goto out; 1331 } 1332 } 1333 1334 /* 1335 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a 1336 * problem and need fixing. 1337 */ 1338 space = sbspace(&so->so_snd); 1339 if (flags & MSG_OOB) 1340 space += 1024; 1341 space -= clen; 1342 SOCKBUF_UNLOCK(&so->so_snd); 1343 if (resid > space) { 1344 error = EMSGSIZE; 1345 goto out; 1346 } 1347 if (uio == NULL) { 1348 resid = 0; 1349 if (flags & MSG_EOR) 1350 top->m_flags |= M_EOR; 1351 } else { 1352 /* 1353 * Copy the data from userland into a mbuf chain. 1354 * If no data is to be copied in, a single empty mbuf 1355 * is returned. 1356 */ 1357 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr, 1358 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0))); 1359 if (top == NULL) { 1360 error = EFAULT; /* only possible error */ 1361 goto out; 1362 } 1363 space -= resid - uio->uio_resid; 1364 resid = uio->uio_resid; 1365 } 1366 KASSERT(resid == 0, ("sosend_dgram: resid != 0")); 1367 /* 1368 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock 1369 * than with. 1370 */ 1371 if (dontroute) { 1372 SOCK_LOCK(so); 1373 so->so_options |= SO_DONTROUTE; 1374 SOCK_UNLOCK(so); 1375 } 1376 /* 1377 * XXX all the SBS_CANTSENDMORE checks previously done could be out 1378 * of date. We could have received a reset packet in an interrupt or 1379 * maybe we slept while doing page faults in uiomove() etc. We could 1380 * probably recheck again inside the locking protection here, but 1381 * there are probably other places that this also happens. We must 1382 * rethink this. 1383 */ 1384 VNET_SO_ASSERT(so); 1385 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1386 (flags & MSG_OOB) ? PRUS_OOB : 1387 /* 1388 * If the user set MSG_EOF, the protocol understands this flag and 1389 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND. 1390 */ 1391 ((flags & MSG_EOF) && 1392 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1393 (resid <= 0)) ? 1394 PRUS_EOF : 1395 /* If there is more to send set PRUS_MORETOCOME */ 1396 (flags & MSG_MORETOCOME) || 1397 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1398 top, addr, control, td); 1399 if (dontroute) { 1400 SOCK_LOCK(so); 1401 so->so_options &= ~SO_DONTROUTE; 1402 SOCK_UNLOCK(so); 1403 } 1404 clen = 0; 1405 control = NULL; 1406 top = NULL; 1407 out: 1408 if (top != NULL) 1409 m_freem(top); 1410 if (control != NULL) 1411 m_freem(control); 1412 return (error); 1413 } 1414 1415 /* 1416 * Send on a socket. If send must go all at once and message is larger than 1417 * send buffering, then hard error. Lock against other senders. If must go 1418 * all at once and not enough room now, then inform user that this would 1419 * block and do nothing. Otherwise, if nonblocking, send as much as 1420 * possible. The data to be sent is described by "uio" if nonzero, otherwise 1421 * by the mbuf chain "top" (which must be null if uio is not). Data provided 1422 * in mbuf chain must be small enough to send all at once. 1423 * 1424 * Returns nonzero on error, timeout or signal; callers must check for short 1425 * counts if EINTR/ERESTART are returned. Data and control buffers are freed 1426 * on return. 1427 */ 1428 int 1429 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio, 1430 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1431 { 1432 long space; 1433 ssize_t resid; 1434 int clen = 0, error, dontroute; 1435 int atomic = sosendallatonce(so) || top; 1436 1437 if (uio != NULL) 1438 resid = uio->uio_resid; 1439 else 1440 resid = top->m_pkthdr.len; 1441 /* 1442 * In theory resid should be unsigned. However, space must be 1443 * signed, as it might be less than 0 if we over-committed, and we 1444 * must use a signed comparison of space and resid. On the other 1445 * hand, a negative resid causes us to loop sending 0-length 1446 * segments to the protocol. 1447 * 1448 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM 1449 * type sockets since that's an error. 1450 */ 1451 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) { 1452 error = EINVAL; 1453 goto out; 1454 } 1455 1456 dontroute = 1457 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 && 1458 (so->so_proto->pr_flags & PR_ATOMIC); 1459 if (td != NULL) 1460 td->td_ru.ru_msgsnd++; 1461 if (control != NULL) 1462 clen = control->m_len; 1463 1464 error = sblock(&so->so_snd, SBLOCKWAIT(flags)); 1465 if (error) 1466 goto out; 1467 1468 restart: 1469 do { 1470 SOCKBUF_LOCK(&so->so_snd); 1471 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 1472 SOCKBUF_UNLOCK(&so->so_snd); 1473 error = EPIPE; 1474 goto release; 1475 } 1476 if (so->so_error) { 1477 error = so->so_error; 1478 so->so_error = 0; 1479 SOCKBUF_UNLOCK(&so->so_snd); 1480 goto release; 1481 } 1482 if ((so->so_state & SS_ISCONNECTED) == 0) { 1483 /* 1484 * `sendto' and `sendmsg' is allowed on a connection- 1485 * based socket if it supports implied connect. 1486 * Return ENOTCONN if not connected and no address is 1487 * supplied. 1488 */ 1489 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) && 1490 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) { 1491 if ((so->so_state & SS_ISCONFIRMING) == 0 && 1492 !(resid == 0 && clen != 0)) { 1493 SOCKBUF_UNLOCK(&so->so_snd); 1494 error = ENOTCONN; 1495 goto release; 1496 } 1497 } else if (addr == NULL) { 1498 SOCKBUF_UNLOCK(&so->so_snd); 1499 if (so->so_proto->pr_flags & PR_CONNREQUIRED) 1500 error = ENOTCONN; 1501 else 1502 error = EDESTADDRREQ; 1503 goto release; 1504 } 1505 } 1506 space = sbspace(&so->so_snd); 1507 if (flags & MSG_OOB) 1508 space += 1024; 1509 if ((atomic && resid > so->so_snd.sb_hiwat) || 1510 clen > so->so_snd.sb_hiwat) { 1511 SOCKBUF_UNLOCK(&so->so_snd); 1512 error = EMSGSIZE; 1513 goto release; 1514 } 1515 if (space < resid + clen && 1516 (atomic || space < so->so_snd.sb_lowat || space < clen)) { 1517 if ((so->so_state & SS_NBIO) || (flags & MSG_NBIO)) { 1518 SOCKBUF_UNLOCK(&so->so_snd); 1519 error = EWOULDBLOCK; 1520 goto release; 1521 } 1522 error = sbwait(&so->so_snd); 1523 SOCKBUF_UNLOCK(&so->so_snd); 1524 if (error) 1525 goto release; 1526 goto restart; 1527 } 1528 SOCKBUF_UNLOCK(&so->so_snd); 1529 space -= clen; 1530 do { 1531 if (uio == NULL) { 1532 resid = 0; 1533 if (flags & MSG_EOR) 1534 top->m_flags |= M_EOR; 1535 } else { 1536 /* 1537 * Copy the data from userland into a mbuf 1538 * chain. If resid is 0, which can happen 1539 * only if we have control to send, then 1540 * a single empty mbuf is returned. This 1541 * is a workaround to prevent protocol send 1542 * methods to panic. 1543 */ 1544 top = m_uiotombuf(uio, M_WAITOK, space, 1545 (atomic ? max_hdr : 0), 1546 (atomic ? M_PKTHDR : 0) | 1547 ((flags & MSG_EOR) ? M_EOR : 0)); 1548 if (top == NULL) { 1549 error = EFAULT; /* only possible error */ 1550 goto release; 1551 } 1552 space -= resid - uio->uio_resid; 1553 resid = uio->uio_resid; 1554 } 1555 if (dontroute) { 1556 SOCK_LOCK(so); 1557 so->so_options |= SO_DONTROUTE; 1558 SOCK_UNLOCK(so); 1559 } 1560 /* 1561 * XXX all the SBS_CANTSENDMORE checks previously 1562 * done could be out of date. We could have received 1563 * a reset packet in an interrupt or maybe we slept 1564 * while doing page faults in uiomove() etc. We 1565 * could probably recheck again inside the locking 1566 * protection here, but there are probably other 1567 * places that this also happens. We must rethink 1568 * this. 1569 */ 1570 VNET_SO_ASSERT(so); 1571 error = (*so->so_proto->pr_usrreqs->pru_send)(so, 1572 (flags & MSG_OOB) ? PRUS_OOB : 1573 /* 1574 * If the user set MSG_EOF, the protocol understands 1575 * this flag and nothing left to send then use 1576 * PRU_SEND_EOF instead of PRU_SEND. 1577 */ 1578 ((flags & MSG_EOF) && 1579 (so->so_proto->pr_flags & PR_IMPLOPCL) && 1580 (resid <= 0)) ? 1581 PRUS_EOF : 1582 /* If there is more to send set PRUS_MORETOCOME. */ 1583 (flags & MSG_MORETOCOME) || 1584 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0, 1585 top, addr, control, td); 1586 if (dontroute) { 1587 SOCK_LOCK(so); 1588 so->so_options &= ~SO_DONTROUTE; 1589 SOCK_UNLOCK(so); 1590 } 1591 clen = 0; 1592 control = NULL; 1593 top = NULL; 1594 if (error) 1595 goto release; 1596 } while (resid && space > 0); 1597 } while (resid); 1598 1599 release: 1600 sbunlock(&so->so_snd); 1601 out: 1602 if (top != NULL) 1603 m_freem(top); 1604 if (control != NULL) 1605 m_freem(control); 1606 return (error); 1607 } 1608 1609 int 1610 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio, 1611 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 1612 { 1613 int error; 1614 1615 CURVNET_SET(so->so_vnet); 1616 if (!SOLISTENING(so)) 1617 error = so->so_proto->pr_usrreqs->pru_sosend(so, addr, uio, 1618 top, control, flags, td); 1619 else { 1620 m_freem(top); 1621 m_freem(control); 1622 error = ENOTCONN; 1623 } 1624 CURVNET_RESTORE(); 1625 return (error); 1626 } 1627 1628 /* 1629 * The part of soreceive() that implements reading non-inline out-of-band 1630 * data from a socket. For more complete comments, see soreceive(), from 1631 * which this code originated. 1632 * 1633 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is 1634 * unable to return an mbuf chain to the caller. 1635 */ 1636 static int 1637 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags) 1638 { 1639 struct protosw *pr = so->so_proto; 1640 struct mbuf *m; 1641 int error; 1642 1643 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0")); 1644 VNET_SO_ASSERT(so); 1645 1646 m = m_get(M_WAITOK, MT_DATA); 1647 error = (*pr->pr_usrreqs->pru_rcvoob)(so, m, flags & MSG_PEEK); 1648 if (error) 1649 goto bad; 1650 do { 1651 error = uiomove(mtod(m, void *), 1652 (int) min(uio->uio_resid, m->m_len), uio); 1653 m = m_free(m); 1654 } while (uio->uio_resid && error == 0 && m); 1655 bad: 1656 if (m != NULL) 1657 m_freem(m); 1658 return (error); 1659 } 1660 1661 /* 1662 * Following replacement or removal of the first mbuf on the first mbuf chain 1663 * of a socket buffer, push necessary state changes back into the socket 1664 * buffer so that other consumers see the values consistently. 'nextrecord' 1665 * is the callers locally stored value of the original value of 1666 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes. 1667 * NOTE: 'nextrecord' may be NULL. 1668 */ 1669 static __inline void 1670 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord) 1671 { 1672 1673 SOCKBUF_LOCK_ASSERT(sb); 1674 /* 1675 * First, update for the new value of nextrecord. If necessary, make 1676 * it the first record. 1677 */ 1678 if (sb->sb_mb != NULL) 1679 sb->sb_mb->m_nextpkt = nextrecord; 1680 else 1681 sb->sb_mb = nextrecord; 1682 1683 /* 1684 * Now update any dependent socket buffer fields to reflect the new 1685 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the 1686 * addition of a second clause that takes care of the case where 1687 * sb_mb has been updated, but remains the last record. 1688 */ 1689 if (sb->sb_mb == NULL) { 1690 sb->sb_mbtail = NULL; 1691 sb->sb_lastrecord = NULL; 1692 } else if (sb->sb_mb->m_nextpkt == NULL) 1693 sb->sb_lastrecord = sb->sb_mb; 1694 } 1695 1696 /* 1697 * Implement receive operations on a socket. We depend on the way that 1698 * records are added to the sockbuf by sbappend. In particular, each record 1699 * (mbufs linked through m_next) must begin with an address if the protocol 1700 * so specifies, followed by an optional mbuf or mbufs containing ancillary 1701 * data, and then zero or more mbufs of data. In order to allow parallelism 1702 * between network receive and copying to user space, as well as avoid 1703 * sleeping with a mutex held, we release the socket buffer mutex during the 1704 * user space copy. Although the sockbuf is locked, new data may still be 1705 * appended, and thus we must maintain consistency of the sockbuf during that 1706 * time. 1707 * 1708 * The caller may receive the data as a single mbuf chain by supplying an 1709 * mbuf **mp0 for use in returning the chain. The uio is then used only for 1710 * the count in uio_resid. 1711 */ 1712 int 1713 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio, 1714 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 1715 { 1716 struct mbuf *m, **mp; 1717 int flags, error, offset; 1718 ssize_t len; 1719 struct protosw *pr = so->so_proto; 1720 struct mbuf *nextrecord; 1721 int moff, type = 0; 1722 ssize_t orig_resid = uio->uio_resid; 1723 1724 mp = mp0; 1725 if (psa != NULL) 1726 *psa = NULL; 1727 if (controlp != NULL) 1728 *controlp = NULL; 1729 if (flagsp != NULL) 1730 flags = *flagsp &~ MSG_EOR; 1731 else 1732 flags = 0; 1733 if (flags & MSG_OOB) 1734 return (soreceive_rcvoob(so, uio, flags)); 1735 if (mp != NULL) 1736 *mp = NULL; 1737 if ((pr->pr_flags & PR_WANTRCVD) && (so->so_state & SS_ISCONFIRMING) 1738 && uio->uio_resid) { 1739 VNET_SO_ASSERT(so); 1740 (*pr->pr_usrreqs->pru_rcvd)(so, 0); 1741 } 1742 1743 error = sblock(&so->so_rcv, SBLOCKWAIT(flags)); 1744 if (error) 1745 return (error); 1746 1747 restart: 1748 SOCKBUF_LOCK(&so->so_rcv); 1749 m = so->so_rcv.sb_mb; 1750 /* 1751 * If we have less data than requested, block awaiting more (subject 1752 * to any timeout) if: 1753 * 1. the current count is less than the low water mark, or 1754 * 2. MSG_DONTWAIT is not set 1755 */ 1756 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 && 1757 sbavail(&so->so_rcv) < uio->uio_resid) && 1758 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat && 1759 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) { 1760 KASSERT(m != NULL || !sbavail(&so->so_rcv), 1761 ("receive: m == %p sbavail == %u", 1762 m, sbavail(&so->so_rcv))); 1763 if (so->so_error) { 1764 if (m != NULL) 1765 goto dontblock; 1766 error = so->so_error; 1767 if ((flags & MSG_PEEK) == 0) 1768 so->so_error = 0; 1769 SOCKBUF_UNLOCK(&so->so_rcv); 1770 goto release; 1771 } 1772 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1773 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 1774 if (m == NULL) { 1775 SOCKBUF_UNLOCK(&so->so_rcv); 1776 goto release; 1777 } else 1778 goto dontblock; 1779 } 1780 for (; m != NULL; m = m->m_next) 1781 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) { 1782 m = so->so_rcv.sb_mb; 1783 goto dontblock; 1784 } 1785 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 && 1786 (so->so_proto->pr_flags & PR_CONNREQUIRED)) { 1787 SOCKBUF_UNLOCK(&so->so_rcv); 1788 error = ENOTCONN; 1789 goto release; 1790 } 1791 if (uio->uio_resid == 0) { 1792 SOCKBUF_UNLOCK(&so->so_rcv); 1793 goto release; 1794 } 1795 if ((so->so_state & SS_NBIO) || 1796 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 1797 SOCKBUF_UNLOCK(&so->so_rcv); 1798 error = EWOULDBLOCK; 1799 goto release; 1800 } 1801 SBLASTRECORDCHK(&so->so_rcv); 1802 SBLASTMBUFCHK(&so->so_rcv); 1803 error = sbwait(&so->so_rcv); 1804 SOCKBUF_UNLOCK(&so->so_rcv); 1805 if (error) 1806 goto release; 1807 goto restart; 1808 } 1809 dontblock: 1810 /* 1811 * From this point onward, we maintain 'nextrecord' as a cache of the 1812 * pointer to the next record in the socket buffer. We must keep the 1813 * various socket buffer pointers and local stack versions of the 1814 * pointers in sync, pushing out modifications before dropping the 1815 * socket buffer mutex, and re-reading them when picking it up. 1816 * 1817 * Otherwise, we will race with the network stack appending new data 1818 * or records onto the socket buffer by using inconsistent/stale 1819 * versions of the field, possibly resulting in socket buffer 1820 * corruption. 1821 * 1822 * By holding the high-level sblock(), we prevent simultaneous 1823 * readers from pulling off the front of the socket buffer. 1824 */ 1825 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1826 if (uio->uio_td) 1827 uio->uio_td->td_ru.ru_msgrcv++; 1828 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb")); 1829 SBLASTRECORDCHK(&so->so_rcv); 1830 SBLASTMBUFCHK(&so->so_rcv); 1831 nextrecord = m->m_nextpkt; 1832 if (pr->pr_flags & PR_ADDR) { 1833 KASSERT(m->m_type == MT_SONAME, 1834 ("m->m_type == %d", m->m_type)); 1835 orig_resid = 0; 1836 if (psa != NULL) 1837 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 1838 M_NOWAIT); 1839 if (flags & MSG_PEEK) { 1840 m = m->m_next; 1841 } else { 1842 sbfree(&so->so_rcv, m); 1843 so->so_rcv.sb_mb = m_free(m); 1844 m = so->so_rcv.sb_mb; 1845 sockbuf_pushsync(&so->so_rcv, nextrecord); 1846 } 1847 } 1848 1849 /* 1850 * Process one or more MT_CONTROL mbufs present before any data mbufs 1851 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we 1852 * just copy the data; if !MSG_PEEK, we call into the protocol to 1853 * perform externalization (or freeing if controlp == NULL). 1854 */ 1855 if (m != NULL && m->m_type == MT_CONTROL) { 1856 struct mbuf *cm = NULL, *cmn; 1857 struct mbuf **cme = &cm; 1858 1859 do { 1860 if (flags & MSG_PEEK) { 1861 if (controlp != NULL) { 1862 *controlp = m_copym(m, 0, m->m_len, 1863 M_NOWAIT); 1864 controlp = &(*controlp)->m_next; 1865 } 1866 m = m->m_next; 1867 } else { 1868 sbfree(&so->so_rcv, m); 1869 so->so_rcv.sb_mb = m->m_next; 1870 m->m_next = NULL; 1871 *cme = m; 1872 cme = &(*cme)->m_next; 1873 m = so->so_rcv.sb_mb; 1874 } 1875 } while (m != NULL && m->m_type == MT_CONTROL); 1876 if ((flags & MSG_PEEK) == 0) 1877 sockbuf_pushsync(&so->so_rcv, nextrecord); 1878 while (cm != NULL) { 1879 cmn = cm->m_next; 1880 cm->m_next = NULL; 1881 if (pr->pr_domain->dom_externalize != NULL) { 1882 SOCKBUF_UNLOCK(&so->so_rcv); 1883 VNET_SO_ASSERT(so); 1884 error = (*pr->pr_domain->dom_externalize) 1885 (cm, controlp, flags); 1886 SOCKBUF_LOCK(&so->so_rcv); 1887 } else if (controlp != NULL) 1888 *controlp = cm; 1889 else 1890 m_freem(cm); 1891 if (controlp != NULL) { 1892 orig_resid = 0; 1893 while (*controlp != NULL) 1894 controlp = &(*controlp)->m_next; 1895 } 1896 cm = cmn; 1897 } 1898 if (m != NULL) 1899 nextrecord = so->so_rcv.sb_mb->m_nextpkt; 1900 else 1901 nextrecord = so->so_rcv.sb_mb; 1902 orig_resid = 0; 1903 } 1904 if (m != NULL) { 1905 if ((flags & MSG_PEEK) == 0) { 1906 KASSERT(m->m_nextpkt == nextrecord, 1907 ("soreceive: post-control, nextrecord !sync")); 1908 if (nextrecord == NULL) { 1909 KASSERT(so->so_rcv.sb_mb == m, 1910 ("soreceive: post-control, sb_mb!=m")); 1911 KASSERT(so->so_rcv.sb_lastrecord == m, 1912 ("soreceive: post-control, lastrecord!=m")); 1913 } 1914 } 1915 type = m->m_type; 1916 if (type == MT_OOBDATA) 1917 flags |= MSG_OOB; 1918 } else { 1919 if ((flags & MSG_PEEK) == 0) { 1920 KASSERT(so->so_rcv.sb_mb == nextrecord, 1921 ("soreceive: sb_mb != nextrecord")); 1922 if (so->so_rcv.sb_mb == NULL) { 1923 KASSERT(so->so_rcv.sb_lastrecord == NULL, 1924 ("soreceive: sb_lastercord != NULL")); 1925 } 1926 } 1927 } 1928 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1929 SBLASTRECORDCHK(&so->so_rcv); 1930 SBLASTMBUFCHK(&so->so_rcv); 1931 1932 /* 1933 * Now continue to read any data mbufs off of the head of the socket 1934 * buffer until the read request is satisfied. Note that 'type' is 1935 * used to store the type of any mbuf reads that have happened so far 1936 * such that soreceive() can stop reading if the type changes, which 1937 * causes soreceive() to return only one of regular data and inline 1938 * out-of-band data in a single socket receive operation. 1939 */ 1940 moff = 0; 1941 offset = 0; 1942 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0 1943 && error == 0) { 1944 /* 1945 * If the type of mbuf has changed since the last mbuf 1946 * examined ('type'), end the receive operation. 1947 */ 1948 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1949 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) { 1950 if (type != m->m_type) 1951 break; 1952 } else if (type == MT_OOBDATA) 1953 break; 1954 else 1955 KASSERT(m->m_type == MT_DATA, 1956 ("m->m_type == %d", m->m_type)); 1957 so->so_rcv.sb_state &= ~SBS_RCVATMARK; 1958 len = uio->uio_resid; 1959 if (so->so_oobmark && len > so->so_oobmark - offset) 1960 len = so->so_oobmark - offset; 1961 if (len > m->m_len - moff) 1962 len = m->m_len - moff; 1963 /* 1964 * If mp is set, just pass back the mbufs. Otherwise copy 1965 * them out via the uio, then free. Sockbuf must be 1966 * consistent here (points to current mbuf, it points to next 1967 * record) when we drop priority; we must note any additions 1968 * to the sockbuf when we block interrupts again. 1969 */ 1970 if (mp == NULL) { 1971 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1972 SBLASTRECORDCHK(&so->so_rcv); 1973 SBLASTMBUFCHK(&so->so_rcv); 1974 SOCKBUF_UNLOCK(&so->so_rcv); 1975 error = uiomove(mtod(m, char *) + moff, (int)len, uio); 1976 SOCKBUF_LOCK(&so->so_rcv); 1977 if (error) { 1978 /* 1979 * The MT_SONAME mbuf has already been removed 1980 * from the record, so it is necessary to 1981 * remove the data mbufs, if any, to preserve 1982 * the invariant in the case of PR_ADDR that 1983 * requires MT_SONAME mbufs at the head of 1984 * each record. 1985 */ 1986 if (pr->pr_flags & PR_ATOMIC && 1987 ((flags & MSG_PEEK) == 0)) 1988 (void)sbdroprecord_locked(&so->so_rcv); 1989 SOCKBUF_UNLOCK(&so->so_rcv); 1990 goto release; 1991 } 1992 } else 1993 uio->uio_resid -= len; 1994 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 1995 if (len == m->m_len - moff) { 1996 if (m->m_flags & M_EOR) 1997 flags |= MSG_EOR; 1998 if (flags & MSG_PEEK) { 1999 m = m->m_next; 2000 moff = 0; 2001 } else { 2002 nextrecord = m->m_nextpkt; 2003 sbfree(&so->so_rcv, m); 2004 if (mp != NULL) { 2005 m->m_nextpkt = NULL; 2006 *mp = m; 2007 mp = &m->m_next; 2008 so->so_rcv.sb_mb = m = m->m_next; 2009 *mp = NULL; 2010 } else { 2011 so->so_rcv.sb_mb = m_free(m); 2012 m = so->so_rcv.sb_mb; 2013 } 2014 sockbuf_pushsync(&so->so_rcv, nextrecord); 2015 SBLASTRECORDCHK(&so->so_rcv); 2016 SBLASTMBUFCHK(&so->so_rcv); 2017 } 2018 } else { 2019 if (flags & MSG_PEEK) 2020 moff += len; 2021 else { 2022 if (mp != NULL) { 2023 if (flags & MSG_DONTWAIT) { 2024 *mp = m_copym(m, 0, len, 2025 M_NOWAIT); 2026 if (*mp == NULL) { 2027 /* 2028 * m_copym() couldn't 2029 * allocate an mbuf. 2030 * Adjust uio_resid back 2031 * (it was adjusted 2032 * down by len bytes, 2033 * which we didn't end 2034 * up "copying" over). 2035 */ 2036 uio->uio_resid += len; 2037 break; 2038 } 2039 } else { 2040 SOCKBUF_UNLOCK(&so->so_rcv); 2041 *mp = m_copym(m, 0, len, 2042 M_WAITOK); 2043 SOCKBUF_LOCK(&so->so_rcv); 2044 } 2045 } 2046 sbcut_locked(&so->so_rcv, len); 2047 } 2048 } 2049 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2050 if (so->so_oobmark) { 2051 if ((flags & MSG_PEEK) == 0) { 2052 so->so_oobmark -= len; 2053 if (so->so_oobmark == 0) { 2054 so->so_rcv.sb_state |= SBS_RCVATMARK; 2055 break; 2056 } 2057 } else { 2058 offset += len; 2059 if (offset == so->so_oobmark) 2060 break; 2061 } 2062 } 2063 if (flags & MSG_EOR) 2064 break; 2065 /* 2066 * If the MSG_WAITALL flag is set (for non-atomic socket), we 2067 * must not quit until "uio->uio_resid == 0" or an error 2068 * termination. If a signal/timeout occurs, return with a 2069 * short count but without error. Keep sockbuf locked 2070 * against other readers. 2071 */ 2072 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 && 2073 !sosendallatonce(so) && nextrecord == NULL) { 2074 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2075 if (so->so_error || 2076 so->so_rcv.sb_state & SBS_CANTRCVMORE) 2077 break; 2078 /* 2079 * Notify the protocol that some data has been 2080 * drained before blocking. 2081 */ 2082 if (pr->pr_flags & PR_WANTRCVD) { 2083 SOCKBUF_UNLOCK(&so->so_rcv); 2084 VNET_SO_ASSERT(so); 2085 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2086 SOCKBUF_LOCK(&so->so_rcv); 2087 } 2088 SBLASTRECORDCHK(&so->so_rcv); 2089 SBLASTMBUFCHK(&so->so_rcv); 2090 /* 2091 * We could receive some data while was notifying 2092 * the protocol. Skip blocking in this case. 2093 */ 2094 if (so->so_rcv.sb_mb == NULL) { 2095 error = sbwait(&so->so_rcv); 2096 if (error) { 2097 SOCKBUF_UNLOCK(&so->so_rcv); 2098 goto release; 2099 } 2100 } 2101 m = so->so_rcv.sb_mb; 2102 if (m != NULL) 2103 nextrecord = m->m_nextpkt; 2104 } 2105 } 2106 2107 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2108 if (m != NULL && pr->pr_flags & PR_ATOMIC) { 2109 flags |= MSG_TRUNC; 2110 if ((flags & MSG_PEEK) == 0) 2111 (void) sbdroprecord_locked(&so->so_rcv); 2112 } 2113 if ((flags & MSG_PEEK) == 0) { 2114 if (m == NULL) { 2115 /* 2116 * First part is an inline SB_EMPTY_FIXUP(). Second 2117 * part makes sure sb_lastrecord is up-to-date if 2118 * there is still data in the socket buffer. 2119 */ 2120 so->so_rcv.sb_mb = nextrecord; 2121 if (so->so_rcv.sb_mb == NULL) { 2122 so->so_rcv.sb_mbtail = NULL; 2123 so->so_rcv.sb_lastrecord = NULL; 2124 } else if (nextrecord->m_nextpkt == NULL) 2125 so->so_rcv.sb_lastrecord = nextrecord; 2126 } 2127 SBLASTRECORDCHK(&so->so_rcv); 2128 SBLASTMBUFCHK(&so->so_rcv); 2129 /* 2130 * If soreceive() is being done from the socket callback, 2131 * then don't need to generate ACK to peer to update window, 2132 * since ACK will be generated on return to TCP. 2133 */ 2134 if (!(flags & MSG_SOCALLBCK) && 2135 (pr->pr_flags & PR_WANTRCVD)) { 2136 SOCKBUF_UNLOCK(&so->so_rcv); 2137 VNET_SO_ASSERT(so); 2138 (*pr->pr_usrreqs->pru_rcvd)(so, flags); 2139 SOCKBUF_LOCK(&so->so_rcv); 2140 } 2141 } 2142 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2143 if (orig_resid == uio->uio_resid && orig_resid && 2144 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) { 2145 SOCKBUF_UNLOCK(&so->so_rcv); 2146 goto restart; 2147 } 2148 SOCKBUF_UNLOCK(&so->so_rcv); 2149 2150 if (flagsp != NULL) 2151 *flagsp |= flags; 2152 release: 2153 sbunlock(&so->so_rcv); 2154 return (error); 2155 } 2156 2157 /* 2158 * Optimized version of soreceive() for stream (TCP) sockets. 2159 * XXXAO: (MSG_WAITALL | MSG_PEEK) isn't properly handled. 2160 */ 2161 int 2162 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio, 2163 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2164 { 2165 int len = 0, error = 0, flags, oresid; 2166 struct sockbuf *sb; 2167 struct mbuf *m, *n = NULL; 2168 2169 /* We only do stream sockets. */ 2170 if (so->so_type != SOCK_STREAM) 2171 return (EINVAL); 2172 if (psa != NULL) 2173 *psa = NULL; 2174 if (controlp != NULL) 2175 return (EINVAL); 2176 if (flagsp != NULL) 2177 flags = *flagsp &~ MSG_EOR; 2178 else 2179 flags = 0; 2180 if (flags & MSG_OOB) 2181 return (soreceive_rcvoob(so, uio, flags)); 2182 if (mp0 != NULL) 2183 *mp0 = NULL; 2184 2185 sb = &so->so_rcv; 2186 2187 /* Prevent other readers from entering the socket. */ 2188 error = sblock(sb, SBLOCKWAIT(flags)); 2189 if (error) 2190 goto out; 2191 SOCKBUF_LOCK(sb); 2192 2193 /* Easy one, no space to copyout anything. */ 2194 if (uio->uio_resid == 0) { 2195 error = EINVAL; 2196 goto out; 2197 } 2198 oresid = uio->uio_resid; 2199 2200 /* We will never ever get anything unless we are or were connected. */ 2201 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) { 2202 error = ENOTCONN; 2203 goto out; 2204 } 2205 2206 restart: 2207 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2208 2209 /* Abort if socket has reported problems. */ 2210 if (so->so_error) { 2211 if (sbavail(sb) > 0) 2212 goto deliver; 2213 if (oresid > uio->uio_resid) 2214 goto out; 2215 error = so->so_error; 2216 if (!(flags & MSG_PEEK)) 2217 so->so_error = 0; 2218 goto out; 2219 } 2220 2221 /* Door is closed. Deliver what is left, if any. */ 2222 if (sb->sb_state & SBS_CANTRCVMORE) { 2223 if (sbavail(sb) > 0) 2224 goto deliver; 2225 else 2226 goto out; 2227 } 2228 2229 /* Socket buffer is empty and we shall not block. */ 2230 if (sbavail(sb) == 0 && 2231 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) { 2232 error = EAGAIN; 2233 goto out; 2234 } 2235 2236 /* Socket buffer got some data that we shall deliver now. */ 2237 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) && 2238 ((so->so_state & SS_NBIO) || 2239 (flags & (MSG_DONTWAIT|MSG_NBIO)) || 2240 sbavail(sb) >= sb->sb_lowat || 2241 sbavail(sb) >= uio->uio_resid || 2242 sbavail(sb) >= sb->sb_hiwat) ) { 2243 goto deliver; 2244 } 2245 2246 /* On MSG_WAITALL we must wait until all data or error arrives. */ 2247 if ((flags & MSG_WAITALL) && 2248 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat)) 2249 goto deliver; 2250 2251 /* 2252 * Wait and block until (more) data comes in. 2253 * NB: Drops the sockbuf lock during wait. 2254 */ 2255 error = sbwait(sb); 2256 if (error) 2257 goto out; 2258 goto restart; 2259 2260 deliver: 2261 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2262 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__)); 2263 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__)); 2264 2265 /* Statistics. */ 2266 if (uio->uio_td) 2267 uio->uio_td->td_ru.ru_msgrcv++; 2268 2269 /* Fill uio until full or current end of socket buffer is reached. */ 2270 len = min(uio->uio_resid, sbavail(sb)); 2271 if (mp0 != NULL) { 2272 /* Dequeue as many mbufs as possible. */ 2273 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) { 2274 if (*mp0 == NULL) 2275 *mp0 = sb->sb_mb; 2276 else 2277 m_cat(*mp0, sb->sb_mb); 2278 for (m = sb->sb_mb; 2279 m != NULL && m->m_len <= len; 2280 m = m->m_next) { 2281 KASSERT(!(m->m_flags & M_NOTAVAIL), 2282 ("%s: m %p not available", __func__, m)); 2283 len -= m->m_len; 2284 uio->uio_resid -= m->m_len; 2285 sbfree(sb, m); 2286 n = m; 2287 } 2288 n->m_next = NULL; 2289 sb->sb_mb = m; 2290 sb->sb_lastrecord = sb->sb_mb; 2291 if (sb->sb_mb == NULL) 2292 SB_EMPTY_FIXUP(sb); 2293 } 2294 /* Copy the remainder. */ 2295 if (len > 0) { 2296 KASSERT(sb->sb_mb != NULL, 2297 ("%s: len > 0 && sb->sb_mb empty", __func__)); 2298 2299 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT); 2300 if (m == NULL) 2301 len = 0; /* Don't flush data from sockbuf. */ 2302 else 2303 uio->uio_resid -= len; 2304 if (*mp0 != NULL) 2305 m_cat(*mp0, m); 2306 else 2307 *mp0 = m; 2308 if (*mp0 == NULL) { 2309 error = ENOBUFS; 2310 goto out; 2311 } 2312 } 2313 } else { 2314 /* NB: Must unlock socket buffer as uiomove may sleep. */ 2315 SOCKBUF_UNLOCK(sb); 2316 error = m_mbuftouio(uio, sb->sb_mb, len); 2317 SOCKBUF_LOCK(sb); 2318 if (error) 2319 goto out; 2320 } 2321 SBLASTRECORDCHK(sb); 2322 SBLASTMBUFCHK(sb); 2323 2324 /* 2325 * Remove the delivered data from the socket buffer unless we 2326 * were only peeking. 2327 */ 2328 if (!(flags & MSG_PEEK)) { 2329 if (len > 0) 2330 sbdrop_locked(sb, len); 2331 2332 /* Notify protocol that we drained some data. */ 2333 if ((so->so_proto->pr_flags & PR_WANTRCVD) && 2334 (((flags & MSG_WAITALL) && uio->uio_resid > 0) || 2335 !(flags & MSG_SOCALLBCK))) { 2336 SOCKBUF_UNLOCK(sb); 2337 VNET_SO_ASSERT(so); 2338 (*so->so_proto->pr_usrreqs->pru_rcvd)(so, flags); 2339 SOCKBUF_LOCK(sb); 2340 } 2341 } 2342 2343 /* 2344 * For MSG_WAITALL we may have to loop again and wait for 2345 * more data to come in. 2346 */ 2347 if ((flags & MSG_WAITALL) && uio->uio_resid > 0) 2348 goto restart; 2349 out: 2350 SOCKBUF_LOCK_ASSERT(sb); 2351 SBLASTRECORDCHK(sb); 2352 SBLASTMBUFCHK(sb); 2353 SOCKBUF_UNLOCK(sb); 2354 sbunlock(sb); 2355 return (error); 2356 } 2357 2358 /* 2359 * Optimized version of soreceive() for simple datagram cases from userspace. 2360 * Unlike in the stream case, we're able to drop a datagram if copyout() 2361 * fails, and because we handle datagrams atomically, we don't need to use a 2362 * sleep lock to prevent I/O interlacing. 2363 */ 2364 int 2365 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio, 2366 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2367 { 2368 struct mbuf *m, *m2; 2369 int flags, error; 2370 ssize_t len; 2371 struct protosw *pr = so->so_proto; 2372 struct mbuf *nextrecord; 2373 2374 if (psa != NULL) 2375 *psa = NULL; 2376 if (controlp != NULL) 2377 *controlp = NULL; 2378 if (flagsp != NULL) 2379 flags = *flagsp &~ MSG_EOR; 2380 else 2381 flags = 0; 2382 2383 /* 2384 * For any complicated cases, fall back to the full 2385 * soreceive_generic(). 2386 */ 2387 if (mp0 != NULL || (flags & MSG_PEEK) || (flags & MSG_OOB)) 2388 return (soreceive_generic(so, psa, uio, mp0, controlp, 2389 flagsp)); 2390 2391 /* 2392 * Enforce restrictions on use. 2393 */ 2394 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0, 2395 ("soreceive_dgram: wantrcvd")); 2396 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic")); 2397 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0, 2398 ("soreceive_dgram: SBS_RCVATMARK")); 2399 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0, 2400 ("soreceive_dgram: P_CONNREQUIRED")); 2401 2402 /* 2403 * Loop blocking while waiting for a datagram. 2404 */ 2405 SOCKBUF_LOCK(&so->so_rcv); 2406 while ((m = so->so_rcv.sb_mb) == NULL) { 2407 KASSERT(sbavail(&so->so_rcv) == 0, 2408 ("soreceive_dgram: sb_mb NULL but sbavail %u", 2409 sbavail(&so->so_rcv))); 2410 if (so->so_error) { 2411 error = so->so_error; 2412 so->so_error = 0; 2413 SOCKBUF_UNLOCK(&so->so_rcv); 2414 return (error); 2415 } 2416 if (so->so_rcv.sb_state & SBS_CANTRCVMORE || 2417 uio->uio_resid == 0) { 2418 SOCKBUF_UNLOCK(&so->so_rcv); 2419 return (0); 2420 } 2421 if ((so->so_state & SS_NBIO) || 2422 (flags & (MSG_DONTWAIT|MSG_NBIO))) { 2423 SOCKBUF_UNLOCK(&so->so_rcv); 2424 return (EWOULDBLOCK); 2425 } 2426 SBLASTRECORDCHK(&so->so_rcv); 2427 SBLASTMBUFCHK(&so->so_rcv); 2428 error = sbwait(&so->so_rcv); 2429 if (error) { 2430 SOCKBUF_UNLOCK(&so->so_rcv); 2431 return (error); 2432 } 2433 } 2434 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 2435 2436 if (uio->uio_td) 2437 uio->uio_td->td_ru.ru_msgrcv++; 2438 SBLASTRECORDCHK(&so->so_rcv); 2439 SBLASTMBUFCHK(&so->so_rcv); 2440 nextrecord = m->m_nextpkt; 2441 if (nextrecord == NULL) { 2442 KASSERT(so->so_rcv.sb_lastrecord == m, 2443 ("soreceive_dgram: lastrecord != m")); 2444 } 2445 2446 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord, 2447 ("soreceive_dgram: m_nextpkt != nextrecord")); 2448 2449 /* 2450 * Pull 'm' and its chain off the front of the packet queue. 2451 */ 2452 so->so_rcv.sb_mb = NULL; 2453 sockbuf_pushsync(&so->so_rcv, nextrecord); 2454 2455 /* 2456 * Walk 'm's chain and free that many bytes from the socket buffer. 2457 */ 2458 for (m2 = m; m2 != NULL; m2 = m2->m_next) 2459 sbfree(&so->so_rcv, m2); 2460 2461 /* 2462 * Do a few last checks before we let go of the lock. 2463 */ 2464 SBLASTRECORDCHK(&so->so_rcv); 2465 SBLASTMBUFCHK(&so->so_rcv); 2466 SOCKBUF_UNLOCK(&so->so_rcv); 2467 2468 if (pr->pr_flags & PR_ADDR) { 2469 KASSERT(m->m_type == MT_SONAME, 2470 ("m->m_type == %d", m->m_type)); 2471 if (psa != NULL) 2472 *psa = sodupsockaddr(mtod(m, struct sockaddr *), 2473 M_NOWAIT); 2474 m = m_free(m); 2475 } 2476 if (m == NULL) { 2477 /* XXXRW: Can this happen? */ 2478 return (0); 2479 } 2480 2481 /* 2482 * Packet to copyout() is now in 'm' and it is disconnected from the 2483 * queue. 2484 * 2485 * Process one or more MT_CONTROL mbufs present before any data mbufs 2486 * in the first mbuf chain on the socket buffer. We call into the 2487 * protocol to perform externalization (or freeing if controlp == 2488 * NULL). In some cases there can be only MT_CONTROL mbufs without 2489 * MT_DATA mbufs. 2490 */ 2491 if (m->m_type == MT_CONTROL) { 2492 struct mbuf *cm = NULL, *cmn; 2493 struct mbuf **cme = &cm; 2494 2495 do { 2496 m2 = m->m_next; 2497 m->m_next = NULL; 2498 *cme = m; 2499 cme = &(*cme)->m_next; 2500 m = m2; 2501 } while (m != NULL && m->m_type == MT_CONTROL); 2502 while (cm != NULL) { 2503 cmn = cm->m_next; 2504 cm->m_next = NULL; 2505 if (pr->pr_domain->dom_externalize != NULL) { 2506 error = (*pr->pr_domain->dom_externalize) 2507 (cm, controlp, flags); 2508 } else if (controlp != NULL) 2509 *controlp = cm; 2510 else 2511 m_freem(cm); 2512 if (controlp != NULL) { 2513 while (*controlp != NULL) 2514 controlp = &(*controlp)->m_next; 2515 } 2516 cm = cmn; 2517 } 2518 } 2519 KASSERT(m == NULL || m->m_type == MT_DATA, 2520 ("soreceive_dgram: !data")); 2521 while (m != NULL && uio->uio_resid > 0) { 2522 len = uio->uio_resid; 2523 if (len > m->m_len) 2524 len = m->m_len; 2525 error = uiomove(mtod(m, char *), (int)len, uio); 2526 if (error) { 2527 m_freem(m); 2528 return (error); 2529 } 2530 if (len == m->m_len) 2531 m = m_free(m); 2532 else { 2533 m->m_data += len; 2534 m->m_len -= len; 2535 } 2536 } 2537 if (m != NULL) { 2538 flags |= MSG_TRUNC; 2539 m_freem(m); 2540 } 2541 if (flagsp != NULL) 2542 *flagsp |= flags; 2543 return (0); 2544 } 2545 2546 int 2547 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio, 2548 struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 2549 { 2550 int error; 2551 2552 CURVNET_SET(so->so_vnet); 2553 if (!SOLISTENING(so)) 2554 error = (so->so_proto->pr_usrreqs->pru_soreceive(so, psa, uio, 2555 mp0, controlp, flagsp)); 2556 else 2557 error = ENOTCONN; 2558 CURVNET_RESTORE(); 2559 return (error); 2560 } 2561 2562 int 2563 soshutdown(struct socket *so, int how) 2564 { 2565 struct protosw *pr = so->so_proto; 2566 int error, soerror_enotconn; 2567 2568 if (!(how == SHUT_RD || how == SHUT_WR || how == SHUT_RDWR)) 2569 return (EINVAL); 2570 2571 soerror_enotconn = 0; 2572 if ((so->so_state & 2573 (SS_ISCONNECTED | SS_ISCONNECTING | SS_ISDISCONNECTING)) == 0) { 2574 /* 2575 * POSIX mandates us to return ENOTCONN when shutdown(2) is 2576 * invoked on a datagram sockets, however historically we would 2577 * actually tear socket down. This is known to be leveraged by 2578 * some applications to unblock process waiting in recvXXX(2) 2579 * by other process that it shares that socket with. Try to meet 2580 * both backward-compatibility and POSIX requirements by forcing 2581 * ENOTCONN but still asking protocol to perform pru_shutdown(). 2582 */ 2583 if (so->so_type != SOCK_DGRAM) 2584 return (ENOTCONN); 2585 soerror_enotconn = 1; 2586 } 2587 2588 CURVNET_SET(so->so_vnet); 2589 if (pr->pr_usrreqs->pru_flush != NULL) 2590 (*pr->pr_usrreqs->pru_flush)(so, how); 2591 if (how != SHUT_WR) 2592 sorflush(so); 2593 if (how != SHUT_RD) { 2594 error = (*pr->pr_usrreqs->pru_shutdown)(so); 2595 wakeup(&so->so_timeo); 2596 CURVNET_RESTORE(); 2597 return ((error == 0 && soerror_enotconn) ? ENOTCONN : error); 2598 } 2599 wakeup(&so->so_timeo); 2600 CURVNET_RESTORE(); 2601 2602 return (soerror_enotconn ? ENOTCONN : 0); 2603 } 2604 2605 void 2606 sorflush(struct socket *so) 2607 { 2608 struct sockbuf *sb = &so->so_rcv; 2609 struct protosw *pr = so->so_proto; 2610 struct socket aso; 2611 2612 VNET_SO_ASSERT(so); 2613 2614 /* 2615 * In order to avoid calling dom_dispose with the socket buffer mutex 2616 * held, and in order to generally avoid holding the lock for a long 2617 * time, we make a copy of the socket buffer and clear the original 2618 * (except locks, state). The new socket buffer copy won't have 2619 * initialized locks so we can only call routines that won't use or 2620 * assert those locks. 2621 * 2622 * Dislodge threads currently blocked in receive and wait to acquire 2623 * a lock against other simultaneous readers before clearing the 2624 * socket buffer. Don't let our acquire be interrupted by a signal 2625 * despite any existing socket disposition on interruptable waiting. 2626 */ 2627 socantrcvmore(so); 2628 (void) sblock(sb, SBL_WAIT | SBL_NOINTR); 2629 2630 /* 2631 * Invalidate/clear most of the sockbuf structure, but leave selinfo 2632 * and mutex data unchanged. 2633 */ 2634 SOCKBUF_LOCK(sb); 2635 bzero(&aso, sizeof(aso)); 2636 aso.so_pcb = so->so_pcb; 2637 bcopy(&sb->sb_startzero, &aso.so_rcv.sb_startzero, 2638 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2639 bzero(&sb->sb_startzero, 2640 sizeof(*sb) - offsetof(struct sockbuf, sb_startzero)); 2641 SOCKBUF_UNLOCK(sb); 2642 sbunlock(sb); 2643 2644 /* 2645 * Dispose of special rights and flush the copied socket. Don't call 2646 * any unsafe routines (that rely on locks being initialized) on aso. 2647 */ 2648 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose != NULL) 2649 (*pr->pr_domain->dom_dispose)(&aso); 2650 sbrelease_internal(&aso.so_rcv, so); 2651 } 2652 2653 /* 2654 * Wrapper for Socket established helper hook. 2655 * Parameters: socket, context of the hook point, hook id. 2656 */ 2657 static int inline 2658 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id) 2659 { 2660 struct socket_hhook_data hhook_data = { 2661 .so = so, 2662 .hctx = hctx, 2663 .m = NULL, 2664 .status = 0 2665 }; 2666 2667 CURVNET_SET(so->so_vnet); 2668 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd); 2669 CURVNET_RESTORE(); 2670 2671 /* Ugly but needed, since hhooks return void for now */ 2672 return (hhook_data.status); 2673 } 2674 2675 /* 2676 * Perhaps this routine, and sooptcopyout(), below, ought to come in an 2677 * additional variant to handle the case where the option value needs to be 2678 * some kind of integer, but not a specific size. In addition to their use 2679 * here, these functions are also called by the protocol-level pr_ctloutput() 2680 * routines. 2681 */ 2682 int 2683 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen) 2684 { 2685 size_t valsize; 2686 2687 /* 2688 * If the user gives us more than we wanted, we ignore it, but if we 2689 * don't get the minimum length the caller wants, we return EINVAL. 2690 * On success, sopt->sopt_valsize is set to however much we actually 2691 * retrieved. 2692 */ 2693 if ((valsize = sopt->sopt_valsize) < minlen) 2694 return EINVAL; 2695 if (valsize > len) 2696 sopt->sopt_valsize = valsize = len; 2697 2698 if (sopt->sopt_td != NULL) 2699 return (copyin(sopt->sopt_val, buf, valsize)); 2700 2701 bcopy(sopt->sopt_val, buf, valsize); 2702 return (0); 2703 } 2704 2705 /* 2706 * Kernel version of setsockopt(2). 2707 * 2708 * XXX: optlen is size_t, not socklen_t 2709 */ 2710 int 2711 so_setsockopt(struct socket *so, int level, int optname, void *optval, 2712 size_t optlen) 2713 { 2714 struct sockopt sopt; 2715 2716 sopt.sopt_level = level; 2717 sopt.sopt_name = optname; 2718 sopt.sopt_dir = SOPT_SET; 2719 sopt.sopt_val = optval; 2720 sopt.sopt_valsize = optlen; 2721 sopt.sopt_td = NULL; 2722 return (sosetopt(so, &sopt)); 2723 } 2724 2725 int 2726 sosetopt(struct socket *so, struct sockopt *sopt) 2727 { 2728 int error, optval; 2729 struct linger l; 2730 struct timeval tv; 2731 sbintime_t val; 2732 uint32_t val32; 2733 #ifdef MAC 2734 struct mac extmac; 2735 #endif 2736 2737 CURVNET_SET(so->so_vnet); 2738 error = 0; 2739 if (sopt->sopt_level != SOL_SOCKET) { 2740 if (so->so_proto->pr_ctloutput != NULL) { 2741 error = (*so->so_proto->pr_ctloutput)(so, sopt); 2742 CURVNET_RESTORE(); 2743 return (error); 2744 } 2745 error = ENOPROTOOPT; 2746 } else { 2747 switch (sopt->sopt_name) { 2748 case SO_ACCEPTFILTER: 2749 error = accept_filt_setopt(so, sopt); 2750 if (error) 2751 goto bad; 2752 break; 2753 2754 case SO_LINGER: 2755 error = sooptcopyin(sopt, &l, sizeof l, sizeof l); 2756 if (error) 2757 goto bad; 2758 2759 SOCK_LOCK(so); 2760 so->so_linger = l.l_linger; 2761 if (l.l_onoff) 2762 so->so_options |= SO_LINGER; 2763 else 2764 so->so_options &= ~SO_LINGER; 2765 SOCK_UNLOCK(so); 2766 break; 2767 2768 case SO_DEBUG: 2769 case SO_KEEPALIVE: 2770 case SO_DONTROUTE: 2771 case SO_USELOOPBACK: 2772 case SO_BROADCAST: 2773 case SO_REUSEADDR: 2774 case SO_REUSEPORT: 2775 case SO_OOBINLINE: 2776 case SO_TIMESTAMP: 2777 case SO_BINTIME: 2778 case SO_NOSIGPIPE: 2779 case SO_NO_DDP: 2780 case SO_NO_OFFLOAD: 2781 error = sooptcopyin(sopt, &optval, sizeof optval, 2782 sizeof optval); 2783 if (error) 2784 goto bad; 2785 SOCK_LOCK(so); 2786 if (optval) 2787 so->so_options |= sopt->sopt_name; 2788 else 2789 so->so_options &= ~sopt->sopt_name; 2790 SOCK_UNLOCK(so); 2791 break; 2792 2793 case SO_SETFIB: 2794 error = sooptcopyin(sopt, &optval, sizeof optval, 2795 sizeof optval); 2796 if (error) 2797 goto bad; 2798 2799 if (optval < 0 || optval >= rt_numfibs) { 2800 error = EINVAL; 2801 goto bad; 2802 } 2803 if (((so->so_proto->pr_domain->dom_family == PF_INET) || 2804 (so->so_proto->pr_domain->dom_family == PF_INET6) || 2805 (so->so_proto->pr_domain->dom_family == PF_ROUTE))) 2806 so->so_fibnum = optval; 2807 else 2808 so->so_fibnum = 0; 2809 break; 2810 2811 case SO_USER_COOKIE: 2812 error = sooptcopyin(sopt, &val32, sizeof val32, 2813 sizeof val32); 2814 if (error) 2815 goto bad; 2816 so->so_user_cookie = val32; 2817 break; 2818 2819 case SO_SNDBUF: 2820 case SO_RCVBUF: 2821 case SO_SNDLOWAT: 2822 case SO_RCVLOWAT: 2823 error = sooptcopyin(sopt, &optval, sizeof optval, 2824 sizeof optval); 2825 if (error) 2826 goto bad; 2827 2828 /* 2829 * Values < 1 make no sense for any of these options, 2830 * so disallow them. 2831 */ 2832 if (optval < 1) { 2833 error = EINVAL; 2834 goto bad; 2835 } 2836 2837 switch (sopt->sopt_name) { 2838 case SO_SNDBUF: 2839 case SO_RCVBUF: 2840 if (sbreserve(sopt->sopt_name == SO_SNDBUF ? 2841 &so->so_snd : &so->so_rcv, (u_long)optval, 2842 so, curthread) == 0) { 2843 error = ENOBUFS; 2844 goto bad; 2845 } 2846 (sopt->sopt_name == SO_SNDBUF ? &so->so_snd : 2847 &so->so_rcv)->sb_flags &= ~SB_AUTOSIZE; 2848 break; 2849 2850 /* 2851 * Make sure the low-water is never greater than the 2852 * high-water. 2853 */ 2854 case SO_SNDLOWAT: 2855 SOCKBUF_LOCK(&so->so_snd); 2856 so->so_snd.sb_lowat = 2857 (optval > so->so_snd.sb_hiwat) ? 2858 so->so_snd.sb_hiwat : optval; 2859 SOCKBUF_UNLOCK(&so->so_snd); 2860 break; 2861 case SO_RCVLOWAT: 2862 SOCKBUF_LOCK(&so->so_rcv); 2863 so->so_rcv.sb_lowat = 2864 (optval > so->so_rcv.sb_hiwat) ? 2865 so->so_rcv.sb_hiwat : optval; 2866 SOCKBUF_UNLOCK(&so->so_rcv); 2867 break; 2868 } 2869 break; 2870 2871 case SO_SNDTIMEO: 2872 case SO_RCVTIMEO: 2873 #ifdef COMPAT_FREEBSD32 2874 if (SV_CURPROC_FLAG(SV_ILP32)) { 2875 struct timeval32 tv32; 2876 2877 error = sooptcopyin(sopt, &tv32, sizeof tv32, 2878 sizeof tv32); 2879 CP(tv32, tv, tv_sec); 2880 CP(tv32, tv, tv_usec); 2881 } else 2882 #endif 2883 error = sooptcopyin(sopt, &tv, sizeof tv, 2884 sizeof tv); 2885 if (error) 2886 goto bad; 2887 if (tv.tv_sec < 0 || tv.tv_usec < 0 || 2888 tv.tv_usec >= 1000000) { 2889 error = EDOM; 2890 goto bad; 2891 } 2892 if (tv.tv_sec > INT32_MAX) 2893 val = SBT_MAX; 2894 else 2895 val = tvtosbt(tv); 2896 switch (sopt->sopt_name) { 2897 case SO_SNDTIMEO: 2898 so->so_snd.sb_timeo = val; 2899 break; 2900 case SO_RCVTIMEO: 2901 so->so_rcv.sb_timeo = val; 2902 break; 2903 } 2904 break; 2905 2906 case SO_LABEL: 2907 #ifdef MAC 2908 error = sooptcopyin(sopt, &extmac, sizeof extmac, 2909 sizeof extmac); 2910 if (error) 2911 goto bad; 2912 error = mac_setsockopt_label(sopt->sopt_td->td_ucred, 2913 so, &extmac); 2914 #else 2915 error = EOPNOTSUPP; 2916 #endif 2917 break; 2918 2919 case SO_TS_CLOCK: 2920 error = sooptcopyin(sopt, &optval, sizeof optval, 2921 sizeof optval); 2922 if (error) 2923 goto bad; 2924 if (optval < 0 || optval > SO_TS_CLOCK_MAX) { 2925 error = EINVAL; 2926 goto bad; 2927 } 2928 so->so_ts_clock = optval; 2929 break; 2930 2931 case SO_MAX_PACING_RATE: 2932 error = sooptcopyin(sopt, &val32, sizeof(val32), 2933 sizeof(val32)); 2934 if (error) 2935 goto bad; 2936 so->so_max_pacing_rate = val32; 2937 break; 2938 2939 default: 2940 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 2941 error = hhook_run_socket(so, sopt, 2942 HHOOK_SOCKET_OPT); 2943 else 2944 error = ENOPROTOOPT; 2945 break; 2946 } 2947 if (error == 0 && so->so_proto->pr_ctloutput != NULL) 2948 (void)(*so->so_proto->pr_ctloutput)(so, sopt); 2949 } 2950 bad: 2951 CURVNET_RESTORE(); 2952 return (error); 2953 } 2954 2955 /* 2956 * Helper routine for getsockopt. 2957 */ 2958 int 2959 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len) 2960 { 2961 int error; 2962 size_t valsize; 2963 2964 error = 0; 2965 2966 /* 2967 * Documented get behavior is that we always return a value, possibly 2968 * truncated to fit in the user's buffer. Traditional behavior is 2969 * that we always tell the user precisely how much we copied, rather 2970 * than something useful like the total amount we had available for 2971 * her. Note that this interface is not idempotent; the entire 2972 * answer must be generated ahead of time. 2973 */ 2974 valsize = min(len, sopt->sopt_valsize); 2975 sopt->sopt_valsize = valsize; 2976 if (sopt->sopt_val != NULL) { 2977 if (sopt->sopt_td != NULL) 2978 error = copyout(buf, sopt->sopt_val, valsize); 2979 else 2980 bcopy(buf, sopt->sopt_val, valsize); 2981 } 2982 return (error); 2983 } 2984 2985 int 2986 sogetopt(struct socket *so, struct sockopt *sopt) 2987 { 2988 int error, optval; 2989 struct linger l; 2990 struct timeval tv; 2991 #ifdef MAC 2992 struct mac extmac; 2993 #endif 2994 2995 CURVNET_SET(so->so_vnet); 2996 error = 0; 2997 if (sopt->sopt_level != SOL_SOCKET) { 2998 if (so->so_proto->pr_ctloutput != NULL) 2999 error = (*so->so_proto->pr_ctloutput)(so, sopt); 3000 else 3001 error = ENOPROTOOPT; 3002 CURVNET_RESTORE(); 3003 return (error); 3004 } else { 3005 switch (sopt->sopt_name) { 3006 case SO_ACCEPTFILTER: 3007 error = accept_filt_getopt(so, sopt); 3008 break; 3009 3010 case SO_LINGER: 3011 SOCK_LOCK(so); 3012 l.l_onoff = so->so_options & SO_LINGER; 3013 l.l_linger = so->so_linger; 3014 SOCK_UNLOCK(so); 3015 error = sooptcopyout(sopt, &l, sizeof l); 3016 break; 3017 3018 case SO_USELOOPBACK: 3019 case SO_DONTROUTE: 3020 case SO_DEBUG: 3021 case SO_KEEPALIVE: 3022 case SO_REUSEADDR: 3023 case SO_REUSEPORT: 3024 case SO_BROADCAST: 3025 case SO_OOBINLINE: 3026 case SO_ACCEPTCONN: 3027 case SO_TIMESTAMP: 3028 case SO_BINTIME: 3029 case SO_NOSIGPIPE: 3030 optval = so->so_options & sopt->sopt_name; 3031 integer: 3032 error = sooptcopyout(sopt, &optval, sizeof optval); 3033 break; 3034 3035 case SO_TYPE: 3036 optval = so->so_type; 3037 goto integer; 3038 3039 case SO_PROTOCOL: 3040 optval = so->so_proto->pr_protocol; 3041 goto integer; 3042 3043 case SO_ERROR: 3044 SOCK_LOCK(so); 3045 optval = so->so_error; 3046 so->so_error = 0; 3047 SOCK_UNLOCK(so); 3048 goto integer; 3049 3050 case SO_SNDBUF: 3051 optval = so->so_snd.sb_hiwat; 3052 goto integer; 3053 3054 case SO_RCVBUF: 3055 optval = so->so_rcv.sb_hiwat; 3056 goto integer; 3057 3058 case SO_SNDLOWAT: 3059 optval = so->so_snd.sb_lowat; 3060 goto integer; 3061 3062 case SO_RCVLOWAT: 3063 optval = so->so_rcv.sb_lowat; 3064 goto integer; 3065 3066 case SO_SNDTIMEO: 3067 case SO_RCVTIMEO: 3068 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ? 3069 so->so_snd.sb_timeo : so->so_rcv.sb_timeo); 3070 #ifdef COMPAT_FREEBSD32 3071 if (SV_CURPROC_FLAG(SV_ILP32)) { 3072 struct timeval32 tv32; 3073 3074 CP(tv, tv32, tv_sec); 3075 CP(tv, tv32, tv_usec); 3076 error = sooptcopyout(sopt, &tv32, sizeof tv32); 3077 } else 3078 #endif 3079 error = sooptcopyout(sopt, &tv, sizeof tv); 3080 break; 3081 3082 case SO_LABEL: 3083 #ifdef MAC 3084 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3085 sizeof(extmac)); 3086 if (error) 3087 goto bad; 3088 error = mac_getsockopt_label(sopt->sopt_td->td_ucred, 3089 so, &extmac); 3090 if (error) 3091 goto bad; 3092 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3093 #else 3094 error = EOPNOTSUPP; 3095 #endif 3096 break; 3097 3098 case SO_PEERLABEL: 3099 #ifdef MAC 3100 error = sooptcopyin(sopt, &extmac, sizeof(extmac), 3101 sizeof(extmac)); 3102 if (error) 3103 goto bad; 3104 error = mac_getsockopt_peerlabel( 3105 sopt->sopt_td->td_ucred, so, &extmac); 3106 if (error) 3107 goto bad; 3108 error = sooptcopyout(sopt, &extmac, sizeof extmac); 3109 #else 3110 error = EOPNOTSUPP; 3111 #endif 3112 break; 3113 3114 case SO_LISTENQLIMIT: 3115 optval = SOLISTENING(so) ? so->sol_qlimit : 0; 3116 goto integer; 3117 3118 case SO_LISTENQLEN: 3119 optval = SOLISTENING(so) ? so->sol_qlen : 0; 3120 goto integer; 3121 3122 case SO_LISTENINCQLEN: 3123 optval = SOLISTENING(so) ? so->sol_incqlen : 0; 3124 goto integer; 3125 3126 case SO_TS_CLOCK: 3127 optval = so->so_ts_clock; 3128 goto integer; 3129 3130 case SO_MAX_PACING_RATE: 3131 optval = so->so_max_pacing_rate; 3132 goto integer; 3133 3134 default: 3135 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0) 3136 error = hhook_run_socket(so, sopt, 3137 HHOOK_SOCKET_OPT); 3138 else 3139 error = ENOPROTOOPT; 3140 break; 3141 } 3142 } 3143 #ifdef MAC 3144 bad: 3145 #endif 3146 CURVNET_RESTORE(); 3147 return (error); 3148 } 3149 3150 int 3151 soopt_getm(struct sockopt *sopt, struct mbuf **mp) 3152 { 3153 struct mbuf *m, *m_prev; 3154 int sopt_size = sopt->sopt_valsize; 3155 3156 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3157 if (m == NULL) 3158 return ENOBUFS; 3159 if (sopt_size > MLEN) { 3160 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT); 3161 if ((m->m_flags & M_EXT) == 0) { 3162 m_free(m); 3163 return ENOBUFS; 3164 } 3165 m->m_len = min(MCLBYTES, sopt_size); 3166 } else { 3167 m->m_len = min(MLEN, sopt_size); 3168 } 3169 sopt_size -= m->m_len; 3170 *mp = m; 3171 m_prev = m; 3172 3173 while (sopt_size) { 3174 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA); 3175 if (m == NULL) { 3176 m_freem(*mp); 3177 return ENOBUFS; 3178 } 3179 if (sopt_size > MLEN) { 3180 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK : 3181 M_NOWAIT); 3182 if ((m->m_flags & M_EXT) == 0) { 3183 m_freem(m); 3184 m_freem(*mp); 3185 return ENOBUFS; 3186 } 3187 m->m_len = min(MCLBYTES, sopt_size); 3188 } else { 3189 m->m_len = min(MLEN, sopt_size); 3190 } 3191 sopt_size -= m->m_len; 3192 m_prev->m_next = m; 3193 m_prev = m; 3194 } 3195 return (0); 3196 } 3197 3198 int 3199 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m) 3200 { 3201 struct mbuf *m0 = m; 3202 3203 if (sopt->sopt_val == NULL) 3204 return (0); 3205 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3206 if (sopt->sopt_td != NULL) { 3207 int error; 3208 3209 error = copyin(sopt->sopt_val, mtod(m, char *), 3210 m->m_len); 3211 if (error != 0) { 3212 m_freem(m0); 3213 return(error); 3214 } 3215 } else 3216 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len); 3217 sopt->sopt_valsize -= m->m_len; 3218 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3219 m = m->m_next; 3220 } 3221 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */ 3222 panic("ip6_sooptmcopyin"); 3223 return (0); 3224 } 3225 3226 int 3227 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m) 3228 { 3229 struct mbuf *m0 = m; 3230 size_t valsize = 0; 3231 3232 if (sopt->sopt_val == NULL) 3233 return (0); 3234 while (m != NULL && sopt->sopt_valsize >= m->m_len) { 3235 if (sopt->sopt_td != NULL) { 3236 int error; 3237 3238 error = copyout(mtod(m, char *), sopt->sopt_val, 3239 m->m_len); 3240 if (error != 0) { 3241 m_freem(m0); 3242 return(error); 3243 } 3244 } else 3245 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len); 3246 sopt->sopt_valsize -= m->m_len; 3247 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len; 3248 valsize += m->m_len; 3249 m = m->m_next; 3250 } 3251 if (m != NULL) { 3252 /* enough soopt buffer should be given from user-land */ 3253 m_freem(m0); 3254 return(EINVAL); 3255 } 3256 sopt->sopt_valsize = valsize; 3257 return (0); 3258 } 3259 3260 /* 3261 * sohasoutofband(): protocol notifies socket layer of the arrival of new 3262 * out-of-band data, which will then notify socket consumers. 3263 */ 3264 void 3265 sohasoutofband(struct socket *so) 3266 { 3267 3268 if (so->so_sigio != NULL) 3269 pgsigio(&so->so_sigio, SIGURG, 0); 3270 selwakeuppri(&so->so_rdsel, PSOCK); 3271 } 3272 3273 int 3274 sopoll(struct socket *so, int events, struct ucred *active_cred, 3275 struct thread *td) 3276 { 3277 3278 /* 3279 * We do not need to set or assert curvnet as long as everyone uses 3280 * sopoll_generic(). 3281 */ 3282 return (so->so_proto->pr_usrreqs->pru_sopoll(so, events, active_cred, 3283 td)); 3284 } 3285 3286 int 3287 sopoll_generic(struct socket *so, int events, struct ucred *active_cred, 3288 struct thread *td) 3289 { 3290 int revents; 3291 3292 SOCK_LOCK(so); 3293 if (SOLISTENING(so)) { 3294 if (!(events & (POLLIN | POLLRDNORM))) 3295 revents = 0; 3296 else if (!TAILQ_EMPTY(&so->sol_comp)) 3297 revents = events & (POLLIN | POLLRDNORM); 3298 else { 3299 selrecord(td, &so->so_rdsel); 3300 revents = 0; 3301 } 3302 } else { 3303 revents = 0; 3304 SOCKBUF_LOCK(&so->so_snd); 3305 SOCKBUF_LOCK(&so->so_rcv); 3306 if (events & (POLLIN | POLLRDNORM)) 3307 if (soreadabledata(so)) 3308 revents |= events & (POLLIN | POLLRDNORM); 3309 if (events & (POLLOUT | POLLWRNORM)) 3310 if (sowriteable(so)) 3311 revents |= events & (POLLOUT | POLLWRNORM); 3312 if (events & (POLLPRI | POLLRDBAND)) 3313 if (so->so_oobmark || 3314 (so->so_rcv.sb_state & SBS_RCVATMARK)) 3315 revents |= events & (POLLPRI | POLLRDBAND); 3316 if ((events & POLLINIGNEOF) == 0) { 3317 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3318 revents |= events & (POLLIN | POLLRDNORM); 3319 if (so->so_snd.sb_state & SBS_CANTSENDMORE) 3320 revents |= POLLHUP; 3321 } 3322 } 3323 if (revents == 0) { 3324 if (events & 3325 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND)) { 3326 selrecord(td, &so->so_rdsel); 3327 so->so_rcv.sb_flags |= SB_SEL; 3328 } 3329 if (events & (POLLOUT | POLLWRNORM)) { 3330 selrecord(td, &so->so_wrsel); 3331 so->so_snd.sb_flags |= SB_SEL; 3332 } 3333 } 3334 SOCKBUF_UNLOCK(&so->so_rcv); 3335 SOCKBUF_UNLOCK(&so->so_snd); 3336 } 3337 SOCK_UNLOCK(so); 3338 return (revents); 3339 } 3340 3341 int 3342 soo_kqfilter(struct file *fp, struct knote *kn) 3343 { 3344 struct socket *so = kn->kn_fp->f_data; 3345 struct sockbuf *sb; 3346 struct knlist *knl; 3347 3348 switch (kn->kn_filter) { 3349 case EVFILT_READ: 3350 kn->kn_fop = &soread_filtops; 3351 knl = &so->so_rdsel.si_note; 3352 sb = &so->so_rcv; 3353 break; 3354 case EVFILT_WRITE: 3355 kn->kn_fop = &sowrite_filtops; 3356 knl = &so->so_wrsel.si_note; 3357 sb = &so->so_snd; 3358 break; 3359 case EVFILT_EMPTY: 3360 kn->kn_fop = &soempty_filtops; 3361 knl = &so->so_wrsel.si_note; 3362 sb = &so->so_snd; 3363 break; 3364 default: 3365 return (EINVAL); 3366 } 3367 3368 SOCK_LOCK(so); 3369 if (SOLISTENING(so)) { 3370 knlist_add(knl, kn, 1); 3371 } else { 3372 SOCKBUF_LOCK(sb); 3373 knlist_add(knl, kn, 1); 3374 sb->sb_flags |= SB_KNOTE; 3375 SOCKBUF_UNLOCK(sb); 3376 } 3377 SOCK_UNLOCK(so); 3378 return (0); 3379 } 3380 3381 /* 3382 * Some routines that return EOPNOTSUPP for entry points that are not 3383 * supported by a protocol. Fill in as needed. 3384 */ 3385 int 3386 pru_accept_notsupp(struct socket *so, struct sockaddr **nam) 3387 { 3388 3389 return EOPNOTSUPP; 3390 } 3391 3392 int 3393 pru_aio_queue_notsupp(struct socket *so, struct kaiocb *job) 3394 { 3395 3396 return EOPNOTSUPP; 3397 } 3398 3399 int 3400 pru_attach_notsupp(struct socket *so, int proto, struct thread *td) 3401 { 3402 3403 return EOPNOTSUPP; 3404 } 3405 3406 int 3407 pru_bind_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3408 { 3409 3410 return EOPNOTSUPP; 3411 } 3412 3413 int 3414 pru_bindat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3415 struct thread *td) 3416 { 3417 3418 return EOPNOTSUPP; 3419 } 3420 3421 int 3422 pru_connect_notsupp(struct socket *so, struct sockaddr *nam, struct thread *td) 3423 { 3424 3425 return EOPNOTSUPP; 3426 } 3427 3428 int 3429 pru_connectat_notsupp(int fd, struct socket *so, struct sockaddr *nam, 3430 struct thread *td) 3431 { 3432 3433 return EOPNOTSUPP; 3434 } 3435 3436 int 3437 pru_connect2_notsupp(struct socket *so1, struct socket *so2) 3438 { 3439 3440 return EOPNOTSUPP; 3441 } 3442 3443 int 3444 pru_control_notsupp(struct socket *so, u_long cmd, caddr_t data, 3445 struct ifnet *ifp, struct thread *td) 3446 { 3447 3448 return EOPNOTSUPP; 3449 } 3450 3451 int 3452 pru_disconnect_notsupp(struct socket *so) 3453 { 3454 3455 return EOPNOTSUPP; 3456 } 3457 3458 int 3459 pru_listen_notsupp(struct socket *so, int backlog, struct thread *td) 3460 { 3461 3462 return EOPNOTSUPP; 3463 } 3464 3465 int 3466 pru_peeraddr_notsupp(struct socket *so, struct sockaddr **nam) 3467 { 3468 3469 return EOPNOTSUPP; 3470 } 3471 3472 int 3473 pru_rcvd_notsupp(struct socket *so, int flags) 3474 { 3475 3476 return EOPNOTSUPP; 3477 } 3478 3479 int 3480 pru_rcvoob_notsupp(struct socket *so, struct mbuf *m, int flags) 3481 { 3482 3483 return EOPNOTSUPP; 3484 } 3485 3486 int 3487 pru_send_notsupp(struct socket *so, int flags, struct mbuf *m, 3488 struct sockaddr *addr, struct mbuf *control, struct thread *td) 3489 { 3490 3491 return EOPNOTSUPP; 3492 } 3493 3494 int 3495 pru_ready_notsupp(struct socket *so, struct mbuf *m, int count) 3496 { 3497 3498 return (EOPNOTSUPP); 3499 } 3500 3501 /* 3502 * This isn't really a ``null'' operation, but it's the default one and 3503 * doesn't do anything destructive. 3504 */ 3505 int 3506 pru_sense_null(struct socket *so, struct stat *sb) 3507 { 3508 3509 sb->st_blksize = so->so_snd.sb_hiwat; 3510 return 0; 3511 } 3512 3513 int 3514 pru_shutdown_notsupp(struct socket *so) 3515 { 3516 3517 return EOPNOTSUPP; 3518 } 3519 3520 int 3521 pru_sockaddr_notsupp(struct socket *so, struct sockaddr **nam) 3522 { 3523 3524 return EOPNOTSUPP; 3525 } 3526 3527 int 3528 pru_sosend_notsupp(struct socket *so, struct sockaddr *addr, struct uio *uio, 3529 struct mbuf *top, struct mbuf *control, int flags, struct thread *td) 3530 { 3531 3532 return EOPNOTSUPP; 3533 } 3534 3535 int 3536 pru_soreceive_notsupp(struct socket *so, struct sockaddr **paddr, 3537 struct uio *uio, struct mbuf **mp0, struct mbuf **controlp, int *flagsp) 3538 { 3539 3540 return EOPNOTSUPP; 3541 } 3542 3543 int 3544 pru_sopoll_notsupp(struct socket *so, int events, struct ucred *cred, 3545 struct thread *td) 3546 { 3547 3548 return EOPNOTSUPP; 3549 } 3550 3551 static void 3552 filt_sordetach(struct knote *kn) 3553 { 3554 struct socket *so = kn->kn_fp->f_data; 3555 3556 so_rdknl_lock(so); 3557 knlist_remove(&so->so_rdsel.si_note, kn, 1); 3558 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note)) 3559 so->so_rcv.sb_flags &= ~SB_KNOTE; 3560 so_rdknl_unlock(so); 3561 } 3562 3563 /*ARGSUSED*/ 3564 static int 3565 filt_soread(struct knote *kn, long hint) 3566 { 3567 struct socket *so; 3568 3569 so = kn->kn_fp->f_data; 3570 3571 if (SOLISTENING(so)) { 3572 SOCK_LOCK_ASSERT(so); 3573 kn->kn_data = so->sol_qlen; 3574 return (!TAILQ_EMPTY(&so->sol_comp)); 3575 } 3576 3577 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3578 3579 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl; 3580 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) { 3581 kn->kn_flags |= EV_EOF; 3582 kn->kn_fflags = so->so_error; 3583 return (1); 3584 } else if (so->so_error) /* temporary udp error */ 3585 return (1); 3586 3587 if (kn->kn_sfflags & NOTE_LOWAT) { 3588 if (kn->kn_data >= kn->kn_sdata) 3589 return (1); 3590 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat) 3591 return (1); 3592 3593 /* This hook returning non-zero indicates an event, not error */ 3594 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD)); 3595 } 3596 3597 static void 3598 filt_sowdetach(struct knote *kn) 3599 { 3600 struct socket *so = kn->kn_fp->f_data; 3601 3602 so_wrknl_lock(so); 3603 knlist_remove(&so->so_wrsel.si_note, kn, 1); 3604 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note)) 3605 so->so_snd.sb_flags &= ~SB_KNOTE; 3606 so_wrknl_unlock(so); 3607 } 3608 3609 /*ARGSUSED*/ 3610 static int 3611 filt_sowrite(struct knote *kn, long hint) 3612 { 3613 struct socket *so; 3614 3615 so = kn->kn_fp->f_data; 3616 3617 if (SOLISTENING(so)) 3618 return (0); 3619 3620 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3621 kn->kn_data = sbspace(&so->so_snd); 3622 3623 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE); 3624 3625 if (so->so_snd.sb_state & SBS_CANTSENDMORE) { 3626 kn->kn_flags |= EV_EOF; 3627 kn->kn_fflags = so->so_error; 3628 return (1); 3629 } else if (so->so_error) /* temporary udp error */ 3630 return (1); 3631 else if (((so->so_state & SS_ISCONNECTED) == 0) && 3632 (so->so_proto->pr_flags & PR_CONNREQUIRED)) 3633 return (0); 3634 else if (kn->kn_sfflags & NOTE_LOWAT) 3635 return (kn->kn_data >= kn->kn_sdata); 3636 else 3637 return (kn->kn_data >= so->so_snd.sb_lowat); 3638 } 3639 3640 static int 3641 filt_soempty(struct knote *kn, long hint) 3642 { 3643 struct socket *so; 3644 3645 so = kn->kn_fp->f_data; 3646 3647 if (SOLISTENING(so)) 3648 return (1); 3649 3650 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3651 kn->kn_data = sbused(&so->so_snd); 3652 3653 if (kn->kn_data == 0) 3654 return (1); 3655 else 3656 return (0); 3657 } 3658 3659 int 3660 socheckuid(struct socket *so, uid_t uid) 3661 { 3662 3663 if (so == NULL) 3664 return (EPERM); 3665 if (so->so_cred->cr_uid != uid) 3666 return (EPERM); 3667 return (0); 3668 } 3669 3670 /* 3671 * These functions are used by protocols to notify the socket layer (and its 3672 * consumers) of state changes in the sockets driven by protocol-side events. 3673 */ 3674 3675 /* 3676 * Procedures to manipulate state flags of socket and do appropriate wakeups. 3677 * 3678 * Normal sequence from the active (originating) side is that 3679 * soisconnecting() is called during processing of connect() call, resulting 3680 * in an eventual call to soisconnected() if/when the connection is 3681 * established. When the connection is torn down soisdisconnecting() is 3682 * called during processing of disconnect() call, and soisdisconnected() is 3683 * called when the connection to the peer is totally severed. The semantics 3684 * of these routines are such that connectionless protocols can call 3685 * soisconnected() and soisdisconnected() only, bypassing the in-progress 3686 * calls when setting up a ``connection'' takes no time. 3687 * 3688 * From the passive side, a socket is created with two queues of sockets: 3689 * so_incomp for connections in progress and so_comp for connections already 3690 * made and awaiting user acceptance. As a protocol is preparing incoming 3691 * connections, it creates a socket structure queued on so_incomp by calling 3692 * sonewconn(). When the connection is established, soisconnected() is 3693 * called, and transfers the socket structure to so_comp, making it available 3694 * to accept(). 3695 * 3696 * If a socket is closed with sockets on either so_incomp or so_comp, these 3697 * sockets are dropped. 3698 * 3699 * If higher-level protocols are implemented in the kernel, the wakeups done 3700 * here will sometimes cause software-interrupt process scheduling. 3701 */ 3702 void 3703 soisconnecting(struct socket *so) 3704 { 3705 3706 SOCK_LOCK(so); 3707 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING); 3708 so->so_state |= SS_ISCONNECTING; 3709 SOCK_UNLOCK(so); 3710 } 3711 3712 void 3713 soisconnected(struct socket *so) 3714 { 3715 struct socket *head; 3716 int ret; 3717 3718 /* 3719 * XXXGL: this is the only place where we acquire socket locks 3720 * in reverse order: first child, then listening socket. To 3721 * avoid possible LOR, use try semantics. 3722 */ 3723 restart: 3724 SOCK_LOCK(so); 3725 if ((head = so->so_listen) != NULL && 3726 __predict_false(SOLISTEN_TRYLOCK(head) == 0)) { 3727 SOCK_UNLOCK(so); 3728 goto restart; 3729 } 3730 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING|SS_ISCONFIRMING); 3731 so->so_state |= SS_ISCONNECTED; 3732 if (head != NULL && (so->so_qstate == SQ_INCOMP)) { 3733 again: 3734 if ((so->so_options & SO_ACCEPTFILTER) == 0) { 3735 TAILQ_REMOVE(&head->sol_incomp, so, so_list); 3736 head->sol_incqlen--; 3737 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list); 3738 head->sol_qlen++; 3739 so->so_qstate = SQ_COMP; 3740 SOCK_UNLOCK(so); 3741 solisten_wakeup(head); /* unlocks */ 3742 } else { 3743 SOCKBUF_LOCK(&so->so_rcv); 3744 soupcall_set(so, SO_RCV, 3745 head->sol_accept_filter->accf_callback, 3746 head->sol_accept_filter_arg); 3747 so->so_options &= ~SO_ACCEPTFILTER; 3748 ret = head->sol_accept_filter->accf_callback(so, 3749 head->sol_accept_filter_arg, M_NOWAIT); 3750 if (ret == SU_ISCONNECTED) { 3751 soupcall_clear(so, SO_RCV); 3752 SOCKBUF_UNLOCK(&so->so_rcv); 3753 goto again; 3754 } 3755 SOCKBUF_UNLOCK(&so->so_rcv); 3756 SOCK_UNLOCK(so); 3757 SOLISTEN_UNLOCK(head); 3758 } 3759 return; 3760 } 3761 if (head != NULL) 3762 SOLISTEN_UNLOCK(head); 3763 SOCK_UNLOCK(so); 3764 wakeup(&so->so_timeo); 3765 sorwakeup(so); 3766 sowwakeup(so); 3767 } 3768 3769 void 3770 soisdisconnecting(struct socket *so) 3771 { 3772 3773 SOCK_LOCK(so); 3774 so->so_state &= ~SS_ISCONNECTING; 3775 so->so_state |= SS_ISDISCONNECTING; 3776 3777 if (!SOLISTENING(so)) { 3778 SOCKBUF_LOCK(&so->so_rcv); 3779 socantrcvmore_locked(so); 3780 SOCKBUF_LOCK(&so->so_snd); 3781 socantsendmore_locked(so); 3782 } 3783 SOCK_UNLOCK(so); 3784 wakeup(&so->so_timeo); 3785 } 3786 3787 void 3788 soisdisconnected(struct socket *so) 3789 { 3790 3791 SOCK_LOCK(so); 3792 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING); 3793 so->so_state |= SS_ISDISCONNECTED; 3794 3795 if (!SOLISTENING(so)) { 3796 SOCKBUF_LOCK(&so->so_rcv); 3797 socantrcvmore_locked(so); 3798 SOCKBUF_LOCK(&so->so_snd); 3799 sbdrop_locked(&so->so_snd, sbused(&so->so_snd)); 3800 socantsendmore_locked(so); 3801 } 3802 SOCK_UNLOCK(so); 3803 wakeup(&so->so_timeo); 3804 } 3805 3806 /* 3807 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME. 3808 */ 3809 struct sockaddr * 3810 sodupsockaddr(const struct sockaddr *sa, int mflags) 3811 { 3812 struct sockaddr *sa2; 3813 3814 sa2 = malloc(sa->sa_len, M_SONAME, mflags); 3815 if (sa2) 3816 bcopy(sa, sa2, sa->sa_len); 3817 return sa2; 3818 } 3819 3820 /* 3821 * Register per-socket buffer upcalls. 3822 */ 3823 void 3824 soupcall_set(struct socket *so, int which, so_upcall_t func, void *arg) 3825 { 3826 struct sockbuf *sb; 3827 3828 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3829 3830 switch (which) { 3831 case SO_RCV: 3832 sb = &so->so_rcv; 3833 break; 3834 case SO_SND: 3835 sb = &so->so_snd; 3836 break; 3837 default: 3838 panic("soupcall_set: bad which"); 3839 } 3840 SOCKBUF_LOCK_ASSERT(sb); 3841 sb->sb_upcall = func; 3842 sb->sb_upcallarg = arg; 3843 sb->sb_flags |= SB_UPCALL; 3844 } 3845 3846 void 3847 soupcall_clear(struct socket *so, int which) 3848 { 3849 struct sockbuf *sb; 3850 3851 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so)); 3852 3853 switch (which) { 3854 case SO_RCV: 3855 sb = &so->so_rcv; 3856 break; 3857 case SO_SND: 3858 sb = &so->so_snd; 3859 break; 3860 default: 3861 panic("soupcall_clear: bad which"); 3862 } 3863 SOCKBUF_LOCK_ASSERT(sb); 3864 KASSERT(sb->sb_upcall != NULL, 3865 ("%s: so %p no upcall to clear", __func__, so)); 3866 sb->sb_upcall = NULL; 3867 sb->sb_upcallarg = NULL; 3868 sb->sb_flags &= ~SB_UPCALL; 3869 } 3870 3871 void 3872 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg) 3873 { 3874 3875 SOLISTEN_LOCK_ASSERT(so); 3876 so->sol_upcall = func; 3877 so->sol_upcallarg = arg; 3878 } 3879 3880 static void 3881 so_rdknl_lock(void *arg) 3882 { 3883 struct socket *so = arg; 3884 3885 if (SOLISTENING(so)) 3886 SOCK_LOCK(so); 3887 else 3888 SOCKBUF_LOCK(&so->so_rcv); 3889 } 3890 3891 static void 3892 so_rdknl_unlock(void *arg) 3893 { 3894 struct socket *so = arg; 3895 3896 if (SOLISTENING(so)) 3897 SOCK_UNLOCK(so); 3898 else 3899 SOCKBUF_UNLOCK(&so->so_rcv); 3900 } 3901 3902 static void 3903 so_rdknl_assert_locked(void *arg) 3904 { 3905 struct socket *so = arg; 3906 3907 if (SOLISTENING(so)) 3908 SOCK_LOCK_ASSERT(so); 3909 else 3910 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 3911 } 3912 3913 static void 3914 so_rdknl_assert_unlocked(void *arg) 3915 { 3916 struct socket *so = arg; 3917 3918 if (SOLISTENING(so)) 3919 SOCK_UNLOCK_ASSERT(so); 3920 else 3921 SOCKBUF_UNLOCK_ASSERT(&so->so_rcv); 3922 } 3923 3924 static void 3925 so_wrknl_lock(void *arg) 3926 { 3927 struct socket *so = arg; 3928 3929 if (SOLISTENING(so)) 3930 SOCK_LOCK(so); 3931 else 3932 SOCKBUF_LOCK(&so->so_snd); 3933 } 3934 3935 static void 3936 so_wrknl_unlock(void *arg) 3937 { 3938 struct socket *so = arg; 3939 3940 if (SOLISTENING(so)) 3941 SOCK_UNLOCK(so); 3942 else 3943 SOCKBUF_UNLOCK(&so->so_snd); 3944 } 3945 3946 static void 3947 so_wrknl_assert_locked(void *arg) 3948 { 3949 struct socket *so = arg; 3950 3951 if (SOLISTENING(so)) 3952 SOCK_LOCK_ASSERT(so); 3953 else 3954 SOCKBUF_LOCK_ASSERT(&so->so_snd); 3955 } 3956 3957 static void 3958 so_wrknl_assert_unlocked(void *arg) 3959 { 3960 struct socket *so = arg; 3961 3962 if (SOLISTENING(so)) 3963 SOCK_UNLOCK_ASSERT(so); 3964 else 3965 SOCKBUF_UNLOCK_ASSERT(&so->so_snd); 3966 } 3967 3968 /* 3969 * Create an external-format (``xsocket'') structure using the information in 3970 * the kernel-format socket structure pointed to by so. This is done to 3971 * reduce the spew of irrelevant information over this interface, to isolate 3972 * user code from changes in the kernel structure, and potentially to provide 3973 * information-hiding if we decide that some of this information should be 3974 * hidden from users. 3975 */ 3976 void 3977 sotoxsocket(struct socket *so, struct xsocket *xso) 3978 { 3979 3980 xso->xso_len = sizeof *xso; 3981 xso->xso_so = so; 3982 xso->so_type = so->so_type; 3983 xso->so_options = so->so_options; 3984 xso->so_linger = so->so_linger; 3985 xso->so_state = so->so_state; 3986 xso->so_pcb = so->so_pcb; 3987 xso->xso_protocol = so->so_proto->pr_protocol; 3988 xso->xso_family = so->so_proto->pr_domain->dom_family; 3989 xso->so_timeo = so->so_timeo; 3990 xso->so_error = so->so_error; 3991 xso->so_uid = so->so_cred->cr_uid; 3992 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0; 3993 if (SOLISTENING(so)) { 3994 xso->so_qlen = so->sol_qlen; 3995 xso->so_incqlen = so->sol_incqlen; 3996 xso->so_qlimit = so->sol_qlimit; 3997 xso->so_oobmark = 0; 3998 bzero(&xso->so_snd, sizeof(xso->so_snd)); 3999 bzero(&xso->so_rcv, sizeof(xso->so_rcv)); 4000 } else { 4001 xso->so_state |= so->so_qstate; 4002 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0; 4003 xso->so_oobmark = so->so_oobmark; 4004 sbtoxsockbuf(&so->so_snd, &xso->so_snd); 4005 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv); 4006 } 4007 } 4008 4009 struct sockbuf * 4010 so_sockbuf_rcv(struct socket *so) 4011 { 4012 4013 return (&so->so_rcv); 4014 } 4015 4016 struct sockbuf * 4017 so_sockbuf_snd(struct socket *so) 4018 { 4019 4020 return (&so->so_snd); 4021 } 4022 4023 int 4024 so_state_get(const struct socket *so) 4025 { 4026 4027 return (so->so_state); 4028 } 4029 4030 void 4031 so_state_set(struct socket *so, int val) 4032 { 4033 4034 so->so_state = val; 4035 } 4036 4037 int 4038 so_options_get(const struct socket *so) 4039 { 4040 4041 return (so->so_options); 4042 } 4043 4044 void 4045 so_options_set(struct socket *so, int val) 4046 { 4047 4048 so->so_options = val; 4049 } 4050 4051 int 4052 so_error_get(const struct socket *so) 4053 { 4054 4055 return (so->so_error); 4056 } 4057 4058 void 4059 so_error_set(struct socket *so, int val) 4060 { 4061 4062 so->so_error = val; 4063 } 4064 4065 int 4066 so_linger_get(const struct socket *so) 4067 { 4068 4069 return (so->so_linger); 4070 } 4071 4072 void 4073 so_linger_set(struct socket *so, int val) 4074 { 4075 4076 so->so_linger = val; 4077 } 4078 4079 struct protosw * 4080 so_protosw_get(const struct socket *so) 4081 { 4082 4083 return (so->so_proto); 4084 } 4085 4086 void 4087 so_protosw_set(struct socket *so, struct protosw *val) 4088 { 4089 4090 so->so_proto = val; 4091 } 4092 4093 void 4094 so_sorwakeup(struct socket *so) 4095 { 4096 4097 sorwakeup(so); 4098 } 4099 4100 void 4101 so_sowwakeup(struct socket *so) 4102 { 4103 4104 sowwakeup(so); 4105 } 4106 4107 void 4108 so_sorwakeup_locked(struct socket *so) 4109 { 4110 4111 sorwakeup_locked(so); 4112 } 4113 4114 void 4115 so_sowwakeup_locked(struct socket *so) 4116 { 4117 4118 sowwakeup_locked(so); 4119 } 4120 4121 void 4122 so_lock(struct socket *so) 4123 { 4124 4125 SOCK_LOCK(so); 4126 } 4127 4128 void 4129 so_unlock(struct socket *so) 4130 { 4131 4132 SOCK_UNLOCK(so); 4133 } 4134