1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1982, 1986, 1988, 1990, 1993 5 * The Regents of the University of California. All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. Neither the name of the University nor the names of its contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 * 31 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_kern_tls.h" 38 #include "opt_param.h" 39 40 #include <sys/param.h> 41 #include <sys/aio.h> /* for aio_swake proto */ 42 #include <sys/kernel.h> 43 #include <sys/ktls.h> 44 #include <sys/lock.h> 45 #include <sys/malloc.h> 46 #include <sys/mbuf.h> 47 #include <sys/mutex.h> 48 #include <sys/proc.h> 49 #include <sys/protosw.h> 50 #include <sys/resourcevar.h> 51 #include <sys/signalvar.h> 52 #include <sys/socket.h> 53 #include <sys/socketvar.h> 54 #include <sys/sx.h> 55 #include <sys/sysctl.h> 56 57 /* 58 * Function pointer set by the AIO routines so that the socket buffer code 59 * can call back into the AIO module if it is loaded. 60 */ 61 void (*aio_swake)(struct socket *, struct sockbuf *); 62 63 /* 64 * Primitive routines for operating on socket buffers 65 */ 66 67 u_long sb_max = SB_MAX; 68 u_long sb_max_adj = 69 (quad_t)SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 70 71 static u_long sb_efficiency = 8; /* parameter for sbreserve() */ 72 73 static struct mbuf *sbcut_internal(struct sockbuf *sb, int len); 74 static void sbflush_internal(struct sockbuf *sb); 75 76 /* 77 * Our own version of m_clrprotoflags(), that can preserve M_NOTREADY. 78 */ 79 static void 80 sbm_clrprotoflags(struct mbuf *m, int flags) 81 { 82 int mask; 83 84 mask = ~M_PROTOFLAGS; 85 if (flags & PRUS_NOTREADY) 86 mask |= M_NOTREADY; 87 while (m) { 88 m->m_flags &= mask; 89 m = m->m_next; 90 } 91 } 92 93 /* 94 * Compress M_NOTREADY mbufs after they have been readied by sbready(). 95 * 96 * sbcompress() skips M_NOTREADY mbufs since the data is not available to 97 * be copied at the time of sbcompress(). This function combines small 98 * mbufs similar to sbcompress() once mbufs are ready. 'm0' is the first 99 * mbuf sbready() marked ready, and 'end' is the first mbuf still not 100 * ready. 101 */ 102 static void 103 sbready_compress(struct sockbuf *sb, struct mbuf *m0, struct mbuf *end) 104 { 105 struct mbuf *m, *n; 106 int ext_size; 107 108 SOCKBUF_LOCK_ASSERT(sb); 109 110 if ((sb->sb_flags & SB_NOCOALESCE) != 0) 111 return; 112 113 for (m = m0; m != end; m = m->m_next) { 114 MPASS((m->m_flags & M_NOTREADY) == 0); 115 /* 116 * NB: In sbcompress(), 'n' is the last mbuf in the 117 * socket buffer and 'm' is the new mbuf being copied 118 * into the trailing space of 'n'. Here, the roles 119 * are reversed and 'n' is the next mbuf after 'm' 120 * that is being copied into the trailing space of 121 * 'm'. 122 */ 123 n = m->m_next; 124 #ifdef KERN_TLS 125 /* Try to coalesce adjacent ktls mbuf hdr/trailers. */ 126 if ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 && 127 (m->m_flags & M_NOMAP) && 128 (n->m_flags & M_NOMAP) && 129 !mbuf_has_tls_session(m) && 130 !mbuf_has_tls_session(n)) { 131 struct mbuf_ext_pgs *mpgs, *npgs; 132 int hdr_len, trail_len; 133 134 mpgs = m->m_ext.ext_pgs; 135 npgs = n->m_ext.ext_pgs; 136 hdr_len = npgs->hdr_len; 137 trail_len = mpgs->trail_len; 138 if (trail_len != 0 && hdr_len != 0 && 139 trail_len + hdr_len <= MBUF_PEXT_TRAIL_LEN) { 140 /* copy n's header to m's trailer */ 141 memcpy(&mpgs->trail[trail_len], npgs->hdr, 142 hdr_len); 143 mpgs->trail_len += hdr_len; 144 m->m_len += hdr_len; 145 npgs->hdr_len = 0; 146 n->m_len -= hdr_len; 147 } 148 } 149 #endif 150 151 /* Compress small unmapped mbufs into plain mbufs. */ 152 if ((m->m_flags & M_NOMAP) && m->m_len <= MLEN && 153 !mbuf_has_tls_session(m)) { 154 MPASS(m->m_flags & M_EXT); 155 ext_size = m->m_ext.ext_size; 156 if (mb_unmapped_compress(m) == 0) { 157 sb->sb_mbcnt -= ext_size; 158 sb->sb_ccnt -= 1; 159 } 160 } 161 162 while ((n != NULL) && (n != end) && (m->m_flags & M_EOR) == 0 && 163 M_WRITABLE(m) && 164 (m->m_flags & M_NOMAP) == 0 && 165 !mbuf_has_tls_session(n) && 166 !mbuf_has_tls_session(m) && 167 n->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 168 n->m_len <= M_TRAILINGSPACE(m) && 169 m->m_type == n->m_type) { 170 KASSERT(sb->sb_lastrecord != n, 171 ("%s: merging start of record (%p) into previous mbuf (%p)", 172 __func__, n, m)); 173 m_copydata(n, 0, n->m_len, mtodo(m, m->m_len)); 174 m->m_len += n->m_len; 175 m->m_next = n->m_next; 176 m->m_flags |= n->m_flags & M_EOR; 177 if (sb->sb_mbtail == n) 178 sb->sb_mbtail = m; 179 180 sb->sb_mbcnt -= MSIZE; 181 sb->sb_mcnt -= 1; 182 if (n->m_flags & M_EXT) { 183 sb->sb_mbcnt -= n->m_ext.ext_size; 184 sb->sb_ccnt -= 1; 185 } 186 m_free(n); 187 n = m->m_next; 188 } 189 } 190 SBLASTRECORDCHK(sb); 191 SBLASTMBUFCHK(sb); 192 } 193 194 /* 195 * Mark ready "count" units of I/O starting with "m". Most mbufs 196 * count as a single unit of I/O except for EXT_PGS-backed mbufs which 197 * can be backed by multiple pages. 198 */ 199 int 200 sbready(struct sockbuf *sb, struct mbuf *m0, int count) 201 { 202 struct mbuf *m; 203 u_int blocker; 204 205 SOCKBUF_LOCK_ASSERT(sb); 206 KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); 207 KASSERT(count > 0, ("%s: invalid count %d", __func__, count)); 208 209 m = m0; 210 blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0; 211 212 while (count > 0) { 213 KASSERT(m->m_flags & M_NOTREADY, 214 ("%s: m %p !M_NOTREADY", __func__, m)); 215 if ((m->m_flags & M_EXT) != 0 && 216 m->m_ext.ext_type == EXT_PGS) { 217 if (count < m->m_ext.ext_pgs->nrdy) { 218 m->m_ext.ext_pgs->nrdy -= count; 219 count = 0; 220 break; 221 } 222 count -= m->m_ext.ext_pgs->nrdy; 223 m->m_ext.ext_pgs->nrdy = 0; 224 } else 225 count--; 226 227 m->m_flags &= ~(M_NOTREADY | blocker); 228 if (blocker) 229 sb->sb_acc += m->m_len; 230 m = m->m_next; 231 } 232 233 /* 234 * If the first mbuf is still not fully ready because only 235 * some of its backing pages were readied, no further progress 236 * can be made. 237 */ 238 if (m0 == m) { 239 MPASS(m->m_flags & M_NOTREADY); 240 return (EINPROGRESS); 241 } 242 243 if (!blocker) { 244 sbready_compress(sb, m0, m); 245 return (EINPROGRESS); 246 } 247 248 /* This one was blocking all the queue. */ 249 for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) { 250 KASSERT(m->m_flags & M_BLOCKED, 251 ("%s: m %p !M_BLOCKED", __func__, m)); 252 m->m_flags &= ~M_BLOCKED; 253 sb->sb_acc += m->m_len; 254 } 255 256 sb->sb_fnrdy = m; 257 sbready_compress(sb, m0, m); 258 259 return (0); 260 } 261 262 /* 263 * Adjust sockbuf state reflecting allocation of m. 264 */ 265 void 266 sballoc(struct sockbuf *sb, struct mbuf *m) 267 { 268 269 SOCKBUF_LOCK_ASSERT(sb); 270 271 sb->sb_ccc += m->m_len; 272 273 if (sb->sb_fnrdy == NULL) { 274 if (m->m_flags & M_NOTREADY) 275 sb->sb_fnrdy = m; 276 else 277 sb->sb_acc += m->m_len; 278 } else 279 m->m_flags |= M_BLOCKED; 280 281 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 282 sb->sb_ctl += m->m_len; 283 284 sb->sb_mbcnt += MSIZE; 285 sb->sb_mcnt += 1; 286 287 if (m->m_flags & M_EXT) { 288 sb->sb_mbcnt += m->m_ext.ext_size; 289 sb->sb_ccnt += 1; 290 } 291 } 292 293 /* 294 * Adjust sockbuf state reflecting freeing of m. 295 */ 296 void 297 sbfree(struct sockbuf *sb, struct mbuf *m) 298 { 299 300 #if 0 /* XXX: not yet: soclose() call path comes here w/o lock. */ 301 SOCKBUF_LOCK_ASSERT(sb); 302 #endif 303 304 sb->sb_ccc -= m->m_len; 305 306 if (!(m->m_flags & M_NOTAVAIL)) 307 sb->sb_acc -= m->m_len; 308 309 if (m == sb->sb_fnrdy) { 310 struct mbuf *n; 311 312 KASSERT(m->m_flags & M_NOTREADY, 313 ("%s: m %p !M_NOTREADY", __func__, m)); 314 315 n = m->m_next; 316 while (n != NULL && !(n->m_flags & M_NOTREADY)) { 317 n->m_flags &= ~M_BLOCKED; 318 sb->sb_acc += n->m_len; 319 n = n->m_next; 320 } 321 sb->sb_fnrdy = n; 322 } 323 324 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 325 sb->sb_ctl -= m->m_len; 326 327 sb->sb_mbcnt -= MSIZE; 328 sb->sb_mcnt -= 1; 329 if (m->m_flags & M_EXT) { 330 sb->sb_mbcnt -= m->m_ext.ext_size; 331 sb->sb_ccnt -= 1; 332 } 333 334 if (sb->sb_sndptr == m) { 335 sb->sb_sndptr = NULL; 336 sb->sb_sndptroff = 0; 337 } 338 if (sb->sb_sndptroff != 0) 339 sb->sb_sndptroff -= m->m_len; 340 } 341 342 /* 343 * Socantsendmore indicates that no more data will be sent on the socket; it 344 * would normally be applied to a socket when the user informs the system 345 * that no more data is to be sent, by the protocol code (in case 346 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 347 * received, and will normally be applied to the socket by a protocol when it 348 * detects that the peer will send no more data. Data queued for reading in 349 * the socket may yet be read. 350 */ 351 void 352 socantsendmore_locked(struct socket *so) 353 { 354 355 SOCKBUF_LOCK_ASSERT(&so->so_snd); 356 357 so->so_snd.sb_state |= SBS_CANTSENDMORE; 358 sowwakeup_locked(so); 359 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 360 } 361 362 void 363 socantsendmore(struct socket *so) 364 { 365 366 SOCKBUF_LOCK(&so->so_snd); 367 socantsendmore_locked(so); 368 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 369 } 370 371 void 372 socantrcvmore_locked(struct socket *so) 373 { 374 375 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 376 377 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 378 sorwakeup_locked(so); 379 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 380 } 381 382 void 383 socantrcvmore(struct socket *so) 384 { 385 386 SOCKBUF_LOCK(&so->so_rcv); 387 socantrcvmore_locked(so); 388 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 389 } 390 391 /* 392 * Wait for data to arrive at/drain from a socket buffer. 393 */ 394 int 395 sbwait(struct sockbuf *sb) 396 { 397 398 SOCKBUF_LOCK_ASSERT(sb); 399 400 sb->sb_flags |= SB_WAIT; 401 return (msleep_sbt(&sb->sb_acc, &sb->sb_mtx, 402 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 403 sb->sb_timeo, 0, 0)); 404 } 405 406 int 407 sblock(struct sockbuf *sb, int flags) 408 { 409 410 KASSERT((flags & SBL_VALID) == flags, 411 ("sblock: flags invalid (0x%x)", flags)); 412 413 if (flags & SBL_WAIT) { 414 if ((sb->sb_flags & SB_NOINTR) || 415 (flags & SBL_NOINTR)) { 416 sx_xlock(&sb->sb_sx); 417 return (0); 418 } 419 return (sx_xlock_sig(&sb->sb_sx)); 420 } else { 421 if (sx_try_xlock(&sb->sb_sx) == 0) 422 return (EWOULDBLOCK); 423 return (0); 424 } 425 } 426 427 void 428 sbunlock(struct sockbuf *sb) 429 { 430 431 sx_xunlock(&sb->sb_sx); 432 } 433 434 /* 435 * Wakeup processes waiting on a socket buffer. Do asynchronous notification 436 * via SIGIO if the socket has the SS_ASYNC flag set. 437 * 438 * Called with the socket buffer lock held; will release the lock by the end 439 * of the function. This allows the caller to acquire the socket buffer lock 440 * while testing for the need for various sorts of wakeup and hold it through 441 * to the point where it's no longer required. We currently hold the lock 442 * through calls out to other subsystems (with the exception of kqueue), and 443 * then release it to avoid lock order issues. It's not clear that's 444 * correct. 445 */ 446 void 447 sowakeup(struct socket *so, struct sockbuf *sb) 448 { 449 int ret; 450 451 SOCKBUF_LOCK_ASSERT(sb); 452 453 selwakeuppri(sb->sb_sel, PSOCK); 454 if (!SEL_WAITING(sb->sb_sel)) 455 sb->sb_flags &= ~SB_SEL; 456 if (sb->sb_flags & SB_WAIT) { 457 sb->sb_flags &= ~SB_WAIT; 458 wakeup(&sb->sb_acc); 459 } 460 KNOTE_LOCKED(&sb->sb_sel->si_note, 0); 461 if (sb->sb_upcall != NULL) { 462 ret = sb->sb_upcall(so, sb->sb_upcallarg, M_NOWAIT); 463 if (ret == SU_ISCONNECTED) { 464 KASSERT(sb == &so->so_rcv, 465 ("SO_SND upcall returned SU_ISCONNECTED")); 466 soupcall_clear(so, SO_RCV); 467 } 468 } else 469 ret = SU_OK; 470 if (sb->sb_flags & SB_AIO) 471 sowakeup_aio(so, sb); 472 SOCKBUF_UNLOCK(sb); 473 if (ret == SU_ISCONNECTED) 474 soisconnected(so); 475 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) 476 pgsigio(&so->so_sigio, SIGIO, 0); 477 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 478 } 479 480 /* 481 * Socket buffer (struct sockbuf) utility routines. 482 * 483 * Each socket contains two socket buffers: one for sending data and one for 484 * receiving data. Each buffer contains a queue of mbufs, information about 485 * the number of mbufs and amount of data in the queue, and other fields 486 * allowing select() statements and notification on data availability to be 487 * implemented. 488 * 489 * Data stored in a socket buffer is maintained as a list of records. Each 490 * record is a list of mbufs chained together with the m_next field. Records 491 * are chained together with the m_nextpkt field. The upper level routine 492 * soreceive() expects the following conventions to be observed when placing 493 * information in the receive buffer: 494 * 495 * 1. If the protocol requires each message be preceded by the sender's name, 496 * then a record containing that name must be present before any 497 * associated data (mbuf's must be of type MT_SONAME). 498 * 2. If the protocol supports the exchange of ``access rights'' (really just 499 * additional data associated with the message), and there are ``rights'' 500 * to be received, then a record containing this data should be present 501 * (mbuf's must be of type MT_RIGHTS). 502 * 3. If a name or rights record exists, then it must be followed by a data 503 * record, perhaps of zero length. 504 * 505 * Before using a new socket structure it is first necessary to reserve 506 * buffer space to the socket, by calling sbreserve(). This should commit 507 * some of the available buffer space in the system buffer pool for the 508 * socket (currently, it does nothing but enforce limits). The space should 509 * be released by calling sbrelease() when the socket is destroyed. 510 */ 511 int 512 soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 513 { 514 struct thread *td = curthread; 515 516 SOCKBUF_LOCK(&so->so_snd); 517 SOCKBUF_LOCK(&so->so_rcv); 518 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 519 goto bad; 520 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 521 goto bad2; 522 if (so->so_rcv.sb_lowat == 0) 523 so->so_rcv.sb_lowat = 1; 524 if (so->so_snd.sb_lowat == 0) 525 so->so_snd.sb_lowat = MCLBYTES; 526 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 527 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 528 SOCKBUF_UNLOCK(&so->so_rcv); 529 SOCKBUF_UNLOCK(&so->so_snd); 530 return (0); 531 bad2: 532 sbrelease_locked(&so->so_snd, so); 533 bad: 534 SOCKBUF_UNLOCK(&so->so_rcv); 535 SOCKBUF_UNLOCK(&so->so_snd); 536 return (ENOBUFS); 537 } 538 539 static int 540 sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 541 { 542 int error = 0; 543 u_long tmp_sb_max = sb_max; 544 545 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 546 if (error || !req->newptr) 547 return (error); 548 if (tmp_sb_max < MSIZE + MCLBYTES) 549 return (EINVAL); 550 sb_max = tmp_sb_max; 551 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 552 return (0); 553 } 554 555 /* 556 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 557 * become limiting if buffering efficiency is near the normal case. 558 */ 559 int 560 sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 561 struct thread *td) 562 { 563 rlim_t sbsize_limit; 564 565 SOCKBUF_LOCK_ASSERT(sb); 566 567 /* 568 * When a thread is passed, we take into account the thread's socket 569 * buffer size limit. The caller will generally pass curthread, but 570 * in the TCP input path, NULL will be passed to indicate that no 571 * appropriate thread resource limits are available. In that case, 572 * we don't apply a process limit. 573 */ 574 if (cc > sb_max_adj) 575 return (0); 576 if (td != NULL) { 577 sbsize_limit = lim_cur(td, RLIMIT_SBSIZE); 578 } else 579 sbsize_limit = RLIM_INFINITY; 580 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 581 sbsize_limit)) 582 return (0); 583 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 584 if (sb->sb_lowat > sb->sb_hiwat) 585 sb->sb_lowat = sb->sb_hiwat; 586 return (1); 587 } 588 589 int 590 sbsetopt(struct socket *so, int cmd, u_long cc) 591 { 592 struct sockbuf *sb; 593 short *flags; 594 u_int *hiwat, *lowat; 595 int error; 596 597 sb = NULL; 598 SOCK_LOCK(so); 599 if (SOLISTENING(so)) { 600 switch (cmd) { 601 case SO_SNDLOWAT: 602 case SO_SNDBUF: 603 lowat = &so->sol_sbsnd_lowat; 604 hiwat = &so->sol_sbsnd_hiwat; 605 flags = &so->sol_sbsnd_flags; 606 break; 607 case SO_RCVLOWAT: 608 case SO_RCVBUF: 609 lowat = &so->sol_sbrcv_lowat; 610 hiwat = &so->sol_sbrcv_hiwat; 611 flags = &so->sol_sbrcv_flags; 612 break; 613 } 614 } else { 615 switch (cmd) { 616 case SO_SNDLOWAT: 617 case SO_SNDBUF: 618 sb = &so->so_snd; 619 break; 620 case SO_RCVLOWAT: 621 case SO_RCVBUF: 622 sb = &so->so_rcv; 623 break; 624 } 625 flags = &sb->sb_flags; 626 hiwat = &sb->sb_hiwat; 627 lowat = &sb->sb_lowat; 628 SOCKBUF_LOCK(sb); 629 } 630 631 error = 0; 632 switch (cmd) { 633 case SO_SNDBUF: 634 case SO_RCVBUF: 635 if (SOLISTENING(so)) { 636 if (cc > sb_max_adj) { 637 error = ENOBUFS; 638 break; 639 } 640 *hiwat = cc; 641 if (*lowat > *hiwat) 642 *lowat = *hiwat; 643 } else { 644 if (!sbreserve_locked(sb, cc, so, curthread)) 645 error = ENOBUFS; 646 } 647 if (error == 0) 648 *flags &= ~SB_AUTOSIZE; 649 break; 650 case SO_SNDLOWAT: 651 case SO_RCVLOWAT: 652 /* 653 * Make sure the low-water is never greater than the 654 * high-water. 655 */ 656 *lowat = (cc > *hiwat) ? *hiwat : cc; 657 break; 658 } 659 660 if (!SOLISTENING(so)) 661 SOCKBUF_UNLOCK(sb); 662 SOCK_UNLOCK(so); 663 return (error); 664 } 665 666 /* 667 * Free mbufs held by a socket, and reserved mbuf space. 668 */ 669 void 670 sbrelease_internal(struct sockbuf *sb, struct socket *so) 671 { 672 673 sbflush_internal(sb); 674 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 675 RLIM_INFINITY); 676 sb->sb_mbmax = 0; 677 } 678 679 void 680 sbrelease_locked(struct sockbuf *sb, struct socket *so) 681 { 682 683 SOCKBUF_LOCK_ASSERT(sb); 684 685 sbrelease_internal(sb, so); 686 } 687 688 void 689 sbrelease(struct sockbuf *sb, struct socket *so) 690 { 691 692 SOCKBUF_LOCK(sb); 693 sbrelease_locked(sb, so); 694 SOCKBUF_UNLOCK(sb); 695 } 696 697 void 698 sbdestroy(struct sockbuf *sb, struct socket *so) 699 { 700 701 sbrelease_internal(sb, so); 702 #ifdef KERN_TLS 703 if (sb->sb_tls_info != NULL) 704 ktls_free(sb->sb_tls_info); 705 sb->sb_tls_info = NULL; 706 #endif 707 } 708 709 /* 710 * Routines to add and remove data from an mbuf queue. 711 * 712 * The routines sbappend() or sbappendrecord() are normally called to append 713 * new mbufs to a socket buffer, after checking that adequate space is 714 * available, comparing the function sbspace() with the amount of data to be 715 * added. sbappendrecord() differs from sbappend() in that data supplied is 716 * treated as the beginning of a new record. To place a sender's address, 717 * optional access rights, and data in a socket receive buffer, 718 * sbappendaddr() should be used. To place access rights and data in a 719 * socket receive buffer, sbappendrights() should be used. In either case, 720 * the new data begins a new record. Note that unlike sbappend() and 721 * sbappendrecord(), these routines check for the caller that there will be 722 * enough space to store the data. Each fails if there is not enough space, 723 * or if it cannot find mbufs to store additional information in. 724 * 725 * Reliable protocols may use the socket send buffer to hold data awaiting 726 * acknowledgement. Data is normally copied from a socket send buffer in a 727 * protocol with m_copy for output to a peer, and then removing the data from 728 * the socket buffer with sbdrop() or sbdroprecord() when the data is 729 * acknowledged by the peer. 730 */ 731 #ifdef SOCKBUF_DEBUG 732 void 733 sblastrecordchk(struct sockbuf *sb, const char *file, int line) 734 { 735 struct mbuf *m = sb->sb_mb; 736 737 SOCKBUF_LOCK_ASSERT(sb); 738 739 while (m && m->m_nextpkt) 740 m = m->m_nextpkt; 741 742 if (m != sb->sb_lastrecord) { 743 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 744 __func__, sb->sb_mb, sb->sb_lastrecord, m); 745 printf("packet chain:\n"); 746 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 747 printf("\t%p\n", m); 748 panic("%s from %s:%u", __func__, file, line); 749 } 750 } 751 752 void 753 sblastmbufchk(struct sockbuf *sb, const char *file, int line) 754 { 755 struct mbuf *m = sb->sb_mb; 756 struct mbuf *n; 757 758 SOCKBUF_LOCK_ASSERT(sb); 759 760 while (m && m->m_nextpkt) 761 m = m->m_nextpkt; 762 763 while (m && m->m_next) 764 m = m->m_next; 765 766 if (m != sb->sb_mbtail) { 767 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 768 __func__, sb->sb_mb, sb->sb_mbtail, m); 769 printf("packet tree:\n"); 770 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 771 printf("\t"); 772 for (n = m; n != NULL; n = n->m_next) 773 printf("%p ", n); 774 printf("\n"); 775 } 776 panic("%s from %s:%u", __func__, file, line); 777 } 778 } 779 #endif /* SOCKBUF_DEBUG */ 780 781 #define SBLINKRECORD(sb, m0) do { \ 782 SOCKBUF_LOCK_ASSERT(sb); \ 783 if ((sb)->sb_lastrecord != NULL) \ 784 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 785 else \ 786 (sb)->sb_mb = (m0); \ 787 (sb)->sb_lastrecord = (m0); \ 788 } while (/*CONSTCOND*/0) 789 790 /* 791 * Append mbuf chain m to the last record in the socket buffer sb. The 792 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 793 * are discarded and mbufs are compacted where possible. 794 */ 795 void 796 sbappend_locked(struct sockbuf *sb, struct mbuf *m, int flags) 797 { 798 struct mbuf *n; 799 800 SOCKBUF_LOCK_ASSERT(sb); 801 802 if (m == NULL) 803 return; 804 sbm_clrprotoflags(m, flags); 805 SBLASTRECORDCHK(sb); 806 n = sb->sb_mb; 807 if (n) { 808 while (n->m_nextpkt) 809 n = n->m_nextpkt; 810 do { 811 if (n->m_flags & M_EOR) { 812 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 813 return; 814 } 815 } while (n->m_next && (n = n->m_next)); 816 } else { 817 /* 818 * XXX Would like to simply use sb_mbtail here, but 819 * XXX I need to verify that I won't miss an EOR that 820 * XXX way. 821 */ 822 if ((n = sb->sb_lastrecord) != NULL) { 823 do { 824 if (n->m_flags & M_EOR) { 825 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 826 return; 827 } 828 } while (n->m_next && (n = n->m_next)); 829 } else { 830 /* 831 * If this is the first record in the socket buffer, 832 * it's also the last record. 833 */ 834 sb->sb_lastrecord = m; 835 } 836 } 837 sbcompress(sb, m, n); 838 SBLASTRECORDCHK(sb); 839 } 840 841 /* 842 * Append mbuf chain m to the last record in the socket buffer sb. The 843 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 844 * are discarded and mbufs are compacted where possible. 845 */ 846 void 847 sbappend(struct sockbuf *sb, struct mbuf *m, int flags) 848 { 849 850 SOCKBUF_LOCK(sb); 851 sbappend_locked(sb, m, flags); 852 SOCKBUF_UNLOCK(sb); 853 } 854 855 /* 856 * This version of sbappend() should only be used when the caller absolutely 857 * knows that there will never be more than one record in the socket buffer, 858 * that is, a stream protocol (such as TCP). 859 */ 860 void 861 sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags) 862 { 863 SOCKBUF_LOCK_ASSERT(sb); 864 865 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 866 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 867 868 SBLASTMBUFCHK(sb); 869 870 #ifdef KERN_TLS 871 if (sb->sb_tls_info != NULL) 872 ktls_seq(sb, m); 873 #endif 874 875 /* Remove all packet headers and mbuf tags to get a pure data chain. */ 876 m_demote(m, 1, flags & PRUS_NOTREADY ? M_NOTREADY : 0); 877 878 sbcompress(sb, m, sb->sb_mbtail); 879 880 sb->sb_lastrecord = sb->sb_mb; 881 SBLASTRECORDCHK(sb); 882 } 883 884 /* 885 * This version of sbappend() should only be used when the caller absolutely 886 * knows that there will never be more than one record in the socket buffer, 887 * that is, a stream protocol (such as TCP). 888 */ 889 void 890 sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags) 891 { 892 893 SOCKBUF_LOCK(sb); 894 sbappendstream_locked(sb, m, flags); 895 SOCKBUF_UNLOCK(sb); 896 } 897 898 #ifdef SOCKBUF_DEBUG 899 void 900 sbcheck(struct sockbuf *sb, const char *file, int line) 901 { 902 struct mbuf *m, *n, *fnrdy; 903 u_long acc, ccc, mbcnt; 904 905 SOCKBUF_LOCK_ASSERT(sb); 906 907 acc = ccc = mbcnt = 0; 908 fnrdy = NULL; 909 910 for (m = sb->sb_mb; m; m = n) { 911 n = m->m_nextpkt; 912 for (; m; m = m->m_next) { 913 if (m->m_len == 0) { 914 printf("sb %p empty mbuf %p\n", sb, m); 915 goto fail; 916 } 917 if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) { 918 if (m != sb->sb_fnrdy) { 919 printf("sb %p: fnrdy %p != m %p\n", 920 sb, sb->sb_fnrdy, m); 921 goto fail; 922 } 923 fnrdy = m; 924 } 925 if (fnrdy) { 926 if (!(m->m_flags & M_NOTAVAIL)) { 927 printf("sb %p: fnrdy %p, m %p is avail\n", 928 sb, sb->sb_fnrdy, m); 929 goto fail; 930 } 931 } else 932 acc += m->m_len; 933 ccc += m->m_len; 934 mbcnt += MSIZE; 935 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 936 mbcnt += m->m_ext.ext_size; 937 } 938 } 939 if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) { 940 printf("acc %ld/%u ccc %ld/%u mbcnt %ld/%u\n", 941 acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt); 942 goto fail; 943 } 944 return; 945 fail: 946 panic("%s from %s:%u", __func__, file, line); 947 } 948 #endif 949 950 /* 951 * As above, except the mbuf chain begins a new record. 952 */ 953 void 954 sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 955 { 956 struct mbuf *m; 957 958 SOCKBUF_LOCK_ASSERT(sb); 959 960 if (m0 == NULL) 961 return; 962 m_clrprotoflags(m0); 963 /* 964 * Put the first mbuf on the queue. Note this permits zero length 965 * records. 966 */ 967 sballoc(sb, m0); 968 SBLASTRECORDCHK(sb); 969 SBLINKRECORD(sb, m0); 970 sb->sb_mbtail = m0; 971 m = m0->m_next; 972 m0->m_next = 0; 973 if (m && (m0->m_flags & M_EOR)) { 974 m0->m_flags &= ~M_EOR; 975 m->m_flags |= M_EOR; 976 } 977 /* always call sbcompress() so it can do SBLASTMBUFCHK() */ 978 sbcompress(sb, m, m0); 979 } 980 981 /* 982 * As above, except the mbuf chain begins a new record. 983 */ 984 void 985 sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 986 { 987 988 SOCKBUF_LOCK(sb); 989 sbappendrecord_locked(sb, m0); 990 SOCKBUF_UNLOCK(sb); 991 } 992 993 /* Helper routine that appends data, control, and address to a sockbuf. */ 994 static int 995 sbappendaddr_locked_internal(struct sockbuf *sb, const struct sockaddr *asa, 996 struct mbuf *m0, struct mbuf *control, struct mbuf *ctrl_last) 997 { 998 struct mbuf *m, *n, *nlast; 999 #if MSIZE <= 256 1000 if (asa->sa_len > MLEN) 1001 return (0); 1002 #endif 1003 m = m_get(M_NOWAIT, MT_SONAME); 1004 if (m == NULL) 1005 return (0); 1006 m->m_len = asa->sa_len; 1007 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 1008 if (m0) { 1009 m_clrprotoflags(m0); 1010 m_tag_delete_chain(m0, NULL); 1011 /* 1012 * Clear some persistent info from pkthdr. 1013 * We don't use m_demote(), because some netgraph consumers 1014 * expect M_PKTHDR presence. 1015 */ 1016 m0->m_pkthdr.rcvif = NULL; 1017 m0->m_pkthdr.flowid = 0; 1018 m0->m_pkthdr.csum_flags = 0; 1019 m0->m_pkthdr.fibnum = 0; 1020 m0->m_pkthdr.rsstype = 0; 1021 } 1022 if (ctrl_last) 1023 ctrl_last->m_next = m0; /* concatenate data to control */ 1024 else 1025 control = m0; 1026 m->m_next = control; 1027 for (n = m; n->m_next != NULL; n = n->m_next) 1028 sballoc(sb, n); 1029 sballoc(sb, n); 1030 nlast = n; 1031 SBLINKRECORD(sb, m); 1032 1033 sb->sb_mbtail = nlast; 1034 SBLASTMBUFCHK(sb); 1035 1036 SBLASTRECORDCHK(sb); 1037 return (1); 1038 } 1039 1040 /* 1041 * Append address and data, and optionally, control (ancillary) data to the 1042 * receive queue of a socket. If present, m0 must include a packet header 1043 * with total length. Returns 0 if no space in sockbuf or insufficient 1044 * mbufs. 1045 */ 1046 int 1047 sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 1048 struct mbuf *m0, struct mbuf *control) 1049 { 1050 struct mbuf *ctrl_last; 1051 int space = asa->sa_len; 1052 1053 SOCKBUF_LOCK_ASSERT(sb); 1054 1055 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 1056 panic("sbappendaddr_locked"); 1057 if (m0) 1058 space += m0->m_pkthdr.len; 1059 space += m_length(control, &ctrl_last); 1060 1061 if (space > sbspace(sb)) 1062 return (0); 1063 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 1064 } 1065 1066 /* 1067 * Append address and data, and optionally, control (ancillary) data to the 1068 * receive queue of a socket. If present, m0 must include a packet header 1069 * with total length. Returns 0 if insufficient mbufs. Does not validate space 1070 * on the receiving sockbuf. 1071 */ 1072 int 1073 sbappendaddr_nospacecheck_locked(struct sockbuf *sb, const struct sockaddr *asa, 1074 struct mbuf *m0, struct mbuf *control) 1075 { 1076 struct mbuf *ctrl_last; 1077 1078 SOCKBUF_LOCK_ASSERT(sb); 1079 1080 ctrl_last = (control == NULL) ? NULL : m_last(control); 1081 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 1082 } 1083 1084 /* 1085 * Append address and data, and optionally, control (ancillary) data to the 1086 * receive queue of a socket. If present, m0 must include a packet header 1087 * with total length. Returns 0 if no space in sockbuf or insufficient 1088 * mbufs. 1089 */ 1090 int 1091 sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 1092 struct mbuf *m0, struct mbuf *control) 1093 { 1094 int retval; 1095 1096 SOCKBUF_LOCK(sb); 1097 retval = sbappendaddr_locked(sb, asa, m0, control); 1098 SOCKBUF_UNLOCK(sb); 1099 return (retval); 1100 } 1101 1102 void 1103 sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 1104 struct mbuf *control) 1105 { 1106 struct mbuf *m, *mlast; 1107 1108 m_clrprotoflags(m0); 1109 m_last(control)->m_next = m0; 1110 1111 SBLASTRECORDCHK(sb); 1112 1113 for (m = control; m->m_next; m = m->m_next) 1114 sballoc(sb, m); 1115 sballoc(sb, m); 1116 mlast = m; 1117 SBLINKRECORD(sb, control); 1118 1119 sb->sb_mbtail = mlast; 1120 SBLASTMBUFCHK(sb); 1121 1122 SBLASTRECORDCHK(sb); 1123 } 1124 1125 void 1126 sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 1127 { 1128 1129 SOCKBUF_LOCK(sb); 1130 sbappendcontrol_locked(sb, m0, control); 1131 SOCKBUF_UNLOCK(sb); 1132 } 1133 1134 /* 1135 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 1136 * (n). If (n) is NULL, the buffer is presumed empty. 1137 * 1138 * When the data is compressed, mbufs in the chain may be handled in one of 1139 * three ways: 1140 * 1141 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 1142 * record boundary, and no change in data type). 1143 * 1144 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 1145 * an mbuf already in the socket buffer. This can occur if an 1146 * appropriate mbuf exists, there is room, both mbufs are not marked as 1147 * not ready, and no merging of data types will occur. 1148 * 1149 * (3) The mbuf may be appended to the end of the existing mbuf chain. 1150 * 1151 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 1152 * end-of-record. 1153 */ 1154 void 1155 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 1156 { 1157 int eor = 0; 1158 struct mbuf *o; 1159 1160 SOCKBUF_LOCK_ASSERT(sb); 1161 1162 while (m) { 1163 eor |= m->m_flags & M_EOR; 1164 if (m->m_len == 0 && 1165 (eor == 0 || 1166 (((o = m->m_next) || (o = n)) && 1167 o->m_type == m->m_type))) { 1168 if (sb->sb_lastrecord == m) 1169 sb->sb_lastrecord = m->m_next; 1170 m = m_free(m); 1171 continue; 1172 } 1173 if (n && (n->m_flags & M_EOR) == 0 && 1174 M_WRITABLE(n) && 1175 ((sb->sb_flags & SB_NOCOALESCE) == 0) && 1176 !(m->m_flags & M_NOTREADY) && 1177 !(n->m_flags & (M_NOTREADY | M_NOMAP)) && 1178 !mbuf_has_tls_session(m) && 1179 !mbuf_has_tls_session(n) && 1180 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 1181 m->m_len <= M_TRAILINGSPACE(n) && 1182 n->m_type == m->m_type) { 1183 m_copydata(m, 0, m->m_len, mtodo(n, n->m_len)); 1184 n->m_len += m->m_len; 1185 sb->sb_ccc += m->m_len; 1186 if (sb->sb_fnrdy == NULL) 1187 sb->sb_acc += m->m_len; 1188 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 1189 /* XXX: Probably don't need.*/ 1190 sb->sb_ctl += m->m_len; 1191 m = m_free(m); 1192 continue; 1193 } 1194 if (m->m_len <= MLEN && (m->m_flags & M_NOMAP) && 1195 (m->m_flags & M_NOTREADY) == 0 && 1196 !mbuf_has_tls_session(m)) 1197 (void)mb_unmapped_compress(m); 1198 if (n) 1199 n->m_next = m; 1200 else 1201 sb->sb_mb = m; 1202 sb->sb_mbtail = m; 1203 sballoc(sb, m); 1204 n = m; 1205 m->m_flags &= ~M_EOR; 1206 m = m->m_next; 1207 n->m_next = 0; 1208 } 1209 if (eor) { 1210 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 1211 n->m_flags |= eor; 1212 } 1213 SBLASTMBUFCHK(sb); 1214 } 1215 1216 /* 1217 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 1218 */ 1219 static void 1220 sbflush_internal(struct sockbuf *sb) 1221 { 1222 1223 while (sb->sb_mbcnt) { 1224 /* 1225 * Don't call sbcut(sb, 0) if the leading mbuf is non-empty: 1226 * we would loop forever. Panic instead. 1227 */ 1228 if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 1229 break; 1230 m_freem(sbcut_internal(sb, (int)sb->sb_ccc)); 1231 } 1232 KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0, 1233 ("%s: ccc %u mb %p mbcnt %u", __func__, 1234 sb->sb_ccc, (void *)sb->sb_mb, sb->sb_mbcnt)); 1235 } 1236 1237 void 1238 sbflush_locked(struct sockbuf *sb) 1239 { 1240 1241 SOCKBUF_LOCK_ASSERT(sb); 1242 sbflush_internal(sb); 1243 } 1244 1245 void 1246 sbflush(struct sockbuf *sb) 1247 { 1248 1249 SOCKBUF_LOCK(sb); 1250 sbflush_locked(sb); 1251 SOCKBUF_UNLOCK(sb); 1252 } 1253 1254 /* 1255 * Cut data from (the front of) a sockbuf. 1256 */ 1257 static struct mbuf * 1258 sbcut_internal(struct sockbuf *sb, int len) 1259 { 1260 struct mbuf *m, *next, *mfree; 1261 1262 KASSERT(len >= 0, ("%s: len is %d but it is supposed to be >= 0", 1263 __func__, len)); 1264 KASSERT(len <= sb->sb_ccc, ("%s: len: %d is > ccc: %u", 1265 __func__, len, sb->sb_ccc)); 1266 1267 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 1268 mfree = NULL; 1269 1270 while (len > 0) { 1271 if (m == NULL) { 1272 KASSERT(next, ("%s: no next, len %d", __func__, len)); 1273 m = next; 1274 next = m->m_nextpkt; 1275 } 1276 if (m->m_len > len) { 1277 KASSERT(!(m->m_flags & M_NOTAVAIL), 1278 ("%s: m %p M_NOTAVAIL", __func__, m)); 1279 m->m_len -= len; 1280 m->m_data += len; 1281 sb->sb_ccc -= len; 1282 sb->sb_acc -= len; 1283 if (sb->sb_sndptroff != 0) 1284 sb->sb_sndptroff -= len; 1285 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 1286 sb->sb_ctl -= len; 1287 break; 1288 } 1289 len -= m->m_len; 1290 sbfree(sb, m); 1291 /* 1292 * Do not put M_NOTREADY buffers to the free list, they 1293 * are referenced from outside. 1294 */ 1295 if (m->m_flags & M_NOTREADY) 1296 m = m->m_next; 1297 else { 1298 struct mbuf *n; 1299 1300 n = m->m_next; 1301 m->m_next = mfree; 1302 mfree = m; 1303 m = n; 1304 } 1305 } 1306 /* 1307 * Free any zero-length mbufs from the buffer. 1308 * For SOCK_DGRAM sockets such mbufs represent empty records. 1309 * XXX: For SOCK_STREAM sockets such mbufs can appear in the buffer, 1310 * when sosend_generic() needs to send only control data. 1311 */ 1312 while (m && m->m_len == 0) { 1313 struct mbuf *n; 1314 1315 sbfree(sb, m); 1316 n = m->m_next; 1317 m->m_next = mfree; 1318 mfree = m; 1319 m = n; 1320 } 1321 if (m) { 1322 sb->sb_mb = m; 1323 m->m_nextpkt = next; 1324 } else 1325 sb->sb_mb = next; 1326 /* 1327 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 1328 * sb_lastrecord is up-to-date if we dropped part of the last record. 1329 */ 1330 m = sb->sb_mb; 1331 if (m == NULL) { 1332 sb->sb_mbtail = NULL; 1333 sb->sb_lastrecord = NULL; 1334 } else if (m->m_nextpkt == NULL) { 1335 sb->sb_lastrecord = m; 1336 } 1337 1338 return (mfree); 1339 } 1340 1341 /* 1342 * Drop data from (the front of) a sockbuf. 1343 */ 1344 void 1345 sbdrop_locked(struct sockbuf *sb, int len) 1346 { 1347 1348 SOCKBUF_LOCK_ASSERT(sb); 1349 m_freem(sbcut_internal(sb, len)); 1350 } 1351 1352 /* 1353 * Drop data from (the front of) a sockbuf, 1354 * and return it to caller. 1355 */ 1356 struct mbuf * 1357 sbcut_locked(struct sockbuf *sb, int len) 1358 { 1359 1360 SOCKBUF_LOCK_ASSERT(sb); 1361 return (sbcut_internal(sb, len)); 1362 } 1363 1364 void 1365 sbdrop(struct sockbuf *sb, int len) 1366 { 1367 struct mbuf *mfree; 1368 1369 SOCKBUF_LOCK(sb); 1370 mfree = sbcut_internal(sb, len); 1371 SOCKBUF_UNLOCK(sb); 1372 1373 m_freem(mfree); 1374 } 1375 1376 struct mbuf * 1377 sbsndptr_noadv(struct sockbuf *sb, uint32_t off, uint32_t *moff) 1378 { 1379 struct mbuf *m; 1380 1381 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 1382 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { 1383 *moff = off; 1384 if (sb->sb_sndptr == NULL) { 1385 sb->sb_sndptr = sb->sb_mb; 1386 sb->sb_sndptroff = 0; 1387 } 1388 return (sb->sb_mb); 1389 } else { 1390 m = sb->sb_sndptr; 1391 off -= sb->sb_sndptroff; 1392 } 1393 *moff = off; 1394 return (m); 1395 } 1396 1397 void 1398 sbsndptr_adv(struct sockbuf *sb, struct mbuf *mb, uint32_t len) 1399 { 1400 /* 1401 * A small copy was done, advance forward the sb_sbsndptr to cover 1402 * it. 1403 */ 1404 struct mbuf *m; 1405 1406 if (mb != sb->sb_sndptr) { 1407 /* Did not copyout at the same mbuf */ 1408 return; 1409 } 1410 m = mb; 1411 while (m && (len > 0)) { 1412 if (len >= m->m_len) { 1413 len -= m->m_len; 1414 if (m->m_next) { 1415 sb->sb_sndptroff += m->m_len; 1416 sb->sb_sndptr = m->m_next; 1417 } 1418 m = m->m_next; 1419 } else { 1420 len = 0; 1421 } 1422 } 1423 } 1424 1425 /* 1426 * Return the first mbuf and the mbuf data offset for the provided 1427 * send offset without changing the "sb_sndptroff" field. 1428 */ 1429 struct mbuf * 1430 sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff) 1431 { 1432 struct mbuf *m; 1433 1434 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 1435 1436 /* 1437 * If the "off" is below the stored offset, which happens on 1438 * retransmits, just use "sb_mb": 1439 */ 1440 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { 1441 m = sb->sb_mb; 1442 } else { 1443 m = sb->sb_sndptr; 1444 off -= sb->sb_sndptroff; 1445 } 1446 while (off > 0 && m != NULL) { 1447 if (off < m->m_len) 1448 break; 1449 off -= m->m_len; 1450 m = m->m_next; 1451 } 1452 *moff = off; 1453 return (m); 1454 } 1455 1456 /* 1457 * Drop a record off the front of a sockbuf and move the next record to the 1458 * front. 1459 */ 1460 void 1461 sbdroprecord_locked(struct sockbuf *sb) 1462 { 1463 struct mbuf *m; 1464 1465 SOCKBUF_LOCK_ASSERT(sb); 1466 1467 m = sb->sb_mb; 1468 if (m) { 1469 sb->sb_mb = m->m_nextpkt; 1470 do { 1471 sbfree(sb, m); 1472 m = m_free(m); 1473 } while (m); 1474 } 1475 SB_EMPTY_FIXUP(sb); 1476 } 1477 1478 /* 1479 * Drop a record off the front of a sockbuf and move the next record to the 1480 * front. 1481 */ 1482 void 1483 sbdroprecord(struct sockbuf *sb) 1484 { 1485 1486 SOCKBUF_LOCK(sb); 1487 sbdroprecord_locked(sb); 1488 SOCKBUF_UNLOCK(sb); 1489 } 1490 1491 /* 1492 * Create a "control" mbuf containing the specified data with the specified 1493 * type for presentation on a socket buffer. 1494 */ 1495 struct mbuf * 1496 sbcreatecontrol(caddr_t p, int size, int type, int level) 1497 { 1498 struct cmsghdr *cp; 1499 struct mbuf *m; 1500 1501 if (CMSG_SPACE((u_int)size) > MCLBYTES) 1502 return ((struct mbuf *) NULL); 1503 if (CMSG_SPACE((u_int)size) > MLEN) 1504 m = m_getcl(M_NOWAIT, MT_CONTROL, 0); 1505 else 1506 m = m_get(M_NOWAIT, MT_CONTROL); 1507 if (m == NULL) 1508 return ((struct mbuf *) NULL); 1509 cp = mtod(m, struct cmsghdr *); 1510 m->m_len = 0; 1511 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m), 1512 ("sbcreatecontrol: short mbuf")); 1513 /* 1514 * Don't leave the padding between the msg header and the 1515 * cmsg data and the padding after the cmsg data un-initialized. 1516 */ 1517 bzero(cp, CMSG_SPACE((u_int)size)); 1518 if (p != NULL) 1519 (void)memcpy(CMSG_DATA(cp), p, size); 1520 m->m_len = CMSG_SPACE(size); 1521 cp->cmsg_len = CMSG_LEN(size); 1522 cp->cmsg_level = level; 1523 cp->cmsg_type = type; 1524 return (m); 1525 } 1526 1527 /* 1528 * This does the same for socket buffers that sotoxsocket does for sockets: 1529 * generate an user-format data structure describing the socket buffer. Note 1530 * that the xsockbuf structure, since it is always embedded in a socket, does 1531 * not include a self pointer nor a length. We make this entry point public 1532 * in case some other mechanism needs it. 1533 */ 1534 void 1535 sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) 1536 { 1537 1538 xsb->sb_cc = sb->sb_ccc; 1539 xsb->sb_hiwat = sb->sb_hiwat; 1540 xsb->sb_mbcnt = sb->sb_mbcnt; 1541 xsb->sb_mcnt = sb->sb_mcnt; 1542 xsb->sb_ccnt = sb->sb_ccnt; 1543 xsb->sb_mbmax = sb->sb_mbmax; 1544 xsb->sb_lowat = sb->sb_lowat; 1545 xsb->sb_flags = sb->sb_flags; 1546 xsb->sb_timeo = sb->sb_timeo; 1547 } 1548 1549 /* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 1550 static int dummy; 1551 SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW | CTLFLAG_SKIP, &dummy, 0, ""); 1552 SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, 1553 CTLTYPE_ULONG | CTLFLAG_RW | CTLFLAG_NEEDGIANT, &sb_max, 0, 1554 sysctl_handle_sb_max, "LU", 1555 "Maximum socket buffer size"); 1556 SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 1557 &sb_efficiency, 0, "Socket buffer size waste factor"); 1558