1 /*- 2 * Copyright (c) 1982, 1986, 1988, 1990, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 4. Neither the name of the University nor the names of its contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * @(#)uipc_socket2.c 8.1 (Berkeley) 6/10/93 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_param.h" 36 37 #include <sys/param.h> 38 #include <sys/aio.h> /* for aio_swake proto */ 39 #include <sys/kernel.h> 40 #include <sys/lock.h> 41 #include <sys/malloc.h> 42 #include <sys/mbuf.h> 43 #include <sys/mutex.h> 44 #include <sys/proc.h> 45 #include <sys/protosw.h> 46 #include <sys/resourcevar.h> 47 #include <sys/signalvar.h> 48 #include <sys/socket.h> 49 #include <sys/socketvar.h> 50 #include <sys/sx.h> 51 #include <sys/sysctl.h> 52 53 /* 54 * Function pointer set by the AIO routines so that the socket buffer code 55 * can call back into the AIO module if it is loaded. 56 */ 57 void (*aio_swake)(struct socket *, struct sockbuf *); 58 59 /* 60 * Primitive routines for operating on socket buffers 61 */ 62 63 u_long sb_max = SB_MAX; 64 u_long sb_max_adj = 65 (quad_t)SB_MAX * MCLBYTES / (MSIZE + MCLBYTES); /* adjusted sb_max */ 66 67 static u_long sb_efficiency = 8; /* parameter for sbreserve() */ 68 69 static struct mbuf *sbcut_internal(struct sockbuf *sb, int len); 70 static void sbflush_internal(struct sockbuf *sb); 71 72 /* 73 * Our own version of m_clrprotoflags(), that can preserve M_NOTREADY. 74 */ 75 static void 76 sbm_clrprotoflags(struct mbuf *m, int flags) 77 { 78 int mask; 79 80 mask = ~M_PROTOFLAGS; 81 if (flags & PRUS_NOTREADY) 82 mask |= M_NOTREADY; 83 while (m) { 84 m->m_flags &= mask; 85 m = m->m_next; 86 } 87 } 88 89 /* 90 * Mark ready "count" mbufs starting with "m". 91 */ 92 int 93 sbready(struct sockbuf *sb, struct mbuf *m, int count) 94 { 95 u_int blocker; 96 97 SOCKBUF_LOCK_ASSERT(sb); 98 KASSERT(sb->sb_fnrdy != NULL, ("%s: sb %p NULL fnrdy", __func__, sb)); 99 100 blocker = (sb->sb_fnrdy == m) ? M_BLOCKED : 0; 101 102 for (int i = 0; i < count; i++, m = m->m_next) { 103 KASSERT(m->m_flags & M_NOTREADY, 104 ("%s: m %p !M_NOTREADY", __func__, m)); 105 m->m_flags &= ~(M_NOTREADY | blocker); 106 if (blocker) 107 sb->sb_acc += m->m_len; 108 } 109 110 if (!blocker) 111 return (EINPROGRESS); 112 113 /* This one was blocking all the queue. */ 114 for (; m && (m->m_flags & M_NOTREADY) == 0; m = m->m_next) { 115 KASSERT(m->m_flags & M_BLOCKED, 116 ("%s: m %p !M_BLOCKED", __func__, m)); 117 m->m_flags &= ~M_BLOCKED; 118 sb->sb_acc += m->m_len; 119 } 120 121 sb->sb_fnrdy = m; 122 123 return (0); 124 } 125 126 /* 127 * Adjust sockbuf state reflecting allocation of m. 128 */ 129 void 130 sballoc(struct sockbuf *sb, struct mbuf *m) 131 { 132 133 SOCKBUF_LOCK_ASSERT(sb); 134 135 sb->sb_ccc += m->m_len; 136 137 if (sb->sb_fnrdy == NULL) { 138 if (m->m_flags & M_NOTREADY) 139 sb->sb_fnrdy = m; 140 else 141 sb->sb_acc += m->m_len; 142 } else 143 m->m_flags |= M_BLOCKED; 144 145 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 146 sb->sb_ctl += m->m_len; 147 148 sb->sb_mbcnt += MSIZE; 149 sb->sb_mcnt += 1; 150 151 if (m->m_flags & M_EXT) { 152 sb->sb_mbcnt += m->m_ext.ext_size; 153 sb->sb_ccnt += 1; 154 } 155 } 156 157 /* 158 * Adjust sockbuf state reflecting freeing of m. 159 */ 160 void 161 sbfree(struct sockbuf *sb, struct mbuf *m) 162 { 163 164 #if 0 /* XXX: not yet: soclose() call path comes here w/o lock. */ 165 SOCKBUF_LOCK_ASSERT(sb); 166 #endif 167 168 sb->sb_ccc -= m->m_len; 169 170 if (!(m->m_flags & M_NOTAVAIL)) 171 sb->sb_acc -= m->m_len; 172 173 if (m == sb->sb_fnrdy) { 174 struct mbuf *n; 175 176 KASSERT(m->m_flags & M_NOTREADY, 177 ("%s: m %p !M_NOTREADY", __func__, m)); 178 179 n = m->m_next; 180 while (n != NULL && !(n->m_flags & M_NOTREADY)) { 181 n->m_flags &= ~M_BLOCKED; 182 sb->sb_acc += n->m_len; 183 n = n->m_next; 184 } 185 sb->sb_fnrdy = n; 186 } 187 188 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 189 sb->sb_ctl -= m->m_len; 190 191 sb->sb_mbcnt -= MSIZE; 192 sb->sb_mcnt -= 1; 193 if (m->m_flags & M_EXT) { 194 sb->sb_mbcnt -= m->m_ext.ext_size; 195 sb->sb_ccnt -= 1; 196 } 197 198 if (sb->sb_sndptr == m) { 199 sb->sb_sndptr = NULL; 200 sb->sb_sndptroff = 0; 201 } 202 if (sb->sb_sndptroff != 0) 203 sb->sb_sndptroff -= m->m_len; 204 } 205 206 /* 207 * Socantsendmore indicates that no more data will be sent on the socket; it 208 * would normally be applied to a socket when the user informs the system 209 * that no more data is to be sent, by the protocol code (in case 210 * PRU_SHUTDOWN). Socantrcvmore indicates that no more data will be 211 * received, and will normally be applied to the socket by a protocol when it 212 * detects that the peer will send no more data. Data queued for reading in 213 * the socket may yet be read. 214 */ 215 void 216 socantsendmore_locked(struct socket *so) 217 { 218 219 SOCKBUF_LOCK_ASSERT(&so->so_snd); 220 221 so->so_snd.sb_state |= SBS_CANTSENDMORE; 222 sowwakeup_locked(so); 223 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 224 } 225 226 void 227 socantsendmore(struct socket *so) 228 { 229 230 SOCKBUF_LOCK(&so->so_snd); 231 socantsendmore_locked(so); 232 mtx_assert(SOCKBUF_MTX(&so->so_snd), MA_NOTOWNED); 233 } 234 235 void 236 socantrcvmore_locked(struct socket *so) 237 { 238 239 SOCKBUF_LOCK_ASSERT(&so->so_rcv); 240 241 so->so_rcv.sb_state |= SBS_CANTRCVMORE; 242 sorwakeup_locked(so); 243 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 244 } 245 246 void 247 socantrcvmore(struct socket *so) 248 { 249 250 SOCKBUF_LOCK(&so->so_rcv); 251 socantrcvmore_locked(so); 252 mtx_assert(SOCKBUF_MTX(&so->so_rcv), MA_NOTOWNED); 253 } 254 255 /* 256 * Wait for data to arrive at/drain from a socket buffer. 257 */ 258 int 259 sbwait(struct sockbuf *sb) 260 { 261 262 SOCKBUF_LOCK_ASSERT(sb); 263 264 sb->sb_flags |= SB_WAIT; 265 return (msleep_sbt(&sb->sb_acc, &sb->sb_mtx, 266 (sb->sb_flags & SB_NOINTR) ? PSOCK : PSOCK | PCATCH, "sbwait", 267 sb->sb_timeo, 0, 0)); 268 } 269 270 int 271 sblock(struct sockbuf *sb, int flags) 272 { 273 274 KASSERT((flags & SBL_VALID) == flags, 275 ("sblock: flags invalid (0x%x)", flags)); 276 277 if (flags & SBL_WAIT) { 278 if ((sb->sb_flags & SB_NOINTR) || 279 (flags & SBL_NOINTR)) { 280 sx_xlock(&sb->sb_sx); 281 return (0); 282 } 283 return (sx_xlock_sig(&sb->sb_sx)); 284 } else { 285 if (sx_try_xlock(&sb->sb_sx) == 0) 286 return (EWOULDBLOCK); 287 return (0); 288 } 289 } 290 291 void 292 sbunlock(struct sockbuf *sb) 293 { 294 295 sx_xunlock(&sb->sb_sx); 296 } 297 298 /* 299 * Wakeup processes waiting on a socket buffer. Do asynchronous notification 300 * via SIGIO if the socket has the SS_ASYNC flag set. 301 * 302 * Called with the socket buffer lock held; will release the lock by the end 303 * of the function. This allows the caller to acquire the socket buffer lock 304 * while testing for the need for various sorts of wakeup and hold it through 305 * to the point where it's no longer required. We currently hold the lock 306 * through calls out to other subsystems (with the exception of kqueue), and 307 * then release it to avoid lock order issues. It's not clear that's 308 * correct. 309 */ 310 void 311 sowakeup(struct socket *so, struct sockbuf *sb) 312 { 313 int ret; 314 315 SOCKBUF_LOCK_ASSERT(sb); 316 317 selwakeuppri(&sb->sb_sel, PSOCK); 318 if (!SEL_WAITING(&sb->sb_sel)) 319 sb->sb_flags &= ~SB_SEL; 320 if (sb->sb_flags & SB_WAIT) { 321 sb->sb_flags &= ~SB_WAIT; 322 wakeup(&sb->sb_acc); 323 } 324 KNOTE_LOCKED(&sb->sb_sel.si_note, 0); 325 if (sb->sb_upcall != NULL) { 326 ret = sb->sb_upcall(so, sb->sb_upcallarg, M_NOWAIT); 327 if (ret == SU_ISCONNECTED) { 328 KASSERT(sb == &so->so_rcv, 329 ("SO_SND upcall returned SU_ISCONNECTED")); 330 soupcall_clear(so, SO_RCV); 331 } 332 } else 333 ret = SU_OK; 334 if (sb->sb_flags & SB_AIO) 335 sowakeup_aio(so, sb); 336 SOCKBUF_UNLOCK(sb); 337 if (ret == SU_ISCONNECTED) 338 soisconnected(so); 339 if ((so->so_state & SS_ASYNC) && so->so_sigio != NULL) 340 pgsigio(&so->so_sigio, SIGIO, 0); 341 mtx_assert(SOCKBUF_MTX(sb), MA_NOTOWNED); 342 } 343 344 /* 345 * Socket buffer (struct sockbuf) utility routines. 346 * 347 * Each socket contains two socket buffers: one for sending data and one for 348 * receiving data. Each buffer contains a queue of mbufs, information about 349 * the number of mbufs and amount of data in the queue, and other fields 350 * allowing select() statements and notification on data availability to be 351 * implemented. 352 * 353 * Data stored in a socket buffer is maintained as a list of records. Each 354 * record is a list of mbufs chained together with the m_next field. Records 355 * are chained together with the m_nextpkt field. The upper level routine 356 * soreceive() expects the following conventions to be observed when placing 357 * information in the receive buffer: 358 * 359 * 1. If the protocol requires each message be preceded by the sender's name, 360 * then a record containing that name must be present before any 361 * associated data (mbuf's must be of type MT_SONAME). 362 * 2. If the protocol supports the exchange of ``access rights'' (really just 363 * additional data associated with the message), and there are ``rights'' 364 * to be received, then a record containing this data should be present 365 * (mbuf's must be of type MT_RIGHTS). 366 * 3. If a name or rights record exists, then it must be followed by a data 367 * record, perhaps of zero length. 368 * 369 * Before using a new socket structure it is first necessary to reserve 370 * buffer space to the socket, by calling sbreserve(). This should commit 371 * some of the available buffer space in the system buffer pool for the 372 * socket (currently, it does nothing but enforce limits). The space should 373 * be released by calling sbrelease() when the socket is destroyed. 374 */ 375 int 376 soreserve(struct socket *so, u_long sndcc, u_long rcvcc) 377 { 378 struct thread *td = curthread; 379 380 SOCKBUF_LOCK(&so->so_snd); 381 SOCKBUF_LOCK(&so->so_rcv); 382 if (sbreserve_locked(&so->so_snd, sndcc, so, td) == 0) 383 goto bad; 384 if (sbreserve_locked(&so->so_rcv, rcvcc, so, td) == 0) 385 goto bad2; 386 if (so->so_rcv.sb_lowat == 0) 387 so->so_rcv.sb_lowat = 1; 388 if (so->so_snd.sb_lowat == 0) 389 so->so_snd.sb_lowat = MCLBYTES; 390 if (so->so_snd.sb_lowat > so->so_snd.sb_hiwat) 391 so->so_snd.sb_lowat = so->so_snd.sb_hiwat; 392 SOCKBUF_UNLOCK(&so->so_rcv); 393 SOCKBUF_UNLOCK(&so->so_snd); 394 return (0); 395 bad2: 396 sbrelease_locked(&so->so_snd, so); 397 bad: 398 SOCKBUF_UNLOCK(&so->so_rcv); 399 SOCKBUF_UNLOCK(&so->so_snd); 400 return (ENOBUFS); 401 } 402 403 static int 404 sysctl_handle_sb_max(SYSCTL_HANDLER_ARGS) 405 { 406 int error = 0; 407 u_long tmp_sb_max = sb_max; 408 409 error = sysctl_handle_long(oidp, &tmp_sb_max, arg2, req); 410 if (error || !req->newptr) 411 return (error); 412 if (tmp_sb_max < MSIZE + MCLBYTES) 413 return (EINVAL); 414 sb_max = tmp_sb_max; 415 sb_max_adj = (u_quad_t)sb_max * MCLBYTES / (MSIZE + MCLBYTES); 416 return (0); 417 } 418 419 /* 420 * Allot mbufs to a sockbuf. Attempt to scale mbmax so that mbcnt doesn't 421 * become limiting if buffering efficiency is near the normal case. 422 */ 423 int 424 sbreserve_locked(struct sockbuf *sb, u_long cc, struct socket *so, 425 struct thread *td) 426 { 427 rlim_t sbsize_limit; 428 429 SOCKBUF_LOCK_ASSERT(sb); 430 431 /* 432 * When a thread is passed, we take into account the thread's socket 433 * buffer size limit. The caller will generally pass curthread, but 434 * in the TCP input path, NULL will be passed to indicate that no 435 * appropriate thread resource limits are available. In that case, 436 * we don't apply a process limit. 437 */ 438 if (cc > sb_max_adj) 439 return (0); 440 if (td != NULL) { 441 sbsize_limit = lim_cur(td, RLIMIT_SBSIZE); 442 } else 443 sbsize_limit = RLIM_INFINITY; 444 if (!chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, cc, 445 sbsize_limit)) 446 return (0); 447 sb->sb_mbmax = min(cc * sb_efficiency, sb_max); 448 if (sb->sb_lowat > sb->sb_hiwat) 449 sb->sb_lowat = sb->sb_hiwat; 450 return (1); 451 } 452 453 int 454 sbreserve(struct sockbuf *sb, u_long cc, struct socket *so, 455 struct thread *td) 456 { 457 int error; 458 459 SOCKBUF_LOCK(sb); 460 error = sbreserve_locked(sb, cc, so, td); 461 SOCKBUF_UNLOCK(sb); 462 return (error); 463 } 464 465 /* 466 * Free mbufs held by a socket, and reserved mbuf space. 467 */ 468 void 469 sbrelease_internal(struct sockbuf *sb, struct socket *so) 470 { 471 472 sbflush_internal(sb); 473 (void)chgsbsize(so->so_cred->cr_uidinfo, &sb->sb_hiwat, 0, 474 RLIM_INFINITY); 475 sb->sb_mbmax = 0; 476 } 477 478 void 479 sbrelease_locked(struct sockbuf *sb, struct socket *so) 480 { 481 482 SOCKBUF_LOCK_ASSERT(sb); 483 484 sbrelease_internal(sb, so); 485 } 486 487 void 488 sbrelease(struct sockbuf *sb, struct socket *so) 489 { 490 491 SOCKBUF_LOCK(sb); 492 sbrelease_locked(sb, so); 493 SOCKBUF_UNLOCK(sb); 494 } 495 496 void 497 sbdestroy(struct sockbuf *sb, struct socket *so) 498 { 499 500 sbrelease_internal(sb, so); 501 } 502 503 /* 504 * Routines to add and remove data from an mbuf queue. 505 * 506 * The routines sbappend() or sbappendrecord() are normally called to append 507 * new mbufs to a socket buffer, after checking that adequate space is 508 * available, comparing the function sbspace() with the amount of data to be 509 * added. sbappendrecord() differs from sbappend() in that data supplied is 510 * treated as the beginning of a new record. To place a sender's address, 511 * optional access rights, and data in a socket receive buffer, 512 * sbappendaddr() should be used. To place access rights and data in a 513 * socket receive buffer, sbappendrights() should be used. In either case, 514 * the new data begins a new record. Note that unlike sbappend() and 515 * sbappendrecord(), these routines check for the caller that there will be 516 * enough space to store the data. Each fails if there is not enough space, 517 * or if it cannot find mbufs to store additional information in. 518 * 519 * Reliable protocols may use the socket send buffer to hold data awaiting 520 * acknowledgement. Data is normally copied from a socket send buffer in a 521 * protocol with m_copy for output to a peer, and then removing the data from 522 * the socket buffer with sbdrop() or sbdroprecord() when the data is 523 * acknowledged by the peer. 524 */ 525 #ifdef SOCKBUF_DEBUG 526 void 527 sblastrecordchk(struct sockbuf *sb, const char *file, int line) 528 { 529 struct mbuf *m = sb->sb_mb; 530 531 SOCKBUF_LOCK_ASSERT(sb); 532 533 while (m && m->m_nextpkt) 534 m = m->m_nextpkt; 535 536 if (m != sb->sb_lastrecord) { 537 printf("%s: sb_mb %p sb_lastrecord %p last %p\n", 538 __func__, sb->sb_mb, sb->sb_lastrecord, m); 539 printf("packet chain:\n"); 540 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) 541 printf("\t%p\n", m); 542 panic("%s from %s:%u", __func__, file, line); 543 } 544 } 545 546 void 547 sblastmbufchk(struct sockbuf *sb, const char *file, int line) 548 { 549 struct mbuf *m = sb->sb_mb; 550 struct mbuf *n; 551 552 SOCKBUF_LOCK_ASSERT(sb); 553 554 while (m && m->m_nextpkt) 555 m = m->m_nextpkt; 556 557 while (m && m->m_next) 558 m = m->m_next; 559 560 if (m != sb->sb_mbtail) { 561 printf("%s: sb_mb %p sb_mbtail %p last %p\n", 562 __func__, sb->sb_mb, sb->sb_mbtail, m); 563 printf("packet tree:\n"); 564 for (m = sb->sb_mb; m != NULL; m = m->m_nextpkt) { 565 printf("\t"); 566 for (n = m; n != NULL; n = n->m_next) 567 printf("%p ", n); 568 printf("\n"); 569 } 570 panic("%s from %s:%u", __func__, file, line); 571 } 572 } 573 #endif /* SOCKBUF_DEBUG */ 574 575 #define SBLINKRECORD(sb, m0) do { \ 576 SOCKBUF_LOCK_ASSERT(sb); \ 577 if ((sb)->sb_lastrecord != NULL) \ 578 (sb)->sb_lastrecord->m_nextpkt = (m0); \ 579 else \ 580 (sb)->sb_mb = (m0); \ 581 (sb)->sb_lastrecord = (m0); \ 582 } while (/*CONSTCOND*/0) 583 584 /* 585 * Append mbuf chain m to the last record in the socket buffer sb. The 586 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 587 * are discarded and mbufs are compacted where possible. 588 */ 589 void 590 sbappend_locked(struct sockbuf *sb, struct mbuf *m, int flags) 591 { 592 struct mbuf *n; 593 594 SOCKBUF_LOCK_ASSERT(sb); 595 596 if (m == 0) 597 return; 598 sbm_clrprotoflags(m, flags); 599 SBLASTRECORDCHK(sb); 600 n = sb->sb_mb; 601 if (n) { 602 while (n->m_nextpkt) 603 n = n->m_nextpkt; 604 do { 605 if (n->m_flags & M_EOR) { 606 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 607 return; 608 } 609 } while (n->m_next && (n = n->m_next)); 610 } else { 611 /* 612 * XXX Would like to simply use sb_mbtail here, but 613 * XXX I need to verify that I won't miss an EOR that 614 * XXX way. 615 */ 616 if ((n = sb->sb_lastrecord) != NULL) { 617 do { 618 if (n->m_flags & M_EOR) { 619 sbappendrecord_locked(sb, m); /* XXXXXX!!!! */ 620 return; 621 } 622 } while (n->m_next && (n = n->m_next)); 623 } else { 624 /* 625 * If this is the first record in the socket buffer, 626 * it's also the last record. 627 */ 628 sb->sb_lastrecord = m; 629 } 630 } 631 sbcompress(sb, m, n); 632 SBLASTRECORDCHK(sb); 633 } 634 635 /* 636 * Append mbuf chain m to the last record in the socket buffer sb. The 637 * additional space associated the mbuf chain is recorded in sb. Empty mbufs 638 * are discarded and mbufs are compacted where possible. 639 */ 640 void 641 sbappend(struct sockbuf *sb, struct mbuf *m, int flags) 642 { 643 644 SOCKBUF_LOCK(sb); 645 sbappend_locked(sb, m, flags); 646 SOCKBUF_UNLOCK(sb); 647 } 648 649 /* 650 * This version of sbappend() should only be used when the caller absolutely 651 * knows that there will never be more than one record in the socket buffer, 652 * that is, a stream protocol (such as TCP). 653 */ 654 void 655 sbappendstream_locked(struct sockbuf *sb, struct mbuf *m, int flags) 656 { 657 SOCKBUF_LOCK_ASSERT(sb); 658 659 KASSERT(m->m_nextpkt == NULL,("sbappendstream 0")); 660 KASSERT(sb->sb_mb == sb->sb_lastrecord,("sbappendstream 1")); 661 662 SBLASTMBUFCHK(sb); 663 664 /* Remove all packet headers and mbuf tags to get a pure data chain. */ 665 m_demote(m, 1, flags & PRUS_NOTREADY ? M_NOTREADY : 0); 666 667 sbcompress(sb, m, sb->sb_mbtail); 668 669 sb->sb_lastrecord = sb->sb_mb; 670 SBLASTRECORDCHK(sb); 671 } 672 673 /* 674 * This version of sbappend() should only be used when the caller absolutely 675 * knows that there will never be more than one record in the socket buffer, 676 * that is, a stream protocol (such as TCP). 677 */ 678 void 679 sbappendstream(struct sockbuf *sb, struct mbuf *m, int flags) 680 { 681 682 SOCKBUF_LOCK(sb); 683 sbappendstream_locked(sb, m, flags); 684 SOCKBUF_UNLOCK(sb); 685 } 686 687 #ifdef SOCKBUF_DEBUG 688 void 689 sbcheck(struct sockbuf *sb, const char *file, int line) 690 { 691 struct mbuf *m, *n, *fnrdy; 692 u_long acc, ccc, mbcnt; 693 694 SOCKBUF_LOCK_ASSERT(sb); 695 696 acc = ccc = mbcnt = 0; 697 fnrdy = NULL; 698 699 for (m = sb->sb_mb; m; m = n) { 700 n = m->m_nextpkt; 701 for (; m; m = m->m_next) { 702 if (m->m_len == 0) { 703 printf("sb %p empty mbuf %p\n", sb, m); 704 goto fail; 705 } 706 if ((m->m_flags & M_NOTREADY) && fnrdy == NULL) { 707 if (m != sb->sb_fnrdy) { 708 printf("sb %p: fnrdy %p != m %p\n", 709 sb, sb->sb_fnrdy, m); 710 goto fail; 711 } 712 fnrdy = m; 713 } 714 if (fnrdy) { 715 if (!(m->m_flags & M_NOTAVAIL)) { 716 printf("sb %p: fnrdy %p, m %p is avail\n", 717 sb, sb->sb_fnrdy, m); 718 goto fail; 719 } 720 } else 721 acc += m->m_len; 722 ccc += m->m_len; 723 mbcnt += MSIZE; 724 if (m->m_flags & M_EXT) /*XXX*/ /* pretty sure this is bogus */ 725 mbcnt += m->m_ext.ext_size; 726 } 727 } 728 if (acc != sb->sb_acc || ccc != sb->sb_ccc || mbcnt != sb->sb_mbcnt) { 729 printf("acc %ld/%u ccc %ld/%u mbcnt %ld/%u\n", 730 acc, sb->sb_acc, ccc, sb->sb_ccc, mbcnt, sb->sb_mbcnt); 731 goto fail; 732 } 733 return; 734 fail: 735 panic("%s from %s:%u", __func__, file, line); 736 } 737 #endif 738 739 /* 740 * As above, except the mbuf chain begins a new record. 741 */ 742 void 743 sbappendrecord_locked(struct sockbuf *sb, struct mbuf *m0) 744 { 745 struct mbuf *m; 746 747 SOCKBUF_LOCK_ASSERT(sb); 748 749 if (m0 == 0) 750 return; 751 m_clrprotoflags(m0); 752 /* 753 * Put the first mbuf on the queue. Note this permits zero length 754 * records. 755 */ 756 sballoc(sb, m0); 757 SBLASTRECORDCHK(sb); 758 SBLINKRECORD(sb, m0); 759 sb->sb_mbtail = m0; 760 m = m0->m_next; 761 m0->m_next = 0; 762 if (m && (m0->m_flags & M_EOR)) { 763 m0->m_flags &= ~M_EOR; 764 m->m_flags |= M_EOR; 765 } 766 /* always call sbcompress() so it can do SBLASTMBUFCHK() */ 767 sbcompress(sb, m, m0); 768 } 769 770 /* 771 * As above, except the mbuf chain begins a new record. 772 */ 773 void 774 sbappendrecord(struct sockbuf *sb, struct mbuf *m0) 775 { 776 777 SOCKBUF_LOCK(sb); 778 sbappendrecord_locked(sb, m0); 779 SOCKBUF_UNLOCK(sb); 780 } 781 782 /* Helper routine that appends data, control, and address to a sockbuf. */ 783 static int 784 sbappendaddr_locked_internal(struct sockbuf *sb, const struct sockaddr *asa, 785 struct mbuf *m0, struct mbuf *control, struct mbuf *ctrl_last) 786 { 787 struct mbuf *m, *n, *nlast; 788 #if MSIZE <= 256 789 if (asa->sa_len > MLEN) 790 return (0); 791 #endif 792 m = m_get(M_NOWAIT, MT_SONAME); 793 if (m == NULL) 794 return (0); 795 m->m_len = asa->sa_len; 796 bcopy(asa, mtod(m, caddr_t), asa->sa_len); 797 if (m0) 798 m_clrprotoflags(m0); 799 if (ctrl_last) 800 ctrl_last->m_next = m0; /* concatenate data to control */ 801 else 802 control = m0; 803 m->m_next = control; 804 for (n = m; n->m_next != NULL; n = n->m_next) 805 sballoc(sb, n); 806 sballoc(sb, n); 807 nlast = n; 808 SBLINKRECORD(sb, m); 809 810 sb->sb_mbtail = nlast; 811 SBLASTMBUFCHK(sb); 812 813 SBLASTRECORDCHK(sb); 814 return (1); 815 } 816 817 /* 818 * Append address and data, and optionally, control (ancillary) data to the 819 * receive queue of a socket. If present, m0 must include a packet header 820 * with total length. Returns 0 if no space in sockbuf or insufficient 821 * mbufs. 822 */ 823 int 824 sbappendaddr_locked(struct sockbuf *sb, const struct sockaddr *asa, 825 struct mbuf *m0, struct mbuf *control) 826 { 827 struct mbuf *ctrl_last; 828 int space = asa->sa_len; 829 830 SOCKBUF_LOCK_ASSERT(sb); 831 832 if (m0 && (m0->m_flags & M_PKTHDR) == 0) 833 panic("sbappendaddr_locked"); 834 if (m0) 835 space += m0->m_pkthdr.len; 836 space += m_length(control, &ctrl_last); 837 838 if (space > sbspace(sb)) 839 return (0); 840 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 841 } 842 843 /* 844 * Append address and data, and optionally, control (ancillary) data to the 845 * receive queue of a socket. If present, m0 must include a packet header 846 * with total length. Returns 0 if insufficient mbufs. Does not validate space 847 * on the receiving sockbuf. 848 */ 849 int 850 sbappendaddr_nospacecheck_locked(struct sockbuf *sb, const struct sockaddr *asa, 851 struct mbuf *m0, struct mbuf *control) 852 { 853 struct mbuf *ctrl_last; 854 855 SOCKBUF_LOCK_ASSERT(sb); 856 857 ctrl_last = (control == NULL) ? NULL : m_last(control); 858 return (sbappendaddr_locked_internal(sb, asa, m0, control, ctrl_last)); 859 } 860 861 /* 862 * Append address and data, and optionally, control (ancillary) data to the 863 * receive queue of a socket. If present, m0 must include a packet header 864 * with total length. Returns 0 if no space in sockbuf or insufficient 865 * mbufs. 866 */ 867 int 868 sbappendaddr(struct sockbuf *sb, const struct sockaddr *asa, 869 struct mbuf *m0, struct mbuf *control) 870 { 871 int retval; 872 873 SOCKBUF_LOCK(sb); 874 retval = sbappendaddr_locked(sb, asa, m0, control); 875 SOCKBUF_UNLOCK(sb); 876 return (retval); 877 } 878 879 int 880 sbappendcontrol_locked(struct sockbuf *sb, struct mbuf *m0, 881 struct mbuf *control) 882 { 883 struct mbuf *m, *n, *mlast; 884 int space; 885 886 SOCKBUF_LOCK_ASSERT(sb); 887 888 if (control == 0) 889 panic("sbappendcontrol_locked"); 890 space = m_length(control, &n) + m_length(m0, NULL); 891 892 if (space > sbspace(sb)) 893 return (0); 894 m_clrprotoflags(m0); 895 n->m_next = m0; /* concatenate data to control */ 896 897 SBLASTRECORDCHK(sb); 898 899 for (m = control; m->m_next; m = m->m_next) 900 sballoc(sb, m); 901 sballoc(sb, m); 902 mlast = m; 903 SBLINKRECORD(sb, control); 904 905 sb->sb_mbtail = mlast; 906 SBLASTMBUFCHK(sb); 907 908 SBLASTRECORDCHK(sb); 909 return (1); 910 } 911 912 int 913 sbappendcontrol(struct sockbuf *sb, struct mbuf *m0, struct mbuf *control) 914 { 915 int retval; 916 917 SOCKBUF_LOCK(sb); 918 retval = sbappendcontrol_locked(sb, m0, control); 919 SOCKBUF_UNLOCK(sb); 920 return (retval); 921 } 922 923 /* 924 * Append the data in mbuf chain (m) into the socket buffer sb following mbuf 925 * (n). If (n) is NULL, the buffer is presumed empty. 926 * 927 * When the data is compressed, mbufs in the chain may be handled in one of 928 * three ways: 929 * 930 * (1) The mbuf may simply be dropped, if it contributes nothing (no data, no 931 * record boundary, and no change in data type). 932 * 933 * (2) The mbuf may be coalesced -- i.e., data in the mbuf may be copied into 934 * an mbuf already in the socket buffer. This can occur if an 935 * appropriate mbuf exists, there is room, both mbufs are not marked as 936 * not ready, and no merging of data types will occur. 937 * 938 * (3) The mbuf may be appended to the end of the existing mbuf chain. 939 * 940 * If any of the new mbufs is marked as M_EOR, mark the last mbuf appended as 941 * end-of-record. 942 */ 943 void 944 sbcompress(struct sockbuf *sb, struct mbuf *m, struct mbuf *n) 945 { 946 int eor = 0; 947 struct mbuf *o; 948 949 SOCKBUF_LOCK_ASSERT(sb); 950 951 while (m) { 952 eor |= m->m_flags & M_EOR; 953 if (m->m_len == 0 && 954 (eor == 0 || 955 (((o = m->m_next) || (o = n)) && 956 o->m_type == m->m_type))) { 957 if (sb->sb_lastrecord == m) 958 sb->sb_lastrecord = m->m_next; 959 m = m_free(m); 960 continue; 961 } 962 if (n && (n->m_flags & M_EOR) == 0 && 963 M_WRITABLE(n) && 964 ((sb->sb_flags & SB_NOCOALESCE) == 0) && 965 !(m->m_flags & M_NOTREADY) && 966 !(n->m_flags & M_NOTREADY) && 967 m->m_len <= MCLBYTES / 4 && /* XXX: Don't copy too much */ 968 m->m_len <= M_TRAILINGSPACE(n) && 969 n->m_type == m->m_type) { 970 bcopy(mtod(m, caddr_t), mtod(n, caddr_t) + n->m_len, 971 (unsigned)m->m_len); 972 n->m_len += m->m_len; 973 sb->sb_ccc += m->m_len; 974 if (sb->sb_fnrdy == NULL) 975 sb->sb_acc += m->m_len; 976 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 977 /* XXX: Probably don't need.*/ 978 sb->sb_ctl += m->m_len; 979 m = m_free(m); 980 continue; 981 } 982 if (n) 983 n->m_next = m; 984 else 985 sb->sb_mb = m; 986 sb->sb_mbtail = m; 987 sballoc(sb, m); 988 n = m; 989 m->m_flags &= ~M_EOR; 990 m = m->m_next; 991 n->m_next = 0; 992 } 993 if (eor) { 994 KASSERT(n != NULL, ("sbcompress: eor && n == NULL")); 995 n->m_flags |= eor; 996 } 997 SBLASTMBUFCHK(sb); 998 } 999 1000 /* 1001 * Free all mbufs in a sockbuf. Check that all resources are reclaimed. 1002 */ 1003 static void 1004 sbflush_internal(struct sockbuf *sb) 1005 { 1006 1007 while (sb->sb_mbcnt) { 1008 /* 1009 * Don't call sbcut(sb, 0) if the leading mbuf is non-empty: 1010 * we would loop forever. Panic instead. 1011 */ 1012 if (sb->sb_ccc == 0 && (sb->sb_mb == NULL || sb->sb_mb->m_len)) 1013 break; 1014 m_freem(sbcut_internal(sb, (int)sb->sb_ccc)); 1015 } 1016 KASSERT(sb->sb_ccc == 0 && sb->sb_mb == 0 && sb->sb_mbcnt == 0, 1017 ("%s: ccc %u mb %p mbcnt %u", __func__, 1018 sb->sb_ccc, (void *)sb->sb_mb, sb->sb_mbcnt)); 1019 } 1020 1021 void 1022 sbflush_locked(struct sockbuf *sb) 1023 { 1024 1025 SOCKBUF_LOCK_ASSERT(sb); 1026 sbflush_internal(sb); 1027 } 1028 1029 void 1030 sbflush(struct sockbuf *sb) 1031 { 1032 1033 SOCKBUF_LOCK(sb); 1034 sbflush_locked(sb); 1035 SOCKBUF_UNLOCK(sb); 1036 } 1037 1038 /* 1039 * Cut data from (the front of) a sockbuf. 1040 */ 1041 static struct mbuf * 1042 sbcut_internal(struct sockbuf *sb, int len) 1043 { 1044 struct mbuf *m, *next, *mfree; 1045 1046 next = (m = sb->sb_mb) ? m->m_nextpkt : 0; 1047 mfree = NULL; 1048 1049 while (len > 0) { 1050 if (m == NULL) { 1051 KASSERT(next, ("%s: no next, len %d", __func__, len)); 1052 m = next; 1053 next = m->m_nextpkt; 1054 } 1055 if (m->m_len > len) { 1056 KASSERT(!(m->m_flags & M_NOTAVAIL), 1057 ("%s: m %p M_NOTAVAIL", __func__, m)); 1058 m->m_len -= len; 1059 m->m_data += len; 1060 sb->sb_ccc -= len; 1061 sb->sb_acc -= len; 1062 if (sb->sb_sndptroff != 0) 1063 sb->sb_sndptroff -= len; 1064 if (m->m_type != MT_DATA && m->m_type != MT_OOBDATA) 1065 sb->sb_ctl -= len; 1066 break; 1067 } 1068 len -= m->m_len; 1069 sbfree(sb, m); 1070 /* 1071 * Do not put M_NOTREADY buffers to the free list, they 1072 * are referenced from outside. 1073 */ 1074 if (m->m_flags & M_NOTREADY) 1075 m = m->m_next; 1076 else { 1077 struct mbuf *n; 1078 1079 n = m->m_next; 1080 m->m_next = mfree; 1081 mfree = m; 1082 m = n; 1083 } 1084 } 1085 /* 1086 * Free any zero-length mbufs from the buffer. 1087 * For SOCK_DGRAM sockets such mbufs represent empty records. 1088 * XXX: For SOCK_STREAM sockets such mbufs can appear in the buffer, 1089 * when sosend_generic() needs to send only control data. 1090 */ 1091 while (m && m->m_len == 0) { 1092 struct mbuf *n; 1093 1094 sbfree(sb, m); 1095 n = m->m_next; 1096 m->m_next = mfree; 1097 mfree = m; 1098 m = n; 1099 } 1100 if (m) { 1101 sb->sb_mb = m; 1102 m->m_nextpkt = next; 1103 } else 1104 sb->sb_mb = next; 1105 /* 1106 * First part is an inline SB_EMPTY_FIXUP(). Second part makes sure 1107 * sb_lastrecord is up-to-date if we dropped part of the last record. 1108 */ 1109 m = sb->sb_mb; 1110 if (m == NULL) { 1111 sb->sb_mbtail = NULL; 1112 sb->sb_lastrecord = NULL; 1113 } else if (m->m_nextpkt == NULL) { 1114 sb->sb_lastrecord = m; 1115 } 1116 1117 return (mfree); 1118 } 1119 1120 /* 1121 * Drop data from (the front of) a sockbuf. 1122 */ 1123 void 1124 sbdrop_locked(struct sockbuf *sb, int len) 1125 { 1126 1127 SOCKBUF_LOCK_ASSERT(sb); 1128 m_freem(sbcut_internal(sb, len)); 1129 } 1130 1131 /* 1132 * Drop data from (the front of) a sockbuf, 1133 * and return it to caller. 1134 */ 1135 struct mbuf * 1136 sbcut_locked(struct sockbuf *sb, int len) 1137 { 1138 1139 SOCKBUF_LOCK_ASSERT(sb); 1140 return (sbcut_internal(sb, len)); 1141 } 1142 1143 void 1144 sbdrop(struct sockbuf *sb, int len) 1145 { 1146 struct mbuf *mfree; 1147 1148 SOCKBUF_LOCK(sb); 1149 mfree = sbcut_internal(sb, len); 1150 SOCKBUF_UNLOCK(sb); 1151 1152 m_freem(mfree); 1153 } 1154 1155 /* 1156 * Maintain a pointer and offset pair into the socket buffer mbuf chain to 1157 * avoid traversal of the entire socket buffer for larger offsets. 1158 */ 1159 struct mbuf * 1160 sbsndptr(struct sockbuf *sb, u_int off, u_int len, u_int *moff) 1161 { 1162 struct mbuf *m, *ret; 1163 1164 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 1165 KASSERT(off + len <= sb->sb_acc, ("%s: beyond sb", __func__)); 1166 KASSERT(sb->sb_sndptroff <= sb->sb_acc, ("%s: sndptroff broken", __func__)); 1167 1168 /* 1169 * Is off below stored offset? Happens on retransmits. 1170 * Just return, we can't help here. 1171 */ 1172 if (sb->sb_sndptroff > off) { 1173 *moff = off; 1174 return (sb->sb_mb); 1175 } 1176 1177 /* Return closest mbuf in chain for current offset. */ 1178 *moff = off - sb->sb_sndptroff; 1179 m = ret = sb->sb_sndptr ? sb->sb_sndptr : sb->sb_mb; 1180 if (*moff == m->m_len) { 1181 *moff = 0; 1182 sb->sb_sndptroff += m->m_len; 1183 m = ret = m->m_next; 1184 KASSERT(ret->m_len > 0, 1185 ("mbuf %p in sockbuf %p chain has no valid data", ret, sb)); 1186 } 1187 1188 /* Advance by len to be as close as possible for the next transmit. */ 1189 for (off = off - sb->sb_sndptroff + len - 1; 1190 off > 0 && m != NULL && off >= m->m_len; 1191 m = m->m_next) { 1192 sb->sb_sndptroff += m->m_len; 1193 off -= m->m_len; 1194 } 1195 if (off > 0 && m == NULL) 1196 panic("%s: sockbuf %p and mbuf %p clashing", __func__, sb, ret); 1197 sb->sb_sndptr = m; 1198 1199 return (ret); 1200 } 1201 1202 /* 1203 * Return the first mbuf and the mbuf data offset for the provided 1204 * send offset without changing the "sb_sndptroff" field. 1205 */ 1206 struct mbuf * 1207 sbsndmbuf(struct sockbuf *sb, u_int off, u_int *moff) 1208 { 1209 struct mbuf *m; 1210 1211 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb is NULL", __func__)); 1212 1213 /* 1214 * If the "off" is below the stored offset, which happens on 1215 * retransmits, just use "sb_mb": 1216 */ 1217 if (sb->sb_sndptr == NULL || sb->sb_sndptroff > off) { 1218 m = sb->sb_mb; 1219 } else { 1220 m = sb->sb_sndptr; 1221 off -= sb->sb_sndptroff; 1222 } 1223 while (off > 0 && m != NULL) { 1224 if (off < m->m_len) 1225 break; 1226 off -= m->m_len; 1227 m = m->m_next; 1228 } 1229 *moff = off; 1230 return (m); 1231 } 1232 1233 /* 1234 * Drop a record off the front of a sockbuf and move the next record to the 1235 * front. 1236 */ 1237 void 1238 sbdroprecord_locked(struct sockbuf *sb) 1239 { 1240 struct mbuf *m; 1241 1242 SOCKBUF_LOCK_ASSERT(sb); 1243 1244 m = sb->sb_mb; 1245 if (m) { 1246 sb->sb_mb = m->m_nextpkt; 1247 do { 1248 sbfree(sb, m); 1249 m = m_free(m); 1250 } while (m); 1251 } 1252 SB_EMPTY_FIXUP(sb); 1253 } 1254 1255 /* 1256 * Drop a record off the front of a sockbuf and move the next record to the 1257 * front. 1258 */ 1259 void 1260 sbdroprecord(struct sockbuf *sb) 1261 { 1262 1263 SOCKBUF_LOCK(sb); 1264 sbdroprecord_locked(sb); 1265 SOCKBUF_UNLOCK(sb); 1266 } 1267 1268 /* 1269 * Create a "control" mbuf containing the specified data with the specified 1270 * type for presentation on a socket buffer. 1271 */ 1272 struct mbuf * 1273 sbcreatecontrol(caddr_t p, int size, int type, int level) 1274 { 1275 struct cmsghdr *cp; 1276 struct mbuf *m; 1277 1278 if (CMSG_SPACE((u_int)size) > MCLBYTES) 1279 return ((struct mbuf *) NULL); 1280 if (CMSG_SPACE((u_int)size) > MLEN) 1281 m = m_getcl(M_NOWAIT, MT_CONTROL, 0); 1282 else 1283 m = m_get(M_NOWAIT, MT_CONTROL); 1284 if (m == NULL) 1285 return ((struct mbuf *) NULL); 1286 cp = mtod(m, struct cmsghdr *); 1287 m->m_len = 0; 1288 KASSERT(CMSG_SPACE((u_int)size) <= M_TRAILINGSPACE(m), 1289 ("sbcreatecontrol: short mbuf")); 1290 /* 1291 * Don't leave the padding between the msg header and the 1292 * cmsg data and the padding after the cmsg data un-initialized. 1293 */ 1294 bzero(cp, CMSG_SPACE((u_int)size)); 1295 if (p != NULL) 1296 (void)memcpy(CMSG_DATA(cp), p, size); 1297 m->m_len = CMSG_SPACE(size); 1298 cp->cmsg_len = CMSG_LEN(size); 1299 cp->cmsg_level = level; 1300 cp->cmsg_type = type; 1301 return (m); 1302 } 1303 1304 /* 1305 * This does the same for socket buffers that sotoxsocket does for sockets: 1306 * generate an user-format data structure describing the socket buffer. Note 1307 * that the xsockbuf structure, since it is always embedded in a socket, does 1308 * not include a self pointer nor a length. We make this entry point public 1309 * in case some other mechanism needs it. 1310 */ 1311 void 1312 sbtoxsockbuf(struct sockbuf *sb, struct xsockbuf *xsb) 1313 { 1314 1315 xsb->sb_cc = sb->sb_ccc; 1316 xsb->sb_hiwat = sb->sb_hiwat; 1317 xsb->sb_mbcnt = sb->sb_mbcnt; 1318 xsb->sb_mcnt = sb->sb_mcnt; 1319 xsb->sb_ccnt = sb->sb_ccnt; 1320 xsb->sb_mbmax = sb->sb_mbmax; 1321 xsb->sb_lowat = sb->sb_lowat; 1322 xsb->sb_flags = sb->sb_flags; 1323 xsb->sb_timeo = sb->sb_timeo; 1324 } 1325 1326 /* This takes the place of kern.maxsockbuf, which moved to kern.ipc. */ 1327 static int dummy; 1328 SYSCTL_INT(_kern, KERN_DUMMY, dummy, CTLFLAG_RW, &dummy, 0, ""); 1329 SYSCTL_OID(_kern_ipc, KIPC_MAXSOCKBUF, maxsockbuf, CTLTYPE_ULONG|CTLFLAG_RW, 1330 &sb_max, 0, sysctl_handle_sb_max, "LU", "Maximum socket buffer size"); 1331 SYSCTL_ULONG(_kern_ipc, KIPC_SOCKBUF_WASTE, sockbuf_waste_factor, CTLFLAG_RW, 1332 &sb_efficiency, 0, "Socket buffer size waste factor"); 1333