1 /* 2 * Copyright (c) 1982, 1986, 1988, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 34 * $FreeBSD$ 35 */ 36 37 #include "opt_param.h" 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/lock.h> 42 #include <sys/malloc.h> 43 #include <sys/mbuf.h> 44 #include <sys/sysctl.h> 45 #include <sys/domain.h> 46 #include <sys/protosw.h> 47 48 int max_linkhdr; 49 int max_protohdr; 50 int max_hdr; 51 int max_datalen; 52 53 /* 54 * sysctl(8) exported objects 55 */ 56 SYSCTL_DECL(_kern_ipc); 57 SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RW, 58 &max_linkhdr, 0, ""); 59 SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RW, 60 &max_protohdr, 0, ""); 61 SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RW, &max_hdr, 0, ""); 62 SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RW, 63 &max_datalen, 0, ""); 64 65 /* 66 * Lesser-used path for M_PREPEND: 67 * allocate new mbuf to prepend to chain, 68 * copy junk along. 69 */ 70 struct mbuf * 71 m_prepend(struct mbuf *m, int len, int how) 72 { 73 struct mbuf *mn; 74 75 MGET(mn, how, m->m_type); 76 if (mn == NULL) { 77 m_freem(m); 78 return (NULL); 79 } 80 if (m->m_flags & M_PKTHDR) { 81 M_COPY_PKTHDR(mn, m); 82 m->m_flags &= ~M_PKTHDR; 83 } 84 mn->m_next = m; 85 m = mn; 86 if (len < MHLEN) 87 MH_ALIGN(m, len); 88 m->m_len = len; 89 return (m); 90 } 91 92 /* 93 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 94 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 95 * The wait parameter is a choice of M_TRYWAIT/M_DONTWAIT from caller. 96 * Note that the copy is read-only, because clusters are not copied, 97 * only their reference counts are incremented. 98 */ 99 struct mbuf * 100 m_copym(struct mbuf *m, int off0, int len, int wait) 101 { 102 struct mbuf *n, **np; 103 int off = off0; 104 struct mbuf *top; 105 int copyhdr = 0; 106 107 KASSERT(off >= 0, ("m_copym, negative off %d", off)); 108 KASSERT(len >= 0, ("m_copym, negative len %d", len)); 109 if (off == 0 && m->m_flags & M_PKTHDR) 110 copyhdr = 1; 111 while (off > 0) { 112 KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 113 if (off < m->m_len) 114 break; 115 off -= m->m_len; 116 m = m->m_next; 117 } 118 np = ⊤ 119 top = 0; 120 while (len > 0) { 121 if (m == NULL) { 122 KASSERT(len == M_COPYALL, 123 ("m_copym, length > size of mbuf chain")); 124 break; 125 } 126 MGET(n, wait, m->m_type); 127 *np = n; 128 if (n == NULL) 129 goto nospace; 130 if (copyhdr) { 131 M_COPY_PKTHDR(n, m); 132 if (len == M_COPYALL) 133 n->m_pkthdr.len -= off0; 134 else 135 n->m_pkthdr.len = len; 136 copyhdr = 0; 137 } 138 n->m_len = min(len, m->m_len - off); 139 if (m->m_flags & M_EXT) { 140 n->m_data = m->m_data + off; 141 n->m_ext = m->m_ext; 142 n->m_flags |= M_EXT; 143 MEXT_ADD_REF(m); 144 } else 145 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 146 (unsigned)n->m_len); 147 if (len != M_COPYALL) 148 len -= n->m_len; 149 off = 0; 150 m = m->m_next; 151 np = &n->m_next; 152 } 153 if (top == NULL) 154 mbstat.m_mcfail++; /* XXX: No consistency. */ 155 156 return (top); 157 nospace: 158 m_freem(top); 159 mbstat.m_mcfail++; /* XXX: No consistency. */ 160 return (NULL); 161 } 162 163 /* 164 * Copy an entire packet, including header (which must be present). 165 * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 166 * Note that the copy is read-only, because clusters are not copied, 167 * only their reference counts are incremented. 168 * Preserve alignment of the first mbuf so if the creator has left 169 * some room at the beginning (e.g. for inserting protocol headers) 170 * the copies still have the room available. 171 */ 172 struct mbuf * 173 m_copypacket(struct mbuf *m, int how) 174 { 175 struct mbuf *top, *n, *o; 176 177 MGET(n, how, m->m_type); 178 top = n; 179 if (n == NULL) 180 goto nospace; 181 182 M_COPY_PKTHDR(n, m); 183 n->m_len = m->m_len; 184 if (m->m_flags & M_EXT) { 185 n->m_data = m->m_data; 186 n->m_ext = m->m_ext; 187 n->m_flags |= M_EXT; 188 MEXT_ADD_REF(m); 189 } else { 190 n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 191 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 192 } 193 194 m = m->m_next; 195 while (m) { 196 MGET(o, how, m->m_type); 197 if (o == NULL) 198 goto nospace; 199 200 n->m_next = o; 201 n = n->m_next; 202 203 n->m_len = m->m_len; 204 if (m->m_flags & M_EXT) { 205 n->m_data = m->m_data; 206 n->m_ext = m->m_ext; 207 n->m_flags |= M_EXT; 208 MEXT_ADD_REF(m); 209 } else { 210 bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 211 } 212 213 m = m->m_next; 214 } 215 return top; 216 nospace: 217 m_freem(top); 218 mbstat.m_mcfail++; /* XXX: No consistency. */ 219 return (NULL); 220 } 221 222 /* 223 * Copy data from an mbuf chain starting "off" bytes from the beginning, 224 * continuing for "len" bytes, into the indicated buffer. 225 */ 226 void 227 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) 228 { 229 unsigned count; 230 231 KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 232 KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 233 while (off > 0) { 234 KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 235 if (off < m->m_len) 236 break; 237 off -= m->m_len; 238 m = m->m_next; 239 } 240 while (len > 0) { 241 KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 242 count = min(m->m_len - off, len); 243 bcopy(mtod(m, caddr_t) + off, cp, count); 244 len -= count; 245 cp += count; 246 off = 0; 247 m = m->m_next; 248 } 249 } 250 251 /* 252 * Copy a packet header mbuf chain into a completely new chain, including 253 * copying any mbuf clusters. Use this instead of m_copypacket() when 254 * you need a writable copy of an mbuf chain. 255 */ 256 struct mbuf * 257 m_dup(struct mbuf *m, int how) 258 { 259 struct mbuf **p, *top = NULL; 260 int remain, moff, nsize; 261 262 /* Sanity check */ 263 if (m == NULL) 264 return (NULL); 265 KASSERT((m->m_flags & M_PKTHDR) != 0, ("%s: !PKTHDR", __func__)); 266 267 /* While there's more data, get a new mbuf, tack it on, and fill it */ 268 remain = m->m_pkthdr.len; 269 moff = 0; 270 p = ⊤ 271 while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 272 struct mbuf *n; 273 274 /* Get the next new mbuf */ 275 MGET(n, how, m->m_type); 276 if (n == NULL) 277 goto nospace; 278 if (top == NULL) { /* first one, must be PKTHDR */ 279 M_COPY_PKTHDR(n, m); 280 nsize = MHLEN; 281 } else /* not the first one */ 282 nsize = MLEN; 283 if (remain >= MINCLSIZE) { 284 MCLGET(n, how); 285 if ((n->m_flags & M_EXT) == 0) { 286 (void)m_free(n); 287 goto nospace; 288 } 289 nsize = MCLBYTES; 290 } 291 n->m_len = 0; 292 293 /* Link it into the new chain */ 294 *p = n; 295 p = &n->m_next; 296 297 /* Copy data from original mbuf(s) into new mbuf */ 298 while (n->m_len < nsize && m != NULL) { 299 int chunk = min(nsize - n->m_len, m->m_len - moff); 300 301 bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 302 moff += chunk; 303 n->m_len += chunk; 304 remain -= chunk; 305 if (moff == m->m_len) { 306 m = m->m_next; 307 moff = 0; 308 } 309 } 310 311 /* Check correct total mbuf length */ 312 KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 313 ("%s: bogus m_pkthdr.len", __func__)); 314 } 315 return (top); 316 317 nospace: 318 m_freem(top); 319 mbstat.m_mcfail++; /* XXX: No consistency. */ 320 return (NULL); 321 } 322 323 /* 324 * Concatenate mbuf chain n to m. 325 * Both chains must be of the same type (e.g. MT_DATA). 326 * Any m_pkthdr is not updated. 327 */ 328 void 329 m_cat(struct mbuf *m, struct mbuf *n) 330 { 331 while (m->m_next) 332 m = m->m_next; 333 while (n) { 334 if (m->m_flags & M_EXT || 335 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 336 /* just join the two chains */ 337 m->m_next = n; 338 return; 339 } 340 /* splat the data from one into the other */ 341 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 342 (u_int)n->m_len); 343 m->m_len += n->m_len; 344 n = m_free(n); 345 } 346 } 347 348 void 349 m_adj(struct mbuf *mp, int req_len) 350 { 351 int len = req_len; 352 struct mbuf *m; 353 int count; 354 355 if ((m = mp) == NULL) 356 return; 357 if (len >= 0) { 358 /* 359 * Trim from head. 360 */ 361 while (m != NULL && len > 0) { 362 if (m->m_len <= len) { 363 len -= m->m_len; 364 m->m_len = 0; 365 m = m->m_next; 366 } else { 367 m->m_len -= len; 368 m->m_data += len; 369 len = 0; 370 } 371 } 372 m = mp; 373 if (mp->m_flags & M_PKTHDR) 374 m->m_pkthdr.len -= (req_len - len); 375 } else { 376 /* 377 * Trim from tail. Scan the mbuf chain, 378 * calculating its length and finding the last mbuf. 379 * If the adjustment only affects this mbuf, then just 380 * adjust and return. Otherwise, rescan and truncate 381 * after the remaining size. 382 */ 383 len = -len; 384 count = 0; 385 for (;;) { 386 count += m->m_len; 387 if (m->m_next == (struct mbuf *)0) 388 break; 389 m = m->m_next; 390 } 391 if (m->m_len >= len) { 392 m->m_len -= len; 393 if (mp->m_flags & M_PKTHDR) 394 mp->m_pkthdr.len -= len; 395 return; 396 } 397 count -= len; 398 if (count < 0) 399 count = 0; 400 /* 401 * Correct length for chain is "count". 402 * Find the mbuf with last data, adjust its length, 403 * and toss data from remaining mbufs on chain. 404 */ 405 m = mp; 406 if (m->m_flags & M_PKTHDR) 407 m->m_pkthdr.len = count; 408 for (; m; m = m->m_next) { 409 if (m->m_len >= count) { 410 m->m_len = count; 411 break; 412 } 413 count -= m->m_len; 414 } 415 while (m->m_next) 416 (m = m->m_next) ->m_len = 0; 417 } 418 } 419 420 /* 421 * Rearange an mbuf chain so that len bytes are contiguous 422 * and in the data area of an mbuf (so that mtod and dtom 423 * will work for a structure of size len). Returns the resulting 424 * mbuf chain on success, frees it and returns null on failure. 425 * If there is room, it will add up to max_protohdr-len extra bytes to the 426 * contiguous region in an attempt to avoid being called next time. 427 */ 428 struct mbuf * 429 m_pullup(struct mbuf *n, int len) 430 { 431 struct mbuf *m; 432 int count; 433 int space; 434 435 /* 436 * If first mbuf has no cluster, and has room for len bytes 437 * without shifting current data, pullup into it, 438 * otherwise allocate a new mbuf to prepend to the chain. 439 */ 440 if ((n->m_flags & M_EXT) == 0 && 441 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 442 if (n->m_len >= len) 443 return (n); 444 m = n; 445 n = n->m_next; 446 len -= m->m_len; 447 } else { 448 if (len > MHLEN) 449 goto bad; 450 MGET(m, M_DONTWAIT, n->m_type); 451 if (m == NULL) 452 goto bad; 453 m->m_len = 0; 454 if (n->m_flags & M_PKTHDR) { 455 M_COPY_PKTHDR(m, n); 456 n->m_flags &= ~M_PKTHDR; 457 } 458 } 459 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 460 do { 461 count = min(min(max(len, max_protohdr), space), n->m_len); 462 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 463 (unsigned)count); 464 len -= count; 465 m->m_len += count; 466 n->m_len -= count; 467 space -= count; 468 if (n->m_len) 469 n->m_data += count; 470 else 471 n = m_free(n); 472 } while (len > 0 && n); 473 if (len > 0) { 474 (void) m_free(m); 475 goto bad; 476 } 477 m->m_next = n; 478 return (m); 479 bad: 480 m_freem(n); 481 mbstat.m_mpfail++; /* XXX: No consistency. */ 482 return (NULL); 483 } 484 485 /* 486 * Partition an mbuf chain in two pieces, returning the tail -- 487 * all but the first len0 bytes. In case of failure, it returns NULL and 488 * attempts to restore the chain to its original state. 489 * 490 * Note that the resulting mbufs might be read-only, because the new 491 * mbuf can end up sharing an mbuf cluster with the original mbuf if 492 * the "breaking point" happens to lie within a cluster mbuf. Use the 493 * M_WRITABLE() macro to check for this case. 494 */ 495 struct mbuf * 496 m_split(struct mbuf *m0, int len0, int wait) 497 { 498 struct mbuf *m, *n; 499 unsigned len = len0, remain; 500 501 for (m = m0; m && len > m->m_len; m = m->m_next) 502 len -= m->m_len; 503 if (m == NULL) 504 return (NULL); 505 remain = m->m_len - len; 506 if (m0->m_flags & M_PKTHDR) { 507 MGETHDR(n, wait, m0->m_type); 508 if (n == NULL) 509 return (NULL); 510 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 511 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 512 m0->m_pkthdr.len = len0; 513 if (m->m_flags & M_EXT) 514 goto extpacket; 515 if (remain > MHLEN) { 516 /* m can't be the lead packet */ 517 MH_ALIGN(n, 0); 518 n->m_next = m_split(m, len, wait); 519 if (n->m_next == NULL) { 520 (void) m_free(n); 521 return (NULL); 522 } else { 523 n->m_len = 0; 524 return (n); 525 } 526 } else 527 MH_ALIGN(n, remain); 528 } else if (remain == 0) { 529 n = m->m_next; 530 m->m_next = NULL; 531 return (n); 532 } else { 533 MGET(n, wait, m->m_type); 534 if (n == NULL) 535 return (NULL); 536 M_ALIGN(n, remain); 537 } 538 extpacket: 539 if (m->m_flags & M_EXT) { 540 n->m_flags |= M_EXT; 541 n->m_ext = m->m_ext; 542 MEXT_ADD_REF(m); 543 n->m_data = m->m_data + len; 544 } else { 545 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 546 } 547 n->m_len = remain; 548 m->m_len = len; 549 n->m_next = m->m_next; 550 m->m_next = NULL; 551 return (n); 552 } 553 /* 554 * Routine to copy from device local memory into mbufs. 555 * Note that `off' argument is offset into first mbuf of target chain from 556 * which to begin copying the data to. 557 */ 558 struct mbuf * 559 m_devget(char *buf, int totlen, int off, struct ifnet *ifp, 560 void (*copy)(char *from, caddr_t to, u_int len)) 561 { 562 struct mbuf *m; 563 struct mbuf *top = 0, **mp = ⊤ 564 int len; 565 566 if (off < 0 || off > MHLEN) 567 return (NULL); 568 569 MGETHDR(m, M_DONTWAIT, MT_DATA); 570 if (m == NULL) 571 return (NULL); 572 m->m_pkthdr.rcvif = ifp; 573 m->m_pkthdr.len = totlen; 574 len = MHLEN; 575 576 while (totlen > 0) { 577 if (top) { 578 MGET(m, M_DONTWAIT, MT_DATA); 579 if (m == NULL) { 580 m_freem(top); 581 return (NULL); 582 } 583 len = MLEN; 584 } 585 if (totlen + off >= MINCLSIZE) { 586 MCLGET(m, M_DONTWAIT); 587 if (m->m_flags & M_EXT) 588 len = MCLBYTES; 589 } else { 590 /* 591 * Place initial small packet/header at end of mbuf. 592 */ 593 if (top == NULL && totlen + off + max_linkhdr <= len) { 594 m->m_data += max_linkhdr; 595 len -= max_linkhdr; 596 } 597 } 598 if (off) { 599 m->m_data += off; 600 len -= off; 601 off = 0; 602 } 603 m->m_len = len = min(totlen, len); 604 if (copy) 605 copy(buf, mtod(m, caddr_t), (unsigned)len); 606 else 607 bcopy(buf, mtod(m, caddr_t), (unsigned)len); 608 buf += len; 609 *mp = m; 610 mp = &m->m_next; 611 totlen -= len; 612 } 613 return (top); 614 } 615 616 /* 617 * Copy data from a buffer back into the indicated mbuf chain, 618 * starting "off" bytes from the beginning, extending the mbuf 619 * chain if necessary. 620 */ 621 void 622 m_copyback(struct mbuf *m0, int off, int len, caddr_t cp) 623 { 624 int mlen; 625 struct mbuf *m = m0, *n; 626 int totlen = 0; 627 628 if (m0 == NULL) 629 return; 630 while (off > (mlen = m->m_len)) { 631 off -= mlen; 632 totlen += mlen; 633 if (m->m_next == NULL) { 634 n = m_get_clrd(M_DONTWAIT, m->m_type); 635 if (n == NULL) 636 goto out; 637 n->m_len = min(MLEN, len + off); 638 m->m_next = n; 639 } 640 m = m->m_next; 641 } 642 while (len > 0) { 643 mlen = min (m->m_len - off, len); 644 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 645 cp += mlen; 646 len -= mlen; 647 mlen += off; 648 off = 0; 649 totlen += mlen; 650 if (len == 0) 651 break; 652 if (m->m_next == NULL) { 653 n = m_get(M_DONTWAIT, m->m_type); 654 if (n == NULL) 655 break; 656 n->m_len = min(MLEN, len); 657 m->m_next = n; 658 } 659 m = m->m_next; 660 } 661 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 662 m->m_pkthdr.len = totlen; 663 } 664 665 void 666 m_print(const struct mbuf *m) 667 { 668 int len; 669 const struct mbuf *m2; 670 671 len = m->m_pkthdr.len; 672 m2 = m; 673 while (len) { 674 printf("%p %*D\n", m2, m2->m_len, (u_char *)m2->m_data, "-"); 675 len -= m2->m_len; 676 m2 = m2->m_next; 677 } 678 return; 679 } 680