1 /* 2 * Copyright (c) 1982, 1986, 1988, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 34 * $Id: uipc_mbuf.c,v 1.8 1995/02/23 19:10:21 davidg Exp $ 35 */ 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #define MBTYPES 42 #include <sys/mbuf.h> 43 #include <sys/kernel.h> 44 #include <sys/syslog.h> 45 #include <sys/domain.h> 46 #include <sys/protosw.h> 47 48 #include <vm/vm.h> 49 50 extern vm_map_t mb_map; 51 struct mbuf *mbutl; 52 char *mclrefcnt; 53 int mb_map_full; 54 55 void 56 mbinit() 57 { 58 int s; 59 60 #if CLBYTES < 4096 61 #define NCL_INIT (4096/CLBYTES) 62 #else 63 #define NCL_INIT 1 64 #endif 65 s = splimp(); 66 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 67 goto bad; 68 splx(s); 69 return; 70 bad: 71 panic("mbinit"); 72 } 73 74 /* 75 * Allocate some number of mbuf clusters 76 * and place on cluster free list. 77 * Must be called at splimp. 78 */ 79 /* ARGSUSED */ 80 int 81 m_clalloc(ncl, nowait) 82 register int ncl; 83 int nowait; 84 { 85 static int logged; 86 register caddr_t p; 87 register int i; 88 int npg; 89 90 /* 91 * Once we run out of map space, it will be impossible 92 * to get any more (nothing is ever freed back to the 93 * map). 94 */ 95 if (mb_map_full) 96 return (0); 97 98 npg = ncl * CLSIZE; 99 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), 100 nowait ? M_NOWAIT : M_WAITOK); 101 /* 102 * Either the map is now full, or this is nowait and there 103 * are no pages left. 104 */ 105 if (p == NULL) 106 return (0); 107 108 ncl = ncl * CLBYTES / MCLBYTES; 109 for (i = 0; i < ncl; i++) { 110 ((union mcluster *)p)->mcl_next = mclfree; 111 mclfree = (union mcluster *)p; 112 p += MCLBYTES; 113 mbstat.m_clfree++; 114 } 115 mbstat.m_clusters += ncl; 116 return (1); 117 } 118 119 /* 120 * When MGET failes, ask protocols to free space when short of memory, 121 * then re-attempt to allocate an mbuf. 122 */ 123 struct mbuf * 124 m_retry(i, t) 125 int i, t; 126 { 127 register struct mbuf *m; 128 129 m_reclaim(); 130 #define m_retry(i, t) (struct mbuf *)0 131 MGET(m, i, t); 132 #undef m_retry 133 if (m != NULL) 134 mbstat.m_wait++; 135 else 136 mbstat.m_drops++; 137 return (m); 138 } 139 140 /* 141 * As above; retry an MGETHDR. 142 */ 143 struct mbuf * 144 m_retryhdr(i, t) 145 int i, t; 146 { 147 register struct mbuf *m; 148 149 m_reclaim(); 150 #define m_retryhdr(i, t) (struct mbuf *)0 151 MGETHDR(m, i, t); 152 #undef m_retryhdr 153 if (m != NULL) 154 mbstat.m_wait++; 155 else 156 mbstat.m_drops++; 157 return (m); 158 } 159 160 void 161 m_reclaim() 162 { 163 register struct domain *dp; 164 register struct protosw *pr; 165 int s = splimp(); 166 167 for (dp = domains; dp; dp = dp->dom_next) 168 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 169 if (pr->pr_drain) 170 (*pr->pr_drain)(); 171 splx(s); 172 mbstat.m_drain++; 173 } 174 175 /* 176 * Space allocation routines. 177 * These are also available as macros 178 * for critical paths. 179 */ 180 struct mbuf * 181 m_get(nowait, type) 182 int nowait, type; 183 { 184 register struct mbuf *m; 185 186 MGET(m, nowait, type); 187 return (m); 188 } 189 190 struct mbuf * 191 m_gethdr(nowait, type) 192 int nowait, type; 193 { 194 register struct mbuf *m; 195 196 MGETHDR(m, nowait, type); 197 return (m); 198 } 199 200 struct mbuf * 201 m_getclr(nowait, type) 202 int nowait, type; 203 { 204 register struct mbuf *m; 205 206 MGET(m, nowait, type); 207 if (m == 0) 208 return (0); 209 bzero(mtod(m, caddr_t), MLEN); 210 return (m); 211 } 212 213 struct mbuf * 214 m_free(m) 215 struct mbuf *m; 216 { 217 register struct mbuf *n; 218 219 MFREE(m, n); 220 return (n); 221 } 222 223 void 224 m_freem(m) 225 register struct mbuf *m; 226 { 227 register struct mbuf *n; 228 229 if (m == NULL) 230 return; 231 do { 232 MFREE(m, n); 233 m = n; 234 } while (m); 235 } 236 237 /* 238 * Mbuffer utility routines. 239 */ 240 241 /* 242 * Lesser-used path for M_PREPEND: 243 * allocate new mbuf to prepend to chain, 244 * copy junk along. 245 */ 246 struct mbuf * 247 m_prepend(m, len, how) 248 register struct mbuf *m; 249 int len, how; 250 { 251 struct mbuf *mn; 252 253 MGET(mn, how, m->m_type); 254 if (mn == (struct mbuf *)NULL) { 255 m_freem(m); 256 return ((struct mbuf *)NULL); 257 } 258 if (m->m_flags & M_PKTHDR) { 259 M_COPY_PKTHDR(mn, m); 260 m->m_flags &= ~M_PKTHDR; 261 } 262 mn->m_next = m; 263 m = mn; 264 if (len < MHLEN) 265 MH_ALIGN(m, len); 266 m->m_len = len; 267 return (m); 268 } 269 270 /* 271 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 272 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 273 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 274 */ 275 int MCFail; 276 277 struct mbuf * 278 m_copym(m, off0, len, wait) 279 register struct mbuf *m; 280 int off0, wait; 281 register int len; 282 { 283 register struct mbuf *n, **np; 284 register int off = off0; 285 struct mbuf *top; 286 int copyhdr = 0; 287 288 if (off < 0 || len < 0) 289 panic("m_copym"); 290 if (off == 0 && m->m_flags & M_PKTHDR) 291 copyhdr = 1; 292 while (off > 0) { 293 if (m == 0) 294 panic("m_copym"); 295 if (off < m->m_len) 296 break; 297 off -= m->m_len; 298 m = m->m_next; 299 } 300 np = ⊤ 301 top = 0; 302 while (len > 0) { 303 if (m == 0) { 304 if (len != M_COPYALL) 305 panic("m_copym"); 306 break; 307 } 308 MGET(n, wait, m->m_type); 309 *np = n; 310 if (n == 0) 311 goto nospace; 312 if (copyhdr) { 313 M_COPY_PKTHDR(n, m); 314 if (len == M_COPYALL) 315 n->m_pkthdr.len -= off0; 316 else 317 n->m_pkthdr.len = len; 318 copyhdr = 0; 319 } 320 n->m_len = min(len, m->m_len - off); 321 if (m->m_flags & M_EXT) { 322 n->m_data = m->m_data + off; 323 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 324 n->m_ext = m->m_ext; 325 n->m_flags |= M_EXT; 326 } else 327 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 328 (unsigned)n->m_len); 329 if (len != M_COPYALL) 330 len -= n->m_len; 331 off = 0; 332 m = m->m_next; 333 np = &n->m_next; 334 } 335 if (top == 0) 336 MCFail++; 337 return (top); 338 nospace: 339 m_freem(top); 340 MCFail++; 341 return (0); 342 } 343 344 /* 345 * Copy data from an mbuf chain starting "off" bytes from the beginning, 346 * continuing for "len" bytes, into the indicated buffer. 347 */ 348 void 349 m_copydata(m, off, len, cp) 350 register struct mbuf *m; 351 register int off; 352 register int len; 353 caddr_t cp; 354 { 355 register unsigned count; 356 357 if (off < 0 || len < 0) 358 panic("m_copydata"); 359 while (off > 0) { 360 if (m == 0) 361 panic("m_copydata"); 362 if (off < m->m_len) 363 break; 364 off -= m->m_len; 365 m = m->m_next; 366 } 367 while (len > 0) { 368 if (m == 0) 369 panic("m_copydata"); 370 count = min(m->m_len - off, len); 371 bcopy(mtod(m, caddr_t) + off, cp, count); 372 len -= count; 373 cp += count; 374 off = 0; 375 m = m->m_next; 376 } 377 } 378 379 /* 380 * Concatenate mbuf chain n to m. 381 * Both chains must be of the same type (e.g. MT_DATA). 382 * Any m_pkthdr is not updated. 383 */ 384 void 385 m_cat(m, n) 386 register struct mbuf *m, *n; 387 { 388 while (m->m_next) 389 m = m->m_next; 390 while (n) { 391 if (m->m_flags & M_EXT || 392 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 393 /* just join the two chains */ 394 m->m_next = n; 395 return; 396 } 397 /* splat the data from one into the other */ 398 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 399 (u_int)n->m_len); 400 m->m_len += n->m_len; 401 n = m_free(n); 402 } 403 } 404 405 void 406 m_adj(mp, req_len) 407 struct mbuf *mp; 408 int req_len; 409 { 410 register int len = req_len; 411 register struct mbuf *m; 412 register count; 413 414 if ((m = mp) == NULL) 415 return; 416 if (len >= 0) { 417 /* 418 * Trim from head. 419 */ 420 while (m != NULL && len > 0) { 421 if (m->m_len <= len) { 422 len -= m->m_len; 423 m->m_len = 0; 424 m = m->m_next; 425 } else { 426 m->m_len -= len; 427 m->m_data += len; 428 len = 0; 429 } 430 } 431 m = mp; 432 if (mp->m_flags & M_PKTHDR) 433 m->m_pkthdr.len -= (req_len - len); 434 } else { 435 /* 436 * Trim from tail. Scan the mbuf chain, 437 * calculating its length and finding the last mbuf. 438 * If the adjustment only affects this mbuf, then just 439 * adjust and return. Otherwise, rescan and truncate 440 * after the remaining size. 441 */ 442 len = -len; 443 count = 0; 444 for (;;) { 445 count += m->m_len; 446 if (m->m_next == (struct mbuf *)0) 447 break; 448 m = m->m_next; 449 } 450 if (m->m_len >= len) { 451 m->m_len -= len; 452 if (mp->m_flags & M_PKTHDR) 453 mp->m_pkthdr.len -= len; 454 return; 455 } 456 count -= len; 457 if (count < 0) 458 count = 0; 459 /* 460 * Correct length for chain is "count". 461 * Find the mbuf with last data, adjust its length, 462 * and toss data from remaining mbufs on chain. 463 */ 464 m = mp; 465 if (m->m_flags & M_PKTHDR) 466 m->m_pkthdr.len = count; 467 for (; m; m = m->m_next) { 468 if (m->m_len >= count) { 469 m->m_len = count; 470 break; 471 } 472 count -= m->m_len; 473 } 474 while (m->m_next) 475 (m = m->m_next) ->m_len = 0; 476 } 477 } 478 479 /* 480 * Rearange an mbuf chain so that len bytes are contiguous 481 * and in the data area of an mbuf (so that mtod and dtom 482 * will work for a structure of size len). Returns the resulting 483 * mbuf chain on success, frees it and returns null on failure. 484 * If there is room, it will add up to max_protohdr-len extra bytes to the 485 * contiguous region in an attempt to avoid being called next time. 486 */ 487 int MPFail; 488 489 struct mbuf * 490 m_pullup(n, len) 491 register struct mbuf *n; 492 int len; 493 { 494 register struct mbuf *m; 495 register int count; 496 int space; 497 498 /* 499 * If first mbuf has no cluster, and has room for len bytes 500 * without shifting current data, pullup into it, 501 * otherwise allocate a new mbuf to prepend to the chain. 502 */ 503 if ((n->m_flags & M_EXT) == 0 && 504 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 505 if (n->m_len >= len) 506 return (n); 507 m = n; 508 n = n->m_next; 509 len -= m->m_len; 510 } else { 511 if (len > MHLEN) 512 goto bad; 513 MGET(m, M_DONTWAIT, n->m_type); 514 if (m == 0) 515 goto bad; 516 m->m_len = 0; 517 if (n->m_flags & M_PKTHDR) { 518 M_COPY_PKTHDR(m, n); 519 n->m_flags &= ~M_PKTHDR; 520 } 521 } 522 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 523 do { 524 count = min(min(max(len, max_protohdr), space), n->m_len); 525 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 526 (unsigned)count); 527 len -= count; 528 m->m_len += count; 529 n->m_len -= count; 530 space -= count; 531 if (n->m_len) 532 n->m_data += count; 533 else 534 n = m_free(n); 535 } while (len > 0 && n); 536 if (len > 0) { 537 (void) m_free(m); 538 goto bad; 539 } 540 m->m_next = n; 541 return (m); 542 bad: 543 m_freem(n); 544 MPFail++; 545 return (0); 546 } 547 548 /* 549 * Partition an mbuf chain in two pieces, returning the tail -- 550 * all but the first len0 bytes. In case of failure, it returns NULL and 551 * attempts to restore the chain to its original state. 552 */ 553 struct mbuf * 554 m_split(m0, len0, wait) 555 register struct mbuf *m0; 556 int len0, wait; 557 { 558 register struct mbuf *m, *n; 559 unsigned len = len0, remain; 560 561 for (m = m0; m && len > m->m_len; m = m->m_next) 562 len -= m->m_len; 563 if (m == 0) 564 return (0); 565 remain = m->m_len - len; 566 if (m0->m_flags & M_PKTHDR) { 567 MGETHDR(n, wait, m0->m_type); 568 if (n == 0) 569 return (0); 570 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 571 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 572 m0->m_pkthdr.len = len0; 573 if (m->m_flags & M_EXT) 574 goto extpacket; 575 if (remain > MHLEN) { 576 /* m can't be the lead packet */ 577 MH_ALIGN(n, 0); 578 n->m_next = m_split(m, len, wait); 579 if (n->m_next == 0) { 580 (void) m_free(n); 581 return (0); 582 } else 583 return (n); 584 } else 585 MH_ALIGN(n, remain); 586 } else if (remain == 0) { 587 n = m->m_next; 588 m->m_next = 0; 589 return (n); 590 } else { 591 MGET(n, wait, m->m_type); 592 if (n == 0) 593 return (0); 594 M_ALIGN(n, remain); 595 } 596 extpacket: 597 if (m->m_flags & M_EXT) { 598 n->m_flags |= M_EXT; 599 n->m_ext = m->m_ext; 600 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 601 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 602 n->m_data = m->m_data + len; 603 } else { 604 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 605 } 606 n->m_len = remain; 607 m->m_len = len; 608 n->m_next = m->m_next; 609 m->m_next = 0; 610 return (n); 611 } 612 /* 613 * Routine to copy from device local memory into mbufs. 614 */ 615 struct mbuf * 616 m_devget(buf, totlen, off0, ifp, copy) 617 char *buf; 618 int totlen, off0; 619 struct ifnet *ifp; 620 void (*copy)(); 621 { 622 register struct mbuf *m; 623 struct mbuf *top = 0, **mp = ⊤ 624 register int off = off0, len; 625 register char *cp; 626 char *epkt; 627 628 cp = buf; 629 epkt = cp + totlen; 630 if (off) { 631 cp += off + 2 * sizeof(u_short); 632 totlen -= 2 * sizeof(u_short); 633 } 634 MGETHDR(m, M_DONTWAIT, MT_DATA); 635 if (m == 0) 636 return (0); 637 m->m_pkthdr.rcvif = ifp; 638 m->m_pkthdr.len = totlen; 639 m->m_len = MHLEN; 640 641 while (totlen > 0) { 642 if (top) { 643 MGET(m, M_DONTWAIT, MT_DATA); 644 if (m == 0) { 645 m_freem(top); 646 return (0); 647 } 648 m->m_len = MLEN; 649 } 650 len = min(totlen, epkt - cp); 651 if (len >= MINCLSIZE) { 652 MCLGET(m, M_DONTWAIT); 653 if (m->m_flags & M_EXT) 654 m->m_len = len = min(len, MCLBYTES); 655 else 656 len = m->m_len; 657 } else { 658 /* 659 * Place initial small packet/header at end of mbuf. 660 */ 661 if (len < m->m_len) { 662 if (top == 0 && len + max_linkhdr <= m->m_len) 663 m->m_data += max_linkhdr; 664 m->m_len = len; 665 } else 666 len = m->m_len; 667 } 668 if (copy) 669 copy(cp, mtod(m, caddr_t), (unsigned)len); 670 else 671 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 672 cp += len; 673 *mp = m; 674 mp = &m->m_next; 675 totlen -= len; 676 if (cp == epkt) 677 cp = buf; 678 } 679 return (top); 680 } 681 682 /* 683 * Copy data from a buffer back into the indicated mbuf chain, 684 * starting "off" bytes from the beginning, extending the mbuf 685 * chain if necessary. 686 */ 687 void 688 m_copyback(m0, off, len, cp) 689 struct mbuf *m0; 690 register int off; 691 register int len; 692 caddr_t cp; 693 { 694 register int mlen; 695 register struct mbuf *m = m0, *n; 696 int totlen = 0; 697 698 if (m0 == 0) 699 return; 700 while (off > (mlen = m->m_len)) { 701 off -= mlen; 702 totlen += mlen; 703 if (m->m_next == 0) { 704 n = m_getclr(M_DONTWAIT, m->m_type); 705 if (n == 0) 706 goto out; 707 n->m_len = min(MLEN, len + off); 708 m->m_next = n; 709 } 710 m = m->m_next; 711 } 712 while (len > 0) { 713 mlen = min (m->m_len - off, len); 714 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 715 cp += mlen; 716 len -= mlen; 717 mlen += off; 718 off = 0; 719 totlen += mlen; 720 if (len == 0) 721 break; 722 if (m->m_next == 0) { 723 n = m_get(M_DONTWAIT, m->m_type); 724 if (n == 0) 725 break; 726 n->m_len = min(MLEN, len); 727 m->m_next = n; 728 } 729 m = m->m_next; 730 } 731 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 732 m->m_pkthdr.len = totlen; 733 } 734