1 /* 2 * Copyright (c) 1982, 1986, 1988, 1991, 1993 3 * The Regents of the University of California. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 3. All advertising materials mentioning features or use of this software 14 * must display the following acknowledgement: 15 * This product includes software developed by the University of 16 * California, Berkeley and its contributors. 17 * 4. Neither the name of the University nor the names of its contributors 18 * may be used to endorse or promote products derived from this software 19 * without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 * 33 * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 34 * $Id: uipc_mbuf.c,v 1.5 1994/10/04 06:50:01 phk Exp $ 35 */ 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/proc.h> 40 #include <sys/malloc.h> 41 #define MBTYPES 42 #include <sys/mbuf.h> 43 #include <sys/kernel.h> 44 #include <sys/syslog.h> 45 #include <sys/domain.h> 46 #include <sys/protosw.h> 47 48 #include <vm/vm.h> 49 50 extern vm_map_t mb_map; 51 struct mbuf *mbutl; 52 char *mclrefcnt; 53 54 void 55 mbinit() 56 { 57 int s; 58 59 #if CLBYTES < 4096 60 #define NCL_INIT (4096/CLBYTES) 61 #else 62 #define NCL_INIT 1 63 #endif 64 s = splimp(); 65 if (m_clalloc(NCL_INIT, M_DONTWAIT) == 0) 66 goto bad; 67 splx(s); 68 return; 69 bad: 70 panic("mbinit"); 71 } 72 73 /* 74 * Allocate some number of mbuf clusters 75 * and place on cluster free list. 76 * Must be called at splimp. 77 */ 78 /* ARGSUSED */ 79 int 80 m_clalloc(ncl, nowait) 81 register int ncl; 82 int nowait; 83 { 84 static int logged; 85 register caddr_t p; 86 register int i; 87 int npg; 88 89 npg = ncl * CLSIZE; 90 p = (caddr_t)kmem_malloc(mb_map, ctob(npg), !nowait); 91 if (p == NULL) { 92 if (logged == 0) { 93 logged++; 94 log(LOG_ERR, "mb_map full\n"); 95 } 96 return (0); 97 } 98 ncl = ncl * CLBYTES / MCLBYTES; 99 for (i = 0; i < ncl; i++) { 100 ((union mcluster *)p)->mcl_next = mclfree; 101 mclfree = (union mcluster *)p; 102 p += MCLBYTES; 103 mbstat.m_clfree++; 104 } 105 mbstat.m_clusters += ncl; 106 return (1); 107 } 108 109 /* 110 * When MGET failes, ask protocols to free space when short of memory, 111 * then re-attempt to allocate an mbuf. 112 */ 113 struct mbuf * 114 m_retry(i, t) 115 int i, t; 116 { 117 register struct mbuf *m; 118 119 m_reclaim(); 120 #define m_retry(i, t) (struct mbuf *)0 121 MGET(m, i, t); 122 #undef m_retry 123 return (m); 124 } 125 126 /* 127 * As above; retry an MGETHDR. 128 */ 129 struct mbuf * 130 m_retryhdr(i, t) 131 int i, t; 132 { 133 register struct mbuf *m; 134 135 m_reclaim(); 136 #define m_retryhdr(i, t) (struct mbuf *)0 137 MGETHDR(m, i, t); 138 #undef m_retryhdr 139 return (m); 140 } 141 142 void 143 m_reclaim() 144 { 145 register struct domain *dp; 146 register struct protosw *pr; 147 int s = splimp(); 148 149 for (dp = domains; dp; dp = dp->dom_next) 150 for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 151 if (pr->pr_drain) 152 (*pr->pr_drain)(); 153 splx(s); 154 mbstat.m_drain++; 155 } 156 157 /* 158 * Space allocation routines. 159 * These are also available as macros 160 * for critical paths. 161 */ 162 struct mbuf * 163 m_get(nowait, type) 164 int nowait, type; 165 { 166 register struct mbuf *m; 167 168 MGET(m, nowait, type); 169 return (m); 170 } 171 172 struct mbuf * 173 m_gethdr(nowait, type) 174 int nowait, type; 175 { 176 register struct mbuf *m; 177 178 MGETHDR(m, nowait, type); 179 return (m); 180 } 181 182 struct mbuf * 183 m_getclr(nowait, type) 184 int nowait, type; 185 { 186 register struct mbuf *m; 187 188 MGET(m, nowait, type); 189 if (m == 0) 190 return (0); 191 bzero(mtod(m, caddr_t), MLEN); 192 return (m); 193 } 194 195 struct mbuf * 196 m_free(m) 197 struct mbuf *m; 198 { 199 register struct mbuf *n; 200 201 MFREE(m, n); 202 return (n); 203 } 204 205 void 206 m_freem(m) 207 register struct mbuf *m; 208 { 209 register struct mbuf *n; 210 211 if (m == NULL) 212 return; 213 do { 214 MFREE(m, n); 215 m = n; 216 } while (m); 217 } 218 219 /* 220 * Mbuffer utility routines. 221 */ 222 223 /* 224 * Lesser-used path for M_PREPEND: 225 * allocate new mbuf to prepend to chain, 226 * copy junk along. 227 */ 228 struct mbuf * 229 m_prepend(m, len, how) 230 register struct mbuf *m; 231 int len, how; 232 { 233 struct mbuf *mn; 234 235 MGET(mn, how, m->m_type); 236 if (mn == (struct mbuf *)NULL) { 237 m_freem(m); 238 return ((struct mbuf *)NULL); 239 } 240 if (m->m_flags & M_PKTHDR) { 241 M_COPY_PKTHDR(mn, m); 242 m->m_flags &= ~M_PKTHDR; 243 } 244 mn->m_next = m; 245 m = mn; 246 if (len < MHLEN) 247 MH_ALIGN(m, len); 248 m->m_len = len; 249 return (m); 250 } 251 252 /* 253 * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 254 * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 255 * The wait parameter is a choice of M_WAIT/M_DONTWAIT from caller. 256 */ 257 int MCFail; 258 259 struct mbuf * 260 m_copym(m, off0, len, wait) 261 register struct mbuf *m; 262 int off0, wait; 263 register int len; 264 { 265 register struct mbuf *n, **np; 266 register int off = off0; 267 struct mbuf *top; 268 int copyhdr = 0; 269 270 if (off < 0 || len < 0) 271 panic("m_copym"); 272 if (off == 0 && m->m_flags & M_PKTHDR) 273 copyhdr = 1; 274 while (off > 0) { 275 if (m == 0) 276 panic("m_copym"); 277 if (off < m->m_len) 278 break; 279 off -= m->m_len; 280 m = m->m_next; 281 } 282 np = ⊤ 283 top = 0; 284 while (len > 0) { 285 if (m == 0) { 286 if (len != M_COPYALL) 287 panic("m_copym"); 288 break; 289 } 290 MGET(n, wait, m->m_type); 291 *np = n; 292 if (n == 0) 293 goto nospace; 294 if (copyhdr) { 295 M_COPY_PKTHDR(n, m); 296 if (len == M_COPYALL) 297 n->m_pkthdr.len -= off0; 298 else 299 n->m_pkthdr.len = len; 300 copyhdr = 0; 301 } 302 n->m_len = min(len, m->m_len - off); 303 if (m->m_flags & M_EXT) { 304 n->m_data = m->m_data + off; 305 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 306 n->m_ext = m->m_ext; 307 n->m_flags |= M_EXT; 308 } else 309 bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 310 (unsigned)n->m_len); 311 if (len != M_COPYALL) 312 len -= n->m_len; 313 off = 0; 314 m = m->m_next; 315 np = &n->m_next; 316 } 317 if (top == 0) 318 MCFail++; 319 return (top); 320 nospace: 321 m_freem(top); 322 MCFail++; 323 return (0); 324 } 325 326 /* 327 * Copy data from an mbuf chain starting "off" bytes from the beginning, 328 * continuing for "len" bytes, into the indicated buffer. 329 */ 330 void 331 m_copydata(m, off, len, cp) 332 register struct mbuf *m; 333 register int off; 334 register int len; 335 caddr_t cp; 336 { 337 register unsigned count; 338 339 if (off < 0 || len < 0) 340 panic("m_copydata"); 341 while (off > 0) { 342 if (m == 0) 343 panic("m_copydata"); 344 if (off < m->m_len) 345 break; 346 off -= m->m_len; 347 m = m->m_next; 348 } 349 while (len > 0) { 350 if (m == 0) 351 panic("m_copydata"); 352 count = min(m->m_len - off, len); 353 bcopy(mtod(m, caddr_t) + off, cp, count); 354 len -= count; 355 cp += count; 356 off = 0; 357 m = m->m_next; 358 } 359 } 360 361 /* 362 * Concatenate mbuf chain n to m. 363 * Both chains must be of the same type (e.g. MT_DATA). 364 * Any m_pkthdr is not updated. 365 */ 366 void 367 m_cat(m, n) 368 register struct mbuf *m, *n; 369 { 370 while (m->m_next) 371 m = m->m_next; 372 while (n) { 373 if (m->m_flags & M_EXT || 374 m->m_data + m->m_len + n->m_len >= &m->m_dat[MLEN]) { 375 /* just join the two chains */ 376 m->m_next = n; 377 return; 378 } 379 /* splat the data from one into the other */ 380 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 381 (u_int)n->m_len); 382 m->m_len += n->m_len; 383 n = m_free(n); 384 } 385 } 386 387 void 388 m_adj(mp, req_len) 389 struct mbuf *mp; 390 int req_len; 391 { 392 register int len = req_len; 393 register struct mbuf *m; 394 register count; 395 396 if ((m = mp) == NULL) 397 return; 398 if (len >= 0) { 399 /* 400 * Trim from head. 401 */ 402 while (m != NULL && len > 0) { 403 if (m->m_len <= len) { 404 len -= m->m_len; 405 m->m_len = 0; 406 m = m->m_next; 407 } else { 408 m->m_len -= len; 409 m->m_data += len; 410 len = 0; 411 } 412 } 413 m = mp; 414 if (mp->m_flags & M_PKTHDR) 415 m->m_pkthdr.len -= (req_len - len); 416 } else { 417 /* 418 * Trim from tail. Scan the mbuf chain, 419 * calculating its length and finding the last mbuf. 420 * If the adjustment only affects this mbuf, then just 421 * adjust and return. Otherwise, rescan and truncate 422 * after the remaining size. 423 */ 424 len = -len; 425 count = 0; 426 for (;;) { 427 count += m->m_len; 428 if (m->m_next == (struct mbuf *)0) 429 break; 430 m = m->m_next; 431 } 432 if (m->m_len >= len) { 433 m->m_len -= len; 434 if (mp->m_flags & M_PKTHDR) 435 mp->m_pkthdr.len -= len; 436 return; 437 } 438 count -= len; 439 if (count < 0) 440 count = 0; 441 /* 442 * Correct length for chain is "count". 443 * Find the mbuf with last data, adjust its length, 444 * and toss data from remaining mbufs on chain. 445 */ 446 m = mp; 447 if (m->m_flags & M_PKTHDR) 448 m->m_pkthdr.len = count; 449 for (; m; m = m->m_next) { 450 if (m->m_len >= count) { 451 m->m_len = count; 452 break; 453 } 454 count -= m->m_len; 455 } 456 while (m->m_next) 457 (m = m->m_next) ->m_len = 0; 458 } 459 } 460 461 /* 462 * Rearange an mbuf chain so that len bytes are contiguous 463 * and in the data area of an mbuf (so that mtod and dtom 464 * will work for a structure of size len). Returns the resulting 465 * mbuf chain on success, frees it and returns null on failure. 466 * If there is room, it will add up to max_protohdr-len extra bytes to the 467 * contiguous region in an attempt to avoid being called next time. 468 */ 469 int MPFail; 470 471 struct mbuf * 472 m_pullup(n, len) 473 register struct mbuf *n; 474 int len; 475 { 476 register struct mbuf *m; 477 register int count; 478 int space; 479 480 /* 481 * If first mbuf has no cluster, and has room for len bytes 482 * without shifting current data, pullup into it, 483 * otherwise allocate a new mbuf to prepend to the chain. 484 */ 485 if ((n->m_flags & M_EXT) == 0 && 486 n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 487 if (n->m_len >= len) 488 return (n); 489 m = n; 490 n = n->m_next; 491 len -= m->m_len; 492 } else { 493 if (len > MHLEN) 494 goto bad; 495 MGET(m, M_DONTWAIT, n->m_type); 496 if (m == 0) 497 goto bad; 498 m->m_len = 0; 499 if (n->m_flags & M_PKTHDR) { 500 M_COPY_PKTHDR(m, n); 501 n->m_flags &= ~M_PKTHDR; 502 } 503 } 504 space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 505 do { 506 count = min(min(max(len, max_protohdr), space), n->m_len); 507 bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 508 (unsigned)count); 509 len -= count; 510 m->m_len += count; 511 n->m_len -= count; 512 space -= count; 513 if (n->m_len) 514 n->m_data += count; 515 else 516 n = m_free(n); 517 } while (len > 0 && n); 518 if (len > 0) { 519 (void) m_free(m); 520 goto bad; 521 } 522 m->m_next = n; 523 return (m); 524 bad: 525 m_freem(n); 526 MPFail++; 527 return (0); 528 } 529 530 /* 531 * Partition an mbuf chain in two pieces, returning the tail -- 532 * all but the first len0 bytes. In case of failure, it returns NULL and 533 * attempts to restore the chain to its original state. 534 */ 535 struct mbuf * 536 m_split(m0, len0, wait) 537 register struct mbuf *m0; 538 int len0, wait; 539 { 540 register struct mbuf *m, *n; 541 unsigned len = len0, remain; 542 543 for (m = m0; m && len > m->m_len; m = m->m_next) 544 len -= m->m_len; 545 if (m == 0) 546 return (0); 547 remain = m->m_len - len; 548 if (m0->m_flags & M_PKTHDR) { 549 MGETHDR(n, wait, m0->m_type); 550 if (n == 0) 551 return (0); 552 n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 553 n->m_pkthdr.len = m0->m_pkthdr.len - len0; 554 m0->m_pkthdr.len = len0; 555 if (m->m_flags & M_EXT) 556 goto extpacket; 557 if (remain > MHLEN) { 558 /* m can't be the lead packet */ 559 MH_ALIGN(n, 0); 560 n->m_next = m_split(m, len, wait); 561 if (n->m_next == 0) { 562 (void) m_free(n); 563 return (0); 564 } else 565 return (n); 566 } else 567 MH_ALIGN(n, remain); 568 } else if (remain == 0) { 569 n = m->m_next; 570 m->m_next = 0; 571 return (n); 572 } else { 573 MGET(n, wait, m->m_type); 574 if (n == 0) 575 return (0); 576 M_ALIGN(n, remain); 577 } 578 extpacket: 579 if (m->m_flags & M_EXT) { 580 n->m_flags |= M_EXT; 581 n->m_ext = m->m_ext; 582 mclrefcnt[mtocl(m->m_ext.ext_buf)]++; 583 m->m_ext.ext_size = 0; /* For Accounting XXXXXX danger */ 584 n->m_data = m->m_data + len; 585 } else { 586 bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 587 } 588 n->m_len = remain; 589 m->m_len = len; 590 n->m_next = m->m_next; 591 m->m_next = 0; 592 return (n); 593 } 594 /* 595 * Routine to copy from device local memory into mbufs. 596 */ 597 struct mbuf * 598 m_devget(buf, totlen, off0, ifp, copy) 599 char *buf; 600 int totlen, off0; 601 struct ifnet *ifp; 602 void (*copy)(); 603 { 604 register struct mbuf *m; 605 struct mbuf *top = 0, **mp = ⊤ 606 register int off = off0, len; 607 register char *cp; 608 char *epkt; 609 610 cp = buf; 611 epkt = cp + totlen; 612 if (off) { 613 cp += off + 2 * sizeof(u_short); 614 totlen -= 2 * sizeof(u_short); 615 } 616 MGETHDR(m, M_DONTWAIT, MT_DATA); 617 if (m == 0) 618 return (0); 619 m->m_pkthdr.rcvif = ifp; 620 m->m_pkthdr.len = totlen; 621 m->m_len = MHLEN; 622 623 while (totlen > 0) { 624 if (top) { 625 MGET(m, M_DONTWAIT, MT_DATA); 626 if (m == 0) { 627 m_freem(top); 628 return (0); 629 } 630 m->m_len = MLEN; 631 } 632 len = min(totlen, epkt - cp); 633 if (len >= MINCLSIZE) { 634 MCLGET(m, M_DONTWAIT); 635 if (m->m_flags & M_EXT) 636 m->m_len = len = min(len, MCLBYTES); 637 else 638 len = m->m_len; 639 } else { 640 /* 641 * Place initial small packet/header at end of mbuf. 642 */ 643 if (len < m->m_len) { 644 if (top == 0 && len + max_linkhdr <= m->m_len) 645 m->m_data += max_linkhdr; 646 m->m_len = len; 647 } else 648 len = m->m_len; 649 } 650 if (copy) 651 copy(cp, mtod(m, caddr_t), (unsigned)len); 652 else 653 bcopy(cp, mtod(m, caddr_t), (unsigned)len); 654 cp += len; 655 *mp = m; 656 mp = &m->m_next; 657 totlen -= len; 658 if (cp == epkt) 659 cp = buf; 660 } 661 return (top); 662 } 663 664 /* 665 * Copy data from a buffer back into the indicated mbuf chain, 666 * starting "off" bytes from the beginning, extending the mbuf 667 * chain if necessary. 668 */ 669 void 670 m_copyback(m0, off, len, cp) 671 struct mbuf *m0; 672 register int off; 673 register int len; 674 caddr_t cp; 675 { 676 register int mlen; 677 register struct mbuf *m = m0, *n; 678 int totlen = 0; 679 680 if (m0 == 0) 681 return; 682 while (off > (mlen = m->m_len)) { 683 off -= mlen; 684 totlen += mlen; 685 if (m->m_next == 0) { 686 n = m_getclr(M_DONTWAIT, m->m_type); 687 if (n == 0) 688 goto out; 689 n->m_len = min(MLEN, len + off); 690 m->m_next = n; 691 } 692 m = m->m_next; 693 } 694 while (len > 0) { 695 mlen = min (m->m_len - off, len); 696 bcopy(cp, off + mtod(m, caddr_t), (unsigned)mlen); 697 cp += mlen; 698 len -= mlen; 699 mlen += off; 700 off = 0; 701 totlen += mlen; 702 if (len == 0) 703 break; 704 if (m->m_next == 0) { 705 n = m_get(M_DONTWAIT, m->m_type); 706 if (n == 0) 707 break; 708 n->m_len = min(MLEN, len); 709 m->m_next = n; 710 } 711 m = m->m_next; 712 } 713 out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 714 m->m_pkthdr.len = totlen; 715 } 716