19454b2d8SWarner Losh /*- 2df8bae1dSRodney W. Grimes * Copyright (c) 1982, 1986, 1988, 1991, 1993 3df8bae1dSRodney W. Grimes * The Regents of the University of California. All rights reserved. 4df8bae1dSRodney W. Grimes * 5df8bae1dSRodney W. Grimes * Redistribution and use in source and binary forms, with or without 6df8bae1dSRodney W. Grimes * modification, are permitted provided that the following conditions 7df8bae1dSRodney W. Grimes * are met: 8df8bae1dSRodney W. Grimes * 1. Redistributions of source code must retain the above copyright 9df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer. 10df8bae1dSRodney W. Grimes * 2. Redistributions in binary form must reproduce the above copyright 11df8bae1dSRodney W. Grimes * notice, this list of conditions and the following disclaimer in the 12df8bae1dSRodney W. Grimes * documentation and/or other materials provided with the distribution. 13df8bae1dSRodney W. Grimes * 4. Neither the name of the University nor the names of its contributors 14df8bae1dSRodney W. Grimes * may be used to endorse or promote products derived from this software 15df8bae1dSRodney W. Grimes * without specific prior written permission. 16df8bae1dSRodney W. Grimes * 17df8bae1dSRodney W. Grimes * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 18df8bae1dSRodney W. Grimes * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19df8bae1dSRodney W. Grimes * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20df8bae1dSRodney W. Grimes * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 21df8bae1dSRodney W. Grimes * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22df8bae1dSRodney W. Grimes * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23df8bae1dSRodney W. Grimes * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24df8bae1dSRodney W. Grimes * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25df8bae1dSRodney W. Grimes * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26df8bae1dSRodney W. Grimes * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27df8bae1dSRodney W. Grimes * SUCH DAMAGE. 28df8bae1dSRodney W. Grimes * 29df8bae1dSRodney W. Grimes * @(#)uipc_mbuf.c 8.2 (Berkeley) 1/4/94 30df8bae1dSRodney W. Grimes */ 31df8bae1dSRodney W. Grimes 32677b542eSDavid E. O'Brien #include <sys/cdefs.h> 33677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$"); 34677b542eSDavid E. O'Brien 35240ef842SDavid E. O'Brien #include "opt_param.h" 36352d050eSMike Silbersack #include "opt_mbuf_stress_test.h" 376eeac1d9SJulian Elischer #include "opt_mbuf_profiling.h" 38e32a5b94SRobert Watson 39df8bae1dSRodney W. Grimes #include <sys/param.h> 40df8bae1dSRodney W. Grimes #include <sys/systm.h> 41fb919e4dSMark Murray #include <sys/kernel.h> 42beb699c7SMike Silbersack #include <sys/limits.h> 43fb919e4dSMark Murray #include <sys/lock.h> 44f9d0d524SRobert Watson #include <sys/malloc.h> 45df8bae1dSRodney W. Grimes #include <sys/mbuf.h> 46639acc13SGarrett Wollman #include <sys/sysctl.h> 47df8bae1dSRodney W. Grimes #include <sys/domain.h> 48df8bae1dSRodney W. Grimes #include <sys/protosw.h> 49beb699c7SMike Silbersack #include <sys/uio.h> 50fb919e4dSMark Murray 5128f8db14SBruce Evans int max_linkhdr; 5228f8db14SBruce Evans int max_protohdr; 5328f8db14SBruce Evans int max_hdr; 5428f8db14SBruce Evans int max_datalen; 5551710a45SMike Silbersack #ifdef MBUF_STRESS_TEST 5655e9f80dSMike Silbersack int m_defragpackets; 5755e9f80dSMike Silbersack int m_defragbytes; 5855e9f80dSMike Silbersack int m_defraguseless; 5955e9f80dSMike Silbersack int m_defragfailure; 60352d050eSMike Silbersack int m_defragrandomfailures; 61352d050eSMike Silbersack #endif 627d032714SBosko Milekic 637d032714SBosko Milekic /* 647d032714SBosko Milekic * sysctl(8) exported objects 657d032714SBosko Milekic */ 6680444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD, 6780444f88SAndre Oppermann &max_linkhdr, 0, "Size of largest link layer header"); 6880444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD, 6980444f88SAndre Oppermann &max_protohdr, 0, "Size of largest protocol layer header"); 7080444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD, 7180444f88SAndre Oppermann &max_hdr, 0, "Size of largest link plus protocol header"); 7280444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD, 7380444f88SAndre Oppermann &max_datalen, 0, "Minimum space left in mbuf after max_hdr"); 7451710a45SMike Silbersack #ifdef MBUF_STRESS_TEST 7555e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD, 7655e9f80dSMike Silbersack &m_defragpackets, 0, ""); 7755e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD, 7855e9f80dSMike Silbersack &m_defragbytes, 0, ""); 7955e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD, 8055e9f80dSMike Silbersack &m_defraguseless, 0, ""); 8155e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD, 8255e9f80dSMike Silbersack &m_defragfailure, 0, ""); 83352d050eSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW, 84352d050eSMike Silbersack &m_defragrandomfailures, 0, ""); 85352d050eSMike Silbersack #endif 86df8bae1dSRodney W. Grimes 87df8bae1dSRodney W. Grimes /* 88f729ede6SAndre Oppermann * Ensure the correct size of various mbuf parameters. It could be off due 89f729ede6SAndre Oppermann * to compiler-induced padding and alignment artifacts. 90f729ede6SAndre Oppermann */ 91f729ede6SAndre Oppermann CTASSERT(sizeof(struct mbuf) == MSIZE); 92f729ede6SAndre Oppermann CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN); 93f729ede6SAndre Oppermann CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN); 94f729ede6SAndre Oppermann 95f729ede6SAndre Oppermann /* 9629110f87SGleb Smirnoff * m_get2() allocates minimum mbuf that would fit "size" argument. 9729110f87SGleb Smirnoff */ 9829110f87SGleb Smirnoff struct mbuf * 9941a7572bSGleb Smirnoff m_get2(int size, int how, short type, int flags) 10029110f87SGleb Smirnoff { 10129110f87SGleb Smirnoff struct mb_args args; 10229110f87SGleb Smirnoff struct mbuf *m, *n; 10329110f87SGleb Smirnoff 10429110f87SGleb Smirnoff args.flags = flags; 10529110f87SGleb Smirnoff args.type = type; 10629110f87SGleb Smirnoff 10729110f87SGleb Smirnoff if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) 10829110f87SGleb Smirnoff return (uma_zalloc_arg(zone_mbuf, &args, how)); 10929110f87SGleb Smirnoff if (size <= MCLBYTES) 11029110f87SGleb Smirnoff return (uma_zalloc_arg(zone_pack, &args, how)); 1113112ae76SGleb Smirnoff 1123112ae76SGleb Smirnoff if (size > MJUMPAGESIZE) 11329110f87SGleb Smirnoff return (NULL); 11429110f87SGleb Smirnoff 11529110f87SGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 11629110f87SGleb Smirnoff if (m == NULL) 11729110f87SGleb Smirnoff return (NULL); 11829110f87SGleb Smirnoff 1193112ae76SGleb Smirnoff n = uma_zalloc_arg(zone_jumbop, m, how); 12029110f87SGleb Smirnoff if (n == NULL) { 12129110f87SGleb Smirnoff uma_zfree(zone_mbuf, m); 12229110f87SGleb Smirnoff return (NULL); 12329110f87SGleb Smirnoff } 12429110f87SGleb Smirnoff 12529110f87SGleb Smirnoff return (m); 12629110f87SGleb Smirnoff } 12729110f87SGleb Smirnoff 12829110f87SGleb Smirnoff /* 12929110f87SGleb Smirnoff * m_getjcl() returns an mbuf with a cluster of the specified size attached. 13029110f87SGleb Smirnoff * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. 13129110f87SGleb Smirnoff */ 13229110f87SGleb Smirnoff struct mbuf * 13329110f87SGleb Smirnoff m_getjcl(int how, short type, int flags, int size) 13429110f87SGleb Smirnoff { 13529110f87SGleb Smirnoff struct mb_args args; 13629110f87SGleb Smirnoff struct mbuf *m, *n; 13729110f87SGleb Smirnoff uma_zone_t zone; 13829110f87SGleb Smirnoff 13929110f87SGleb Smirnoff if (size == MCLBYTES) 14029110f87SGleb Smirnoff return m_getcl(how, type, flags); 14129110f87SGleb Smirnoff 14229110f87SGleb Smirnoff args.flags = flags; 14329110f87SGleb Smirnoff args.type = type; 14429110f87SGleb Smirnoff 14529110f87SGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 14629110f87SGleb Smirnoff if (m == NULL) 14729110f87SGleb Smirnoff return (NULL); 14829110f87SGleb Smirnoff 14929110f87SGleb Smirnoff zone = m_getzone(size); 15029110f87SGleb Smirnoff n = uma_zalloc_arg(zone, m, how); 15129110f87SGleb Smirnoff if (n == NULL) { 15229110f87SGleb Smirnoff uma_zfree(zone_mbuf, m); 15329110f87SGleb Smirnoff return (NULL); 15429110f87SGleb Smirnoff } 15529110f87SGleb Smirnoff return (m); 15629110f87SGleb Smirnoff } 15729110f87SGleb Smirnoff 15829110f87SGleb Smirnoff /* 159099a0e58SBosko Milekic * Allocate a given length worth of mbufs and/or clusters (whatever fits 160099a0e58SBosko Milekic * best) and return a pointer to the top of the allocated chain. If an 161099a0e58SBosko Milekic * existing mbuf chain is provided, then we will append the new chain 162099a0e58SBosko Milekic * to the existing one but still return the top of the newly allocated 163099a0e58SBosko Milekic * chain. 164099a0e58SBosko Milekic */ 165099a0e58SBosko Milekic struct mbuf * 1665e20f43dSAndre Oppermann m_getm2(struct mbuf *m, int len, int how, short type, int flags) 167099a0e58SBosko Milekic { 1685e20f43dSAndre Oppermann struct mbuf *mb, *nm = NULL, *mtail = NULL; 169099a0e58SBosko Milekic 1705e20f43dSAndre Oppermann KASSERT(len >= 0, ("%s: len is < 0", __func__)); 171099a0e58SBosko Milekic 1725e20f43dSAndre Oppermann /* Validate flags. */ 1735e20f43dSAndre Oppermann flags &= (M_PKTHDR | M_EOR); 1745e20f43dSAndre Oppermann 1755e20f43dSAndre Oppermann /* Packet header mbuf must be first in chain. */ 1765e20f43dSAndre Oppermann if ((flags & M_PKTHDR) && m != NULL) 1775e20f43dSAndre Oppermann flags &= ~M_PKTHDR; 1785e20f43dSAndre Oppermann 1795e20f43dSAndre Oppermann /* Loop and append maximum sized mbufs to the chain tail. */ 1805e20f43dSAndre Oppermann while (len > 0) { 1815e20f43dSAndre Oppermann if (len > MCLBYTES) 1825e20f43dSAndre Oppermann mb = m_getjcl(how, type, (flags & M_PKTHDR), 1835e20f43dSAndre Oppermann MJUMPAGESIZE); 1845e20f43dSAndre Oppermann else if (len >= MINCLSIZE) 1855e20f43dSAndre Oppermann mb = m_getcl(how, type, (flags & M_PKTHDR)); 1865e20f43dSAndre Oppermann else if (flags & M_PKTHDR) 1875e20f43dSAndre Oppermann mb = m_gethdr(how, type); 188099a0e58SBosko Milekic else 1895e20f43dSAndre Oppermann mb = m_get(how, type); 190099a0e58SBosko Milekic 1915e20f43dSAndre Oppermann /* Fail the whole operation if one mbuf can't be allocated. */ 1925e20f43dSAndre Oppermann if (mb == NULL) { 1935e20f43dSAndre Oppermann if (nm != NULL) 1945e20f43dSAndre Oppermann m_freem(nm); 1955e20f43dSAndre Oppermann return (NULL); 196099a0e58SBosko Milekic } 197099a0e58SBosko Milekic 1985e20f43dSAndre Oppermann /* Book keeping. */ 1995e20f43dSAndre Oppermann len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size : 2005e20f43dSAndre Oppermann ((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN); 201099a0e58SBosko Milekic if (mtail != NULL) 2025e20f43dSAndre Oppermann mtail->m_next = mb; 2035e20f43dSAndre Oppermann else 2045e20f43dSAndre Oppermann nm = mb; 2055e20f43dSAndre Oppermann mtail = mb; 2065e20f43dSAndre Oppermann flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ 2075e20f43dSAndre Oppermann } 2085e20f43dSAndre Oppermann if (flags & M_EOR) 2095e20f43dSAndre Oppermann mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ 2105e20f43dSAndre Oppermann 2115e20f43dSAndre Oppermann /* If mbuf was supplied, append new chain to the end of it. */ 2125e20f43dSAndre Oppermann if (m != NULL) { 2135e20f43dSAndre Oppermann for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) 2145e20f43dSAndre Oppermann ; 2155e20f43dSAndre Oppermann mtail->m_next = nm; 2165e20f43dSAndre Oppermann mtail->m_flags &= ~M_EOR; 2175e20f43dSAndre Oppermann } else 2185e20f43dSAndre Oppermann m = nm; 2195e20f43dSAndre Oppermann 2205e20f43dSAndre Oppermann return (m); 221099a0e58SBosko Milekic } 222099a0e58SBosko Milekic 223099a0e58SBosko Milekic /* 224099a0e58SBosko Milekic * Free an entire chain of mbufs and associated external buffers, if 225099a0e58SBosko Milekic * applicable. 226099a0e58SBosko Milekic */ 227099a0e58SBosko Milekic void 228099a0e58SBosko Milekic m_freem(struct mbuf *mb) 229099a0e58SBosko Milekic { 230099a0e58SBosko Milekic 231099a0e58SBosko Milekic while (mb != NULL) 232099a0e58SBosko Milekic mb = m_free(mb); 233099a0e58SBosko Milekic } 234099a0e58SBosko Milekic 2351a996ed1SEdward Tomasz Napierala /*- 236099a0e58SBosko Milekic * Configure a provided mbuf to refer to the provided external storage 237099a0e58SBosko Milekic * buffer and setup a reference count for said buffer. If the setting 238099a0e58SBosko Milekic * up of the reference count fails, the M_EXT bit will not be set. If 239099a0e58SBosko Milekic * successfull, the M_EXT bit is set in the mbuf's flags. 240099a0e58SBosko Milekic * 241099a0e58SBosko Milekic * Arguments: 242099a0e58SBosko Milekic * mb The existing mbuf to which to attach the provided buffer. 243099a0e58SBosko Milekic * buf The address of the provided external storage buffer. 244099a0e58SBosko Milekic * size The size of the provided buffer. 245099a0e58SBosko Milekic * freef A pointer to a routine that is responsible for freeing the 246099a0e58SBosko Milekic * provided external storage buffer. 247099a0e58SBosko Milekic * args A pointer to an argument structure (of any type) to be passed 248099a0e58SBosko Milekic * to the provided freef routine (may be NULL). 249099a0e58SBosko Milekic * flags Any other flags to be passed to the provided mbuf. 250099a0e58SBosko Milekic * type The type that the external storage buffer should be 251099a0e58SBosko Milekic * labeled with. 252099a0e58SBosko Milekic * 253099a0e58SBosko Milekic * Returns: 254099a0e58SBosko Milekic * Nothing. 255099a0e58SBosko Milekic */ 2568c629bdfSGleb Smirnoff int 257099a0e58SBosko Milekic m_extadd(struct mbuf *mb, caddr_t buf, u_int size, 25815c28f87SGleb Smirnoff void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2, 2599a736876SAndre Oppermann int flags, int type, int wait) 260099a0e58SBosko Milekic { 26156a4e45aSAndre Oppermann KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); 262099a0e58SBosko Milekic 26356a4e45aSAndre Oppermann if (type != EXT_EXTREF) 264fcc34a23SGleb Smirnoff mb->m_ext.ext_cnt = uma_zalloc(zone_ext_refcnt, wait); 2658c629bdfSGleb Smirnoff 266fcc34a23SGleb Smirnoff if (mb->m_ext.ext_cnt == NULL) 2678c629bdfSGleb Smirnoff return (ENOMEM); 2688c629bdfSGleb Smirnoff 269fcc34a23SGleb Smirnoff *(mb->m_ext.ext_cnt) = 1; 270099a0e58SBosko Milekic mb->m_flags |= (M_EXT | flags); 271099a0e58SBosko Milekic mb->m_ext.ext_buf = buf; 272099a0e58SBosko Milekic mb->m_data = mb->m_ext.ext_buf; 273099a0e58SBosko Milekic mb->m_ext.ext_size = size; 274099a0e58SBosko Milekic mb->m_ext.ext_free = freef; 275cf827063SPoul-Henning Kamp mb->m_ext.ext_arg1 = arg1; 276cf827063SPoul-Henning Kamp mb->m_ext.ext_arg2 = arg2; 277099a0e58SBosko Milekic mb->m_ext.ext_type = type; 278894734cbSAndre Oppermann mb->m_ext.ext_flags = 0; 2798c629bdfSGleb Smirnoff 2808c629bdfSGleb Smirnoff return (0); 281099a0e58SBosko Milekic } 282099a0e58SBosko Milekic 283099a0e58SBosko Milekic /* 284099a0e58SBosko Milekic * Non-directly-exported function to clean up after mbufs with M_EXT 28556a4e45aSAndre Oppermann * storage attached to them if the reference count hits 1. 286099a0e58SBosko Milekic */ 287099a0e58SBosko Milekic void 288099a0e58SBosko Milekic mb_free_ext(struct mbuf *m) 289099a0e58SBosko Milekic { 2901fbe6a82SGleb Smirnoff int freembuf; 291457869b9SKip Macy 2921fbe6a82SGleb Smirnoff KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); 2933d2a3ff2SBosko Milekic 2941227f20dSAndre Oppermann /* 2951fbe6a82SGleb Smirnoff * Check if the header is embedded in the cluster. 2961227f20dSAndre Oppermann */ 2971fbe6a82SGleb Smirnoff freembuf = (m->m_flags & M_NOFREE) ? 0 : 1; 2981227f20dSAndre Oppermann 2991fbe6a82SGleb Smirnoff switch (m->m_ext.ext_type) { 3001fbe6a82SGleb Smirnoff case EXT_SFBUF: 3011fbe6a82SGleb Smirnoff sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2); 3021fbe6a82SGleb Smirnoff break; 3031fbe6a82SGleb Smirnoff default: 3041fbe6a82SGleb Smirnoff KASSERT(m->m_ext.ext_cnt != NULL, 3051fbe6a82SGleb Smirnoff ("%s: no refcounting pointer on %p", __func__, m)); 3061fbe6a82SGleb Smirnoff /* 3071fbe6a82SGleb Smirnoff * Free attached storage if this mbuf is the only 3081fbe6a82SGleb Smirnoff * reference to it. 3091fbe6a82SGleb Smirnoff */ 3101fbe6a82SGleb Smirnoff if (*(m->m_ext.ext_cnt) != 1) { 3111fbe6a82SGleb Smirnoff if (atomic_fetchadd_int(m->m_ext.ext_cnt, -1) != 1) 3121fbe6a82SGleb Smirnoff break; 3131fbe6a82SGleb Smirnoff } 3141fbe6a82SGleb Smirnoff 31556a4e45aSAndre Oppermann switch (m->m_ext.ext_type) { 316cd5bb63bSAndre Oppermann case EXT_PACKET: /* The packet zone is special. */ 317fcc34a23SGleb Smirnoff if (*(m->m_ext.ext_cnt) == 0) 318fcc34a23SGleb Smirnoff *(m->m_ext.ext_cnt) = 1; 319099a0e58SBosko Milekic uma_zfree(zone_pack, m); 32056a4e45aSAndre Oppermann return; /* Job done. */ 321cd5bb63bSAndre Oppermann case EXT_CLUSTER: 322cd5bb63bSAndre Oppermann uma_zfree(zone_clust, m->m_ext.ext_buf); 32356a4e45aSAndre Oppermann break; 324ec63cb90SAndre Oppermann case EXT_JUMBOP: 325ec63cb90SAndre Oppermann uma_zfree(zone_jumbop, m->m_ext.ext_buf); 326d5269a63SAndre Oppermann break; 32756a4e45aSAndre Oppermann case EXT_JUMBO9: 32856a4e45aSAndre Oppermann uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 32956a4e45aSAndre Oppermann break; 33056a4e45aSAndre Oppermann case EXT_JUMBO16: 33156a4e45aSAndre Oppermann uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 33256a4e45aSAndre Oppermann break; 33356a4e45aSAndre Oppermann case EXT_NET_DRV: 33456a4e45aSAndre Oppermann case EXT_MOD_TYPE: 33556a4e45aSAndre Oppermann case EXT_DISPOSABLE: 336fcc34a23SGleb Smirnoff *(m->m_ext.ext_cnt) = 0; 33756a4e45aSAndre Oppermann uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *, 338fcc34a23SGleb Smirnoff m->m_ext.ext_cnt)); 33956a4e45aSAndre Oppermann /* FALLTHROUGH */ 34056a4e45aSAndre Oppermann case EXT_EXTREF: 34156a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_free != NULL, 34256a4e45aSAndre Oppermann ("%s: ext_free not set", __func__)); 34315c28f87SGleb Smirnoff (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, 344cf827063SPoul-Henning Kamp m->m_ext.ext_arg2); 34556a4e45aSAndre Oppermann break; 34656a4e45aSAndre Oppermann default: 34756a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_type == 0, 34856a4e45aSAndre Oppermann ("%s: unknown ext_type", __func__)); 3493d2a3ff2SBosko Milekic } 35056a4e45aSAndre Oppermann } 351457869b9SKip Macy 3521fbe6a82SGleb Smirnoff if (freembuf) 35396e12413SBosko Milekic uma_zfree(zone_mbuf, m); 354b5b2ea9aSBosko Milekic } 355099a0e58SBosko Milekic 356099a0e58SBosko Milekic /* 3576bccea7cSRebecca Cran * Attach the cluster from *m to *n, set up m_ext in *n 35856a4e45aSAndre Oppermann * and bump the refcount of the cluster. 35956a4e45aSAndre Oppermann */ 36056a4e45aSAndre Oppermann static void 36156a4e45aSAndre Oppermann mb_dupcl(struct mbuf *n, struct mbuf *m) 36256a4e45aSAndre Oppermann { 36356a4e45aSAndre Oppermann 3641fbe6a82SGleb Smirnoff KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); 3651fbe6a82SGleb Smirnoff KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n)); 3661fbe6a82SGleb Smirnoff 3671fbe6a82SGleb Smirnoff switch (m->m_ext.ext_type) { 3681fbe6a82SGleb Smirnoff case EXT_SFBUF: 3691fbe6a82SGleb Smirnoff sf_ext_ref(m->m_ext.ext_arg1, m->m_ext.ext_arg2); 3701fbe6a82SGleb Smirnoff break; 3711fbe6a82SGleb Smirnoff default: 3721fbe6a82SGleb Smirnoff KASSERT(m->m_ext.ext_cnt != NULL, 3731fbe6a82SGleb Smirnoff ("%s: no refcounting pointer on %p", __func__, m)); 374fcc34a23SGleb Smirnoff if (*(m->m_ext.ext_cnt) == 1) 375fcc34a23SGleb Smirnoff *(m->m_ext.ext_cnt) += 1; 37656a4e45aSAndre Oppermann else 377fcc34a23SGleb Smirnoff atomic_add_int(m->m_ext.ext_cnt, 1); 3781fbe6a82SGleb Smirnoff } 3791fbe6a82SGleb Smirnoff 380c71b4037SGleb Smirnoff n->m_ext = m->m_ext; 38156a4e45aSAndre Oppermann n->m_flags |= M_EXT; 38232a8b1d8SColin Percival n->m_flags |= m->m_flags & M_RDONLY; 38356a4e45aSAndre Oppermann } 38456a4e45aSAndre Oppermann 38556a4e45aSAndre Oppermann /* 386ed111688SAndre Oppermann * Clean up mbuf (chain) from any tags and packet headers. 387e0068c3aSAndre Oppermann * If "all" is set then the first mbuf in the chain will be 388e0068c3aSAndre Oppermann * cleaned too. 389ed111688SAndre Oppermann */ 390ed111688SAndre Oppermann void 391651e4e6aSGleb Smirnoff m_demote(struct mbuf *m0, int all, int flags) 392ed111688SAndre Oppermann { 393ed111688SAndre Oppermann struct mbuf *m; 394ed111688SAndre Oppermann 395ed111688SAndre Oppermann for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) { 3967ee2d058SGleb Smirnoff KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p", 3977ee2d058SGleb Smirnoff __func__, m, m0)); 398ed111688SAndre Oppermann if (m->m_flags & M_PKTHDR) { 399ed111688SAndre Oppermann m_tag_delete_chain(m, NULL); 400ed111688SAndre Oppermann m->m_flags &= ~M_PKTHDR; 401ed111688SAndre Oppermann bzero(&m->m_pkthdr, sizeof(struct pkthdr)); 402ed111688SAndre Oppermann } 403651e4e6aSGleb Smirnoff m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags); 404ed111688SAndre Oppermann } 405ed111688SAndre Oppermann } 406ed111688SAndre Oppermann 407ed111688SAndre Oppermann /* 408fdcc028dSAndre Oppermann * Sanity checks on mbuf (chain) for use in KASSERT() and general 409fdcc028dSAndre Oppermann * debugging. 410fdcc028dSAndre Oppermann * Returns 0 or panics when bad and 1 on all tests passed. 411fdcc028dSAndre Oppermann * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they 412fdcc028dSAndre Oppermann * blow up later. 413a048affbSAndre Oppermann */ 414a048affbSAndre Oppermann int 415a048affbSAndre Oppermann m_sanity(struct mbuf *m0, int sanitize) 416a048affbSAndre Oppermann { 417a048affbSAndre Oppermann struct mbuf *m; 418a048affbSAndre Oppermann caddr_t a, b; 419a048affbSAndre Oppermann int pktlen = 0; 420a048affbSAndre Oppermann 42121ee3e7aSKip Macy #ifdef INVARIANTS 42221ee3e7aSKip Macy #define M_SANITY_ACTION(s) panic("mbuf %p: " s, m) 42321ee3e7aSKip Macy #else 42421ee3e7aSKip Macy #define M_SANITY_ACTION(s) printf("mbuf %p: " s, m) 42521ee3e7aSKip Macy #endif 426a048affbSAndre Oppermann 427fdcc028dSAndre Oppermann for (m = m0; m != NULL; m = m->m_next) { 428a048affbSAndre Oppermann /* 429a048affbSAndre Oppermann * Basic pointer checks. If any of these fails then some 430a048affbSAndre Oppermann * unrelated kernel memory before or after us is trashed. 431a048affbSAndre Oppermann * No way to recover from that. 432a048affbSAndre Oppermann */ 433fdcc028dSAndre Oppermann a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf : 434fdcc028dSAndre Oppermann ((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) : 435a048affbSAndre Oppermann (caddr_t)(&m->m_dat)) ); 436a048affbSAndre Oppermann b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size : 437fdcc028dSAndre Oppermann ((m->m_flags & M_PKTHDR) ? MHLEN : MLEN))); 438a048affbSAndre Oppermann if ((caddr_t)m->m_data < a) 439a048affbSAndre Oppermann M_SANITY_ACTION("m_data outside mbuf data range left"); 440a048affbSAndre Oppermann if ((caddr_t)m->m_data > b) 441a048affbSAndre Oppermann M_SANITY_ACTION("m_data outside mbuf data range right"); 442a048affbSAndre Oppermann if ((caddr_t)m->m_data + m->m_len > b) 443a048affbSAndre Oppermann M_SANITY_ACTION("m_data + m_len exeeds mbuf space"); 444a048affbSAndre Oppermann 445a048affbSAndre Oppermann /* m->m_nextpkt may only be set on first mbuf in chain. */ 446fdcc028dSAndre Oppermann if (m != m0 && m->m_nextpkt != NULL) { 447a048affbSAndre Oppermann if (sanitize) { 448a048affbSAndre Oppermann m_freem(m->m_nextpkt); 449a048affbSAndre Oppermann m->m_nextpkt = (struct mbuf *)0xDEADC0DE; 450a048affbSAndre Oppermann } else 451a048affbSAndre Oppermann M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf"); 452a048affbSAndre Oppermann } 453a048affbSAndre Oppermann 454a048affbSAndre Oppermann /* packet length (not mbuf length!) calculation */ 455a048affbSAndre Oppermann if (m0->m_flags & M_PKTHDR) 456a048affbSAndre Oppermann pktlen += m->m_len; 457a048affbSAndre Oppermann 458a048affbSAndre Oppermann /* m_tags may only be attached to first mbuf in chain. */ 459a048affbSAndre Oppermann if (m != m0 && m->m_flags & M_PKTHDR && 460a048affbSAndre Oppermann !SLIST_EMPTY(&m->m_pkthdr.tags)) { 461a048affbSAndre Oppermann if (sanitize) { 462a048affbSAndre Oppermann m_tag_delete_chain(m, NULL); 463a048affbSAndre Oppermann /* put in 0xDEADC0DE perhaps? */ 464fdcc028dSAndre Oppermann } else 465a048affbSAndre Oppermann M_SANITY_ACTION("m_tags on in-chain mbuf"); 466a048affbSAndre Oppermann } 467a048affbSAndre Oppermann 468a048affbSAndre Oppermann /* M_PKTHDR may only be set on first mbuf in chain */ 469a048affbSAndre Oppermann if (m != m0 && m->m_flags & M_PKTHDR) { 470a048affbSAndre Oppermann if (sanitize) { 471a048affbSAndre Oppermann bzero(&m->m_pkthdr, sizeof(m->m_pkthdr)); 472a048affbSAndre Oppermann m->m_flags &= ~M_PKTHDR; 473a048affbSAndre Oppermann /* put in 0xDEADCODE and leave hdr flag in */ 474a048affbSAndre Oppermann } else 475a048affbSAndre Oppermann M_SANITY_ACTION("M_PKTHDR on in-chain mbuf"); 476a048affbSAndre Oppermann } 477a048affbSAndre Oppermann } 478fdcc028dSAndre Oppermann m = m0; 479fdcc028dSAndre Oppermann if (pktlen && pktlen != m->m_pkthdr.len) { 480a048affbSAndre Oppermann if (sanitize) 481fdcc028dSAndre Oppermann m->m_pkthdr.len = 0; 482a048affbSAndre Oppermann else 483a048affbSAndre Oppermann M_SANITY_ACTION("m_pkthdr.len != mbuf chain length"); 484a048affbSAndre Oppermann } 485a048affbSAndre Oppermann return 1; 486fdcc028dSAndre Oppermann 487fdcc028dSAndre Oppermann #undef M_SANITY_ACTION 488a048affbSAndre Oppermann } 489a048affbSAndre Oppermann 490a048affbSAndre Oppermann 491a048affbSAndre Oppermann /* 4929967cafcSSam Leffler * "Move" mbuf pkthdr from "from" to "to". 493e37b1fcdSRobert Watson * "from" must have M_PKTHDR set, and "to" must be empty. 494e37b1fcdSRobert Watson */ 495e37b1fcdSRobert Watson void 4969967cafcSSam Leffler m_move_pkthdr(struct mbuf *to, struct mbuf *from) 497e37b1fcdSRobert Watson { 498e37b1fcdSRobert Watson 499e37b1fcdSRobert Watson #if 0 5009967cafcSSam Leffler /* see below for why these are not enabled */ 501fe584538SDag-Erling Smørgrav M_ASSERTPKTHDR(to); 502225bff6fSRobert Watson /* Note: with MAC, this may not be a good assertion. */ 5039967cafcSSam Leffler KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), 5049967cafcSSam Leffler ("m_move_pkthdr: to has tags")); 505e37b1fcdSRobert Watson #endif 506e32a5b94SRobert Watson #ifdef MAC 507225bff6fSRobert Watson /* 508225bff6fSRobert Watson * XXXMAC: It could be this should also occur for non-MAC? 509225bff6fSRobert Watson */ 510e32a5b94SRobert Watson if (to->m_flags & M_PKTHDR) 511225bff6fSRobert Watson m_tag_delete_chain(to, NULL); 512e32a5b94SRobert Watson #endif 513a4e71429SSam Leffler to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 514a4e71429SSam Leffler if ((to->m_flags & M_EXT) == 0) 5159967cafcSSam Leffler to->m_data = to->m_pktdat; 5169967cafcSSam Leffler to->m_pkthdr = from->m_pkthdr; /* especially tags */ 5179967cafcSSam Leffler SLIST_INIT(&from->m_pkthdr.tags); /* purge tags from src */ 5189967cafcSSam Leffler from->m_flags &= ~M_PKTHDR; 5199967cafcSSam Leffler } 5209967cafcSSam Leffler 5219967cafcSSam Leffler /* 5229967cafcSSam Leffler * Duplicate "from"'s mbuf pkthdr in "to". 5239967cafcSSam Leffler * "from" must have M_PKTHDR set, and "to" must be empty. 5249967cafcSSam Leffler * In particular, this does a deep copy of the packet tags. 5259967cafcSSam Leffler */ 5269967cafcSSam Leffler int 5279967cafcSSam Leffler m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how) 5289967cafcSSam Leffler { 5299967cafcSSam Leffler 5309967cafcSSam Leffler #if 0 5319967cafcSSam Leffler /* 5329967cafcSSam Leffler * The mbuf allocator only initializes the pkthdr 533c95be8b5SGleb Smirnoff * when the mbuf is allocated with m_gethdr(). Many users 534c95be8b5SGleb Smirnoff * (e.g. m_copy*, m_prepend) use m_get() and then 5359967cafcSSam Leffler * smash the pkthdr as needed causing these 5369967cafcSSam Leffler * assertions to trip. For now just disable them. 5379967cafcSSam Leffler */ 538fe584538SDag-Erling Smørgrav M_ASSERTPKTHDR(to); 539225bff6fSRobert Watson /* Note: with MAC, this may not be a good assertion. */ 5409967cafcSSam Leffler KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags")); 5419967cafcSSam Leffler #endif 542063d8114SAlfred Perlstein MBUF_CHECKSLEEP(how); 5439967cafcSSam Leffler #ifdef MAC 5449967cafcSSam Leffler if (to->m_flags & M_PKTHDR) 545225bff6fSRobert Watson m_tag_delete_chain(to, NULL); 5469967cafcSSam Leffler #endif 547df8c7fc9SMike Silbersack to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT); 548df8c7fc9SMike Silbersack if ((to->m_flags & M_EXT) == 0) 5499967cafcSSam Leffler to->m_data = to->m_pktdat; 550e37b1fcdSRobert Watson to->m_pkthdr = from->m_pkthdr; 5519967cafcSSam Leffler SLIST_INIT(&to->m_pkthdr.tags); 55294985f74SGleb Smirnoff return (m_tag_copy_chain(to, from, how)); 553e37b1fcdSRobert Watson } 554e37b1fcdSRobert Watson 555e37b1fcdSRobert Watson /* 556df8bae1dSRodney W. Grimes * Lesser-used path for M_PREPEND: 557df8bae1dSRodney W. Grimes * allocate new mbuf to prepend to chain, 558df8bae1dSRodney W. Grimes * copy junk along. 559df8bae1dSRodney W. Grimes */ 560df8bae1dSRodney W. Grimes struct mbuf * 561122a814aSBosko Milekic m_prepend(struct mbuf *m, int len, int how) 562df8bae1dSRodney W. Grimes { 563df8bae1dSRodney W. Grimes struct mbuf *mn; 564df8bae1dSRodney W. Grimes 565f8bf8e39SMike Silbersack if (m->m_flags & M_PKTHDR) 566c95be8b5SGleb Smirnoff mn = m_gethdr(how, m->m_type); 567f8bf8e39SMike Silbersack else 568c95be8b5SGleb Smirnoff mn = m_get(how, m->m_type); 569122a814aSBosko Milekic if (mn == NULL) { 570df8bae1dSRodney W. Grimes m_freem(m); 571122a814aSBosko Milekic return (NULL); 572df8bae1dSRodney W. Grimes } 573225bff6fSRobert Watson if (m->m_flags & M_PKTHDR) 574c95be8b5SGleb Smirnoff m_move_pkthdr(mn, m); 575df8bae1dSRodney W. Grimes mn->m_next = m; 576df8bae1dSRodney W. Grimes m = mn; 577*ed6a66caSRobert Watson if (len < M_SIZE(m)) 5785288989fSRandall Stewart M_ALIGN(m, len); 579df8bae1dSRodney W. Grimes m->m_len = len; 580df8bae1dSRodney W. Grimes return (m); 581df8bae1dSRodney W. Grimes } 582df8bae1dSRodney W. Grimes 583df8bae1dSRodney W. Grimes /* 584df8bae1dSRodney W. Grimes * Make a copy of an mbuf chain starting "off0" bytes from the beginning, 585df8bae1dSRodney W. Grimes * continuing for "len" bytes. If len is M_COPYALL, copy to end of mbuf. 586eb1b1807SGleb Smirnoff * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller. 5871c38f2eaSArchie Cobbs * Note that the copy is read-only, because clusters are not copied, 5881c38f2eaSArchie Cobbs * only their reference counts are incremented. 589df8bae1dSRodney W. Grimes */ 590df8bae1dSRodney W. Grimes struct mbuf * 591122a814aSBosko Milekic m_copym(struct mbuf *m, int off0, int len, int wait) 592df8bae1dSRodney W. Grimes { 593122a814aSBosko Milekic struct mbuf *n, **np; 594122a814aSBosko Milekic int off = off0; 595df8bae1dSRodney W. Grimes struct mbuf *top; 596df8bae1dSRodney W. Grimes int copyhdr = 0; 597df8bae1dSRodney W. Grimes 598e0a653ddSAlfred Perlstein KASSERT(off >= 0, ("m_copym, negative off %d", off)); 599e0a653ddSAlfred Perlstein KASSERT(len >= 0, ("m_copym, negative len %d", len)); 600063d8114SAlfred Perlstein MBUF_CHECKSLEEP(wait); 601df8bae1dSRodney W. Grimes if (off == 0 && m->m_flags & M_PKTHDR) 602df8bae1dSRodney W. Grimes copyhdr = 1; 603df8bae1dSRodney W. Grimes while (off > 0) { 604e0a653ddSAlfred Perlstein KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain")); 605df8bae1dSRodney W. Grimes if (off < m->m_len) 606df8bae1dSRodney W. Grimes break; 607df8bae1dSRodney W. Grimes off -= m->m_len; 608df8bae1dSRodney W. Grimes m = m->m_next; 609df8bae1dSRodney W. Grimes } 610df8bae1dSRodney W. Grimes np = ⊤ 611df8bae1dSRodney W. Grimes top = 0; 612df8bae1dSRodney W. Grimes while (len > 0) { 613122a814aSBosko Milekic if (m == NULL) { 614e0a653ddSAlfred Perlstein KASSERT(len == M_COPYALL, 615e0a653ddSAlfred Perlstein ("m_copym, length > size of mbuf chain")); 616df8bae1dSRodney W. Grimes break; 617df8bae1dSRodney W. Grimes } 618f8bf8e39SMike Silbersack if (copyhdr) 619c95be8b5SGleb Smirnoff n = m_gethdr(wait, m->m_type); 620f8bf8e39SMike Silbersack else 621c95be8b5SGleb Smirnoff n = m_get(wait, m->m_type); 622df8bae1dSRodney W. Grimes *np = n; 623122a814aSBosko Milekic if (n == NULL) 624df8bae1dSRodney W. Grimes goto nospace; 625df8bae1dSRodney W. Grimes if (copyhdr) { 6269967cafcSSam Leffler if (!m_dup_pkthdr(n, m, wait)) 6279967cafcSSam Leffler goto nospace; 628df8bae1dSRodney W. Grimes if (len == M_COPYALL) 629df8bae1dSRodney W. Grimes n->m_pkthdr.len -= off0; 630df8bae1dSRodney W. Grimes else 631df8bae1dSRodney W. Grimes n->m_pkthdr.len = len; 632df8bae1dSRodney W. Grimes copyhdr = 0; 633df8bae1dSRodney W. Grimes } 634df8bae1dSRodney W. Grimes n->m_len = min(len, m->m_len - off); 635df8bae1dSRodney W. Grimes if (m->m_flags & M_EXT) { 636df8bae1dSRodney W. Grimes n->m_data = m->m_data + off; 63756a4e45aSAndre Oppermann mb_dupcl(n, m); 638df8bae1dSRodney W. Grimes } else 639df8bae1dSRodney W. Grimes bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t), 640bd395ae8SBosko Milekic (u_int)n->m_len); 641df8bae1dSRodney W. Grimes if (len != M_COPYALL) 642df8bae1dSRodney W. Grimes len -= n->m_len; 643df8bae1dSRodney W. Grimes off = 0; 644df8bae1dSRodney W. Grimes m = m->m_next; 645df8bae1dSRodney W. Grimes np = &n->m_next; 646df8bae1dSRodney W. Grimes } 64708442f8aSBosko Milekic 648df8bae1dSRodney W. Grimes return (top); 649df8bae1dSRodney W. Grimes nospace: 650df8bae1dSRodney W. Grimes m_freem(top); 651122a814aSBosko Milekic return (NULL); 652df8bae1dSRodney W. Grimes } 653df8bae1dSRodney W. Grimes 654df8bae1dSRodney W. Grimes /* 6554da84431SAndre Oppermann * Returns mbuf chain with new head for the prepending case. 6564da84431SAndre Oppermann * Copies from mbuf (chain) n from off for len to mbuf (chain) m 6574da84431SAndre Oppermann * either prepending or appending the data. 6584da84431SAndre Oppermann * The resulting mbuf (chain) m is fully writeable. 6594da84431SAndre Oppermann * m is destination (is made writeable) 6604da84431SAndre Oppermann * n is source, off is offset in source, len is len from offset 6614da84431SAndre Oppermann * dir, 0 append, 1 prepend 6624da84431SAndre Oppermann * how, wait or nowait 6634da84431SAndre Oppermann */ 6644da84431SAndre Oppermann 6654da84431SAndre Oppermann static int 6664da84431SAndre Oppermann m_bcopyxxx(void *s, void *t, u_int len) 6674da84431SAndre Oppermann { 6684da84431SAndre Oppermann bcopy(s, t, (size_t)len); 6694da84431SAndre Oppermann return 0; 6704da84431SAndre Oppermann } 6714da84431SAndre Oppermann 6724da84431SAndre Oppermann struct mbuf * 6734da84431SAndre Oppermann m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len, 6744da84431SAndre Oppermann int prep, int how) 6754da84431SAndre Oppermann { 67656a4e45aSAndre Oppermann struct mbuf *mm, *x, *z, *prev = NULL; 6774da84431SAndre Oppermann caddr_t p; 67856a4e45aSAndre Oppermann int i, nlen = 0; 6794da84431SAndre Oppermann caddr_t buf[MLEN]; 6804da84431SAndre Oppermann 6814da84431SAndre Oppermann KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source")); 6824da84431SAndre Oppermann KASSERT(off >= 0, ("m_copymdata, negative off %d", off)); 6834da84431SAndre Oppermann KASSERT(len >= 0, ("m_copymdata, negative len %d", len)); 6844da84431SAndre Oppermann KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep)); 6854da84431SAndre Oppermann 68656a4e45aSAndre Oppermann mm = m; 68756a4e45aSAndre Oppermann if (!prep) { 68856a4e45aSAndre Oppermann while(mm->m_next) { 68956a4e45aSAndre Oppermann prev = mm; 69056a4e45aSAndre Oppermann mm = mm->m_next; 6914da84431SAndre Oppermann } 6924da84431SAndre Oppermann } 6934da84431SAndre Oppermann for (z = n; z != NULL; z = z->m_next) 6944da84431SAndre Oppermann nlen += z->m_len; 6954da84431SAndre Oppermann if (len == M_COPYALL) 6964da84431SAndre Oppermann len = nlen - off; 6974da84431SAndre Oppermann if (off + len > nlen || len < 1) 6984da84431SAndre Oppermann return NULL; 6994da84431SAndre Oppermann 70056a4e45aSAndre Oppermann if (!M_WRITABLE(mm)) { 70156a4e45aSAndre Oppermann /* XXX: Use proper m_xxx function instead. */ 70256a4e45aSAndre Oppermann x = m_getcl(how, MT_DATA, mm->m_flags); 70356a4e45aSAndre Oppermann if (x == NULL) 70456a4e45aSAndre Oppermann return NULL; 70556a4e45aSAndre Oppermann bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size); 70656a4e45aSAndre Oppermann p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf); 70756a4e45aSAndre Oppermann x->m_data = p; 70856a4e45aSAndre Oppermann mm->m_next = NULL; 70956a4e45aSAndre Oppermann if (mm != m) 71056a4e45aSAndre Oppermann prev->m_next = x; 71156a4e45aSAndre Oppermann m_free(mm); 71256a4e45aSAndre Oppermann mm = x; 71356a4e45aSAndre Oppermann } 71456a4e45aSAndre Oppermann 7154da84431SAndre Oppermann /* 7164da84431SAndre Oppermann * Append/prepend the data. Allocating mbufs as necessary. 7174da84431SAndre Oppermann */ 7184da84431SAndre Oppermann /* Shortcut if enough free space in first/last mbuf. */ 7194da84431SAndre Oppermann if (!prep && M_TRAILINGSPACE(mm) >= len) { 7204da84431SAndre Oppermann m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) + 7214da84431SAndre Oppermann mm->m_len); 7224da84431SAndre Oppermann mm->m_len += len; 7234da84431SAndre Oppermann mm->m_pkthdr.len += len; 7244da84431SAndre Oppermann return m; 7254da84431SAndre Oppermann } 7264da84431SAndre Oppermann if (prep && M_LEADINGSPACE(mm) >= len) { 7274da84431SAndre Oppermann mm->m_data = mtod(mm, caddr_t) - len; 7284da84431SAndre Oppermann m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t)); 7294da84431SAndre Oppermann mm->m_len += len; 7304da84431SAndre Oppermann mm->m_pkthdr.len += len; 7314da84431SAndre Oppermann return mm; 7324da84431SAndre Oppermann } 7334da84431SAndre Oppermann 7344da84431SAndre Oppermann /* Expand first/last mbuf to cluster if possible. */ 7354da84431SAndre Oppermann if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) { 7364da84431SAndre Oppermann bcopy(mm->m_data, &buf, mm->m_len); 7374da84431SAndre Oppermann m_clget(mm, how); 7384da84431SAndre Oppermann if (!(mm->m_flags & M_EXT)) 7394da84431SAndre Oppermann return NULL; 7404da84431SAndre Oppermann bcopy(&buf, mm->m_ext.ext_buf, mm->m_len); 7414da84431SAndre Oppermann mm->m_data = mm->m_ext.ext_buf; 7424da84431SAndre Oppermann } 7434da84431SAndre Oppermann if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) { 7444da84431SAndre Oppermann bcopy(mm->m_data, &buf, mm->m_len); 7454da84431SAndre Oppermann m_clget(mm, how); 7464da84431SAndre Oppermann if (!(mm->m_flags & M_EXT)) 7474da84431SAndre Oppermann return NULL; 7484da84431SAndre Oppermann bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf + 7494da84431SAndre Oppermann mm->m_ext.ext_size - mm->m_len, mm->m_len); 7504da84431SAndre Oppermann mm->m_data = (caddr_t)mm->m_ext.ext_buf + 7514da84431SAndre Oppermann mm->m_ext.ext_size - mm->m_len; 7524da84431SAndre Oppermann } 7534da84431SAndre Oppermann 7544da84431SAndre Oppermann /* Append/prepend as many mbuf (clusters) as necessary to fit len. */ 7554da84431SAndre Oppermann if (!prep && len > M_TRAILINGSPACE(mm)) { 7564da84431SAndre Oppermann if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA)) 7574da84431SAndre Oppermann return NULL; 7584da84431SAndre Oppermann } 7594da84431SAndre Oppermann if (prep && len > M_LEADINGSPACE(mm)) { 7604da84431SAndre Oppermann if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA))) 7614da84431SAndre Oppermann return NULL; 7624da84431SAndre Oppermann i = 0; 7634da84431SAndre Oppermann for (x = z; x != NULL; x = x->m_next) { 7644da84431SAndre Oppermann i += x->m_flags & M_EXT ? x->m_ext.ext_size : 7654da84431SAndre Oppermann (x->m_flags & M_PKTHDR ? MHLEN : MLEN); 7664da84431SAndre Oppermann if (!x->m_next) 7674da84431SAndre Oppermann break; 7684da84431SAndre Oppermann } 7694da84431SAndre Oppermann z->m_data += i - len; 7704da84431SAndre Oppermann m_move_pkthdr(mm, z); 7714da84431SAndre Oppermann x->m_next = mm; 7724da84431SAndre Oppermann mm = z; 7734da84431SAndre Oppermann } 7744da84431SAndre Oppermann 7754da84431SAndre Oppermann /* Seek to start position in source mbuf. Optimization for long chains. */ 7764da84431SAndre Oppermann while (off > 0) { 7774da84431SAndre Oppermann if (off < n->m_len) 7784da84431SAndre Oppermann break; 7794da84431SAndre Oppermann off -= n->m_len; 7804da84431SAndre Oppermann n = n->m_next; 7814da84431SAndre Oppermann } 7824da84431SAndre Oppermann 7834da84431SAndre Oppermann /* Copy data into target mbuf. */ 7844da84431SAndre Oppermann z = mm; 7854da84431SAndre Oppermann while (len > 0) { 7864da84431SAndre Oppermann KASSERT(z != NULL, ("m_copymdata, falling off target edge")); 7874da84431SAndre Oppermann i = M_TRAILINGSPACE(z); 7884da84431SAndre Oppermann m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len); 7894da84431SAndre Oppermann z->m_len += i; 7904da84431SAndre Oppermann /* fixup pkthdr.len if necessary */ 7914da84431SAndre Oppermann if ((prep ? mm : m)->m_flags & M_PKTHDR) 7924da84431SAndre Oppermann (prep ? mm : m)->m_pkthdr.len += i; 7934da84431SAndre Oppermann off += i; 7944da84431SAndre Oppermann len -= i; 7954da84431SAndre Oppermann z = z->m_next; 7964da84431SAndre Oppermann } 7974da84431SAndre Oppermann return (prep ? mm : m); 7984da84431SAndre Oppermann } 7994da84431SAndre Oppermann 8004da84431SAndre Oppermann /* 8016a06dea0SGarrett Wollman * Copy an entire packet, including header (which must be present). 8026a06dea0SGarrett Wollman * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'. 8031c38f2eaSArchie Cobbs * Note that the copy is read-only, because clusters are not copied, 8041c38f2eaSArchie Cobbs * only their reference counts are incremented. 8055fe86675SLuigi Rizzo * Preserve alignment of the first mbuf so if the creator has left 8065fe86675SLuigi Rizzo * some room at the beginning (e.g. for inserting protocol headers) 8075fe86675SLuigi Rizzo * the copies still have the room available. 8086a06dea0SGarrett Wollman */ 8096a06dea0SGarrett Wollman struct mbuf * 810122a814aSBosko Milekic m_copypacket(struct mbuf *m, int how) 8116a06dea0SGarrett Wollman { 8126a06dea0SGarrett Wollman struct mbuf *top, *n, *o; 8136a06dea0SGarrett Wollman 814063d8114SAlfred Perlstein MBUF_CHECKSLEEP(how); 815c95be8b5SGleb Smirnoff n = m_get(how, m->m_type); 8166a06dea0SGarrett Wollman top = n; 817122a814aSBosko Milekic if (n == NULL) 8186a06dea0SGarrett Wollman goto nospace; 8196a06dea0SGarrett Wollman 8209967cafcSSam Leffler if (!m_dup_pkthdr(n, m, how)) 8219967cafcSSam Leffler goto nospace; 8226a06dea0SGarrett Wollman n->m_len = m->m_len; 8236a06dea0SGarrett Wollman if (m->m_flags & M_EXT) { 8246a06dea0SGarrett Wollman n->m_data = m->m_data; 82556a4e45aSAndre Oppermann mb_dupcl(n, m); 8266a06dea0SGarrett Wollman } else { 8275fe86675SLuigi Rizzo n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat ); 8286a06dea0SGarrett Wollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 8296a06dea0SGarrett Wollman } 8306a06dea0SGarrett Wollman 8316a06dea0SGarrett Wollman m = m->m_next; 8326a06dea0SGarrett Wollman while (m) { 833c95be8b5SGleb Smirnoff o = m_get(how, m->m_type); 834122a814aSBosko Milekic if (o == NULL) 8356a06dea0SGarrett Wollman goto nospace; 8366a06dea0SGarrett Wollman 8376a06dea0SGarrett Wollman n->m_next = o; 8386a06dea0SGarrett Wollman n = n->m_next; 8396a06dea0SGarrett Wollman 8406a06dea0SGarrett Wollman n->m_len = m->m_len; 8416a06dea0SGarrett Wollman if (m->m_flags & M_EXT) { 8426a06dea0SGarrett Wollman n->m_data = m->m_data; 84356a4e45aSAndre Oppermann mb_dupcl(n, m); 8446a06dea0SGarrett Wollman } else { 8456a06dea0SGarrett Wollman bcopy(mtod(m, char *), mtod(n, char *), n->m_len); 8466a06dea0SGarrett Wollman } 8476a06dea0SGarrett Wollman 8486a06dea0SGarrett Wollman m = m->m_next; 8496a06dea0SGarrett Wollman } 8506a06dea0SGarrett Wollman return top; 8516a06dea0SGarrett Wollman nospace: 8526a06dea0SGarrett Wollman m_freem(top); 853122a814aSBosko Milekic return (NULL); 8546a06dea0SGarrett Wollman } 8556a06dea0SGarrett Wollman 8566a06dea0SGarrett Wollman /* 857df8bae1dSRodney W. Grimes * Copy data from an mbuf chain starting "off" bytes from the beginning, 858df8bae1dSRodney W. Grimes * continuing for "len" bytes, into the indicated buffer. 859df8bae1dSRodney W. Grimes */ 86026f9a767SRodney W. Grimes void 861a8cfc0eeSJulian Elischer m_copydata(const struct mbuf *m, int off, int len, caddr_t cp) 862df8bae1dSRodney W. Grimes { 863bd395ae8SBosko Milekic u_int count; 864df8bae1dSRodney W. Grimes 865e0a653ddSAlfred Perlstein KASSERT(off >= 0, ("m_copydata, negative off %d", off)); 866e0a653ddSAlfred Perlstein KASSERT(len >= 0, ("m_copydata, negative len %d", len)); 867df8bae1dSRodney W. Grimes while (off > 0) { 868e0a653ddSAlfred Perlstein KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain")); 869df8bae1dSRodney W. Grimes if (off < m->m_len) 870df8bae1dSRodney W. Grimes break; 871df8bae1dSRodney W. Grimes off -= m->m_len; 872df8bae1dSRodney W. Grimes m = m->m_next; 873df8bae1dSRodney W. Grimes } 874df8bae1dSRodney W. Grimes while (len > 0) { 875e0a653ddSAlfred Perlstein KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain")); 876df8bae1dSRodney W. Grimes count = min(m->m_len - off, len); 877df8bae1dSRodney W. Grimes bcopy(mtod(m, caddr_t) + off, cp, count); 878df8bae1dSRodney W. Grimes len -= count; 879df8bae1dSRodney W. Grimes cp += count; 880df8bae1dSRodney W. Grimes off = 0; 881df8bae1dSRodney W. Grimes m = m->m_next; 882df8bae1dSRodney W. Grimes } 883df8bae1dSRodney W. Grimes } 884df8bae1dSRodney W. Grimes 885df8bae1dSRodney W. Grimes /* 8861c38f2eaSArchie Cobbs * Copy a packet header mbuf chain into a completely new chain, including 8871c38f2eaSArchie Cobbs * copying any mbuf clusters. Use this instead of m_copypacket() when 8881c38f2eaSArchie Cobbs * you need a writable copy of an mbuf chain. 8891c38f2eaSArchie Cobbs */ 8901c38f2eaSArchie Cobbs struct mbuf * 891122a814aSBosko Milekic m_dup(struct mbuf *m, int how) 8921c38f2eaSArchie Cobbs { 8931c38f2eaSArchie Cobbs struct mbuf **p, *top = NULL; 8941c38f2eaSArchie Cobbs int remain, moff, nsize; 8951c38f2eaSArchie Cobbs 896063d8114SAlfred Perlstein MBUF_CHECKSLEEP(how); 8971c38f2eaSArchie Cobbs /* Sanity check */ 8981c38f2eaSArchie Cobbs if (m == NULL) 899122a814aSBosko Milekic return (NULL); 900fe584538SDag-Erling Smørgrav M_ASSERTPKTHDR(m); 9011c38f2eaSArchie Cobbs 9021c38f2eaSArchie Cobbs /* While there's more data, get a new mbuf, tack it on, and fill it */ 9031c38f2eaSArchie Cobbs remain = m->m_pkthdr.len; 9041c38f2eaSArchie Cobbs moff = 0; 9051c38f2eaSArchie Cobbs p = ⊤ 9061c38f2eaSArchie Cobbs while (remain > 0 || top == NULL) { /* allow m->m_pkthdr.len == 0 */ 9071c38f2eaSArchie Cobbs struct mbuf *n; 9081c38f2eaSArchie Cobbs 9091c38f2eaSArchie Cobbs /* Get the next new mbuf */ 910099a0e58SBosko Milekic if (remain >= MINCLSIZE) { 911099a0e58SBosko Milekic n = m_getcl(how, m->m_type, 0); 912099a0e58SBosko Milekic nsize = MCLBYTES; 913099a0e58SBosko Milekic } else { 914099a0e58SBosko Milekic n = m_get(how, m->m_type); 915099a0e58SBosko Milekic nsize = MLEN; 916099a0e58SBosko Milekic } 9171c38f2eaSArchie Cobbs if (n == NULL) 9181c38f2eaSArchie Cobbs goto nospace; 919099a0e58SBosko Milekic 920099a0e58SBosko Milekic if (top == NULL) { /* First one, must be PKTHDR */ 921099a0e58SBosko Milekic if (!m_dup_pkthdr(n, m, how)) { 922099a0e58SBosko Milekic m_free(n); 9231c38f2eaSArchie Cobbs goto nospace; 9241c38f2eaSArchie Cobbs } 92563e6f390SEd Maste if ((n->m_flags & M_EXT) == 0) 926099a0e58SBosko Milekic nsize = MHLEN; 9271c38f2eaSArchie Cobbs } 9281c38f2eaSArchie Cobbs n->m_len = 0; 9291c38f2eaSArchie Cobbs 9301c38f2eaSArchie Cobbs /* Link it into the new chain */ 9311c38f2eaSArchie Cobbs *p = n; 9321c38f2eaSArchie Cobbs p = &n->m_next; 9331c38f2eaSArchie Cobbs 9341c38f2eaSArchie Cobbs /* Copy data from original mbuf(s) into new mbuf */ 9351c38f2eaSArchie Cobbs while (n->m_len < nsize && m != NULL) { 9361c38f2eaSArchie Cobbs int chunk = min(nsize - n->m_len, m->m_len - moff); 9371c38f2eaSArchie Cobbs 9381c38f2eaSArchie Cobbs bcopy(m->m_data + moff, n->m_data + n->m_len, chunk); 9391c38f2eaSArchie Cobbs moff += chunk; 9401c38f2eaSArchie Cobbs n->m_len += chunk; 9411c38f2eaSArchie Cobbs remain -= chunk; 9421c38f2eaSArchie Cobbs if (moff == m->m_len) { 9431c38f2eaSArchie Cobbs m = m->m_next; 9441c38f2eaSArchie Cobbs moff = 0; 9451c38f2eaSArchie Cobbs } 9461c38f2eaSArchie Cobbs } 9471c38f2eaSArchie Cobbs 9481c38f2eaSArchie Cobbs /* Check correct total mbuf length */ 9491c38f2eaSArchie Cobbs KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL), 950a48740b6SDavid E. O'Brien ("%s: bogus m_pkthdr.len", __func__)); 9511c38f2eaSArchie Cobbs } 9521c38f2eaSArchie Cobbs return (top); 9531c38f2eaSArchie Cobbs 9541c38f2eaSArchie Cobbs nospace: 9551c38f2eaSArchie Cobbs m_freem(top); 956122a814aSBosko Milekic return (NULL); 9571c38f2eaSArchie Cobbs } 9581c38f2eaSArchie Cobbs 9591c38f2eaSArchie Cobbs /* 960df8bae1dSRodney W. Grimes * Concatenate mbuf chain n to m. 961df8bae1dSRodney W. Grimes * Both chains must be of the same type (e.g. MT_DATA). 962df8bae1dSRodney W. Grimes * Any m_pkthdr is not updated. 963df8bae1dSRodney W. Grimes */ 96426f9a767SRodney W. Grimes void 965122a814aSBosko Milekic m_cat(struct mbuf *m, struct mbuf *n) 966df8bae1dSRodney W. Grimes { 967df8bae1dSRodney W. Grimes while (m->m_next) 968df8bae1dSRodney W. Grimes m = m->m_next; 969df8bae1dSRodney W. Grimes while (n) { 97014d7c5b1SAndre Oppermann if (!M_WRITABLE(m) || 97114d7c5b1SAndre Oppermann M_TRAILINGSPACE(m) < n->m_len) { 972df8bae1dSRodney W. Grimes /* just join the two chains */ 973df8bae1dSRodney W. Grimes m->m_next = n; 974df8bae1dSRodney W. Grimes return; 975df8bae1dSRodney W. Grimes } 976df8bae1dSRodney W. Grimes /* splat the data from one into the other */ 977df8bae1dSRodney W. Grimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 978df8bae1dSRodney W. Grimes (u_int)n->m_len); 979df8bae1dSRodney W. Grimes m->m_len += n->m_len; 980df8bae1dSRodney W. Grimes n = m_free(n); 981df8bae1dSRodney W. Grimes } 982df8bae1dSRodney W. Grimes } 983df8bae1dSRodney W. Grimes 9841967edbaSGleb Smirnoff /* 9851967edbaSGleb Smirnoff * Concatenate two pkthdr mbuf chains. 9861967edbaSGleb Smirnoff */ 9871967edbaSGleb Smirnoff void 9881967edbaSGleb Smirnoff m_catpkt(struct mbuf *m, struct mbuf *n) 9891967edbaSGleb Smirnoff { 9901967edbaSGleb Smirnoff 9911967edbaSGleb Smirnoff M_ASSERTPKTHDR(m); 9921967edbaSGleb Smirnoff M_ASSERTPKTHDR(n); 9931967edbaSGleb Smirnoff 9941967edbaSGleb Smirnoff m->m_pkthdr.len += n->m_pkthdr.len; 995651e4e6aSGleb Smirnoff m_demote(n, 1, 0); 9961967edbaSGleb Smirnoff 9971967edbaSGleb Smirnoff m_cat(m, n); 9981967edbaSGleb Smirnoff } 9991967edbaSGleb Smirnoff 100026f9a767SRodney W. Grimes void 1001122a814aSBosko Milekic m_adj(struct mbuf *mp, int req_len) 1002df8bae1dSRodney W. Grimes { 1003122a814aSBosko Milekic int len = req_len; 1004122a814aSBosko Milekic struct mbuf *m; 1005122a814aSBosko Milekic int count; 1006df8bae1dSRodney W. Grimes 1007df8bae1dSRodney W. Grimes if ((m = mp) == NULL) 1008df8bae1dSRodney W. Grimes return; 1009df8bae1dSRodney W. Grimes if (len >= 0) { 1010df8bae1dSRodney W. Grimes /* 1011df8bae1dSRodney W. Grimes * Trim from head. 1012df8bae1dSRodney W. Grimes */ 1013df8bae1dSRodney W. Grimes while (m != NULL && len > 0) { 1014df8bae1dSRodney W. Grimes if (m->m_len <= len) { 1015df8bae1dSRodney W. Grimes len -= m->m_len; 1016df8bae1dSRodney W. Grimes m->m_len = 0; 1017df8bae1dSRodney W. Grimes m = m->m_next; 1018df8bae1dSRodney W. Grimes } else { 1019df8bae1dSRodney W. Grimes m->m_len -= len; 1020df8bae1dSRodney W. Grimes m->m_data += len; 1021df8bae1dSRodney W. Grimes len = 0; 1022df8bae1dSRodney W. Grimes } 1023df8bae1dSRodney W. Grimes } 1024df8bae1dSRodney W. Grimes if (mp->m_flags & M_PKTHDR) 1025a83baab6SMarko Zec mp->m_pkthdr.len -= (req_len - len); 1026df8bae1dSRodney W. Grimes } else { 1027df8bae1dSRodney W. Grimes /* 1028df8bae1dSRodney W. Grimes * Trim from tail. Scan the mbuf chain, 1029df8bae1dSRodney W. Grimes * calculating its length and finding the last mbuf. 1030df8bae1dSRodney W. Grimes * If the adjustment only affects this mbuf, then just 1031df8bae1dSRodney W. Grimes * adjust and return. Otherwise, rescan and truncate 1032df8bae1dSRodney W. Grimes * after the remaining size. 1033df8bae1dSRodney W. Grimes */ 1034df8bae1dSRodney W. Grimes len = -len; 1035df8bae1dSRodney W. Grimes count = 0; 1036df8bae1dSRodney W. Grimes for (;;) { 1037df8bae1dSRodney W. Grimes count += m->m_len; 1038df8bae1dSRodney W. Grimes if (m->m_next == (struct mbuf *)0) 1039df8bae1dSRodney W. Grimes break; 1040df8bae1dSRodney W. Grimes m = m->m_next; 1041df8bae1dSRodney W. Grimes } 1042df8bae1dSRodney W. Grimes if (m->m_len >= len) { 1043df8bae1dSRodney W. Grimes m->m_len -= len; 1044df8bae1dSRodney W. Grimes if (mp->m_flags & M_PKTHDR) 1045df8bae1dSRodney W. Grimes mp->m_pkthdr.len -= len; 1046df8bae1dSRodney W. Grimes return; 1047df8bae1dSRodney W. Grimes } 1048df8bae1dSRodney W. Grimes count -= len; 1049df8bae1dSRodney W. Grimes if (count < 0) 1050df8bae1dSRodney W. Grimes count = 0; 1051df8bae1dSRodney W. Grimes /* 1052df8bae1dSRodney W. Grimes * Correct length for chain is "count". 1053df8bae1dSRodney W. Grimes * Find the mbuf with last data, adjust its length, 1054df8bae1dSRodney W. Grimes * and toss data from remaining mbufs on chain. 1055df8bae1dSRodney W. Grimes */ 1056df8bae1dSRodney W. Grimes m = mp; 1057df8bae1dSRodney W. Grimes if (m->m_flags & M_PKTHDR) 1058df8bae1dSRodney W. Grimes m->m_pkthdr.len = count; 1059df8bae1dSRodney W. Grimes for (; m; m = m->m_next) { 1060df8bae1dSRodney W. Grimes if (m->m_len >= count) { 1061df8bae1dSRodney W. Grimes m->m_len = count; 106259d8b310SSam Leffler if (m->m_next != NULL) { 106359d8b310SSam Leffler m_freem(m->m_next); 106459d8b310SSam Leffler m->m_next = NULL; 106559d8b310SSam Leffler } 1066df8bae1dSRodney W. Grimes break; 1067df8bae1dSRodney W. Grimes } 1068df8bae1dSRodney W. Grimes count -= m->m_len; 1069df8bae1dSRodney W. Grimes } 1070df8bae1dSRodney W. Grimes } 1071df8bae1dSRodney W. Grimes } 1072df8bae1dSRodney W. Grimes 1073df8bae1dSRodney W. Grimes /* 1074df8bae1dSRodney W. Grimes * Rearange an mbuf chain so that len bytes are contiguous 1075a2c36a02SKevin Lo * and in the data area of an mbuf (so that mtod will work 1076a2c36a02SKevin Lo * for a structure of size len). Returns the resulting 1077df8bae1dSRodney W. Grimes * mbuf chain on success, frees it and returns null on failure. 1078df8bae1dSRodney W. Grimes * If there is room, it will add up to max_protohdr-len extra bytes to the 1079df8bae1dSRodney W. Grimes * contiguous region in an attempt to avoid being called next time. 1080df8bae1dSRodney W. Grimes */ 1081df8bae1dSRodney W. Grimes struct mbuf * 1082122a814aSBosko Milekic m_pullup(struct mbuf *n, int len) 1083df8bae1dSRodney W. Grimes { 1084122a814aSBosko Milekic struct mbuf *m; 1085122a814aSBosko Milekic int count; 1086df8bae1dSRodney W. Grimes int space; 1087df8bae1dSRodney W. Grimes 1088df8bae1dSRodney W. Grimes /* 1089df8bae1dSRodney W. Grimes * If first mbuf has no cluster, and has room for len bytes 1090df8bae1dSRodney W. Grimes * without shifting current data, pullup into it, 1091df8bae1dSRodney W. Grimes * otherwise allocate a new mbuf to prepend to the chain. 1092df8bae1dSRodney W. Grimes */ 1093df8bae1dSRodney W. Grimes if ((n->m_flags & M_EXT) == 0 && 1094df8bae1dSRodney W. Grimes n->m_data + len < &n->m_dat[MLEN] && n->m_next) { 1095df8bae1dSRodney W. Grimes if (n->m_len >= len) 1096df8bae1dSRodney W. Grimes return (n); 1097df8bae1dSRodney W. Grimes m = n; 1098df8bae1dSRodney W. Grimes n = n->m_next; 1099df8bae1dSRodney W. Grimes len -= m->m_len; 1100df8bae1dSRodney W. Grimes } else { 1101df8bae1dSRodney W. Grimes if (len > MHLEN) 1102df8bae1dSRodney W. Grimes goto bad; 1103c95be8b5SGleb Smirnoff m = m_get(M_NOWAIT, n->m_type); 1104122a814aSBosko Milekic if (m == NULL) 1105df8bae1dSRodney W. Grimes goto bad; 11069967cafcSSam Leffler if (n->m_flags & M_PKTHDR) 1107c95be8b5SGleb Smirnoff m_move_pkthdr(m, n); 1108df8bae1dSRodney W. Grimes } 1109df8bae1dSRodney W. Grimes space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 1110df8bae1dSRodney W. Grimes do { 1111df8bae1dSRodney W. Grimes count = min(min(max(len, max_protohdr), space), n->m_len); 1112df8bae1dSRodney W. Grimes bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len, 1113bd395ae8SBosko Milekic (u_int)count); 1114df8bae1dSRodney W. Grimes len -= count; 1115df8bae1dSRodney W. Grimes m->m_len += count; 1116df8bae1dSRodney W. Grimes n->m_len -= count; 1117df8bae1dSRodney W. Grimes space -= count; 1118df8bae1dSRodney W. Grimes if (n->m_len) 1119df8bae1dSRodney W. Grimes n->m_data += count; 1120df8bae1dSRodney W. Grimes else 1121df8bae1dSRodney W. Grimes n = m_free(n); 1122df8bae1dSRodney W. Grimes } while (len > 0 && n); 1123df8bae1dSRodney W. Grimes if (len > 0) { 1124df8bae1dSRodney W. Grimes (void) m_free(m); 1125df8bae1dSRodney W. Grimes goto bad; 1126df8bae1dSRodney W. Grimes } 1127df8bae1dSRodney W. Grimes m->m_next = n; 1128df8bae1dSRodney W. Grimes return (m); 1129df8bae1dSRodney W. Grimes bad: 1130df8bae1dSRodney W. Grimes m_freem(n); 1131122a814aSBosko Milekic return (NULL); 1132df8bae1dSRodney W. Grimes } 1133df8bae1dSRodney W. Grimes 1134df8bae1dSRodney W. Grimes /* 11357ac139a9SJohn-Mark Gurney * Like m_pullup(), except a new mbuf is always allocated, and we allow 11367ac139a9SJohn-Mark Gurney * the amount of empty space before the data in the new mbuf to be specified 11377ac139a9SJohn-Mark Gurney * (in the event that the caller expects to prepend later). 11387ac139a9SJohn-Mark Gurney */ 11397ac139a9SJohn-Mark Gurney int MSFail; 11407ac139a9SJohn-Mark Gurney 11417ac139a9SJohn-Mark Gurney struct mbuf * 11427ac139a9SJohn-Mark Gurney m_copyup(struct mbuf *n, int len, int dstoff) 11437ac139a9SJohn-Mark Gurney { 11447ac139a9SJohn-Mark Gurney struct mbuf *m; 11457ac139a9SJohn-Mark Gurney int count, space; 11467ac139a9SJohn-Mark Gurney 11477ac139a9SJohn-Mark Gurney if (len > (MHLEN - dstoff)) 11487ac139a9SJohn-Mark Gurney goto bad; 1149c95be8b5SGleb Smirnoff m = m_get(M_NOWAIT, n->m_type); 11507ac139a9SJohn-Mark Gurney if (m == NULL) 11517ac139a9SJohn-Mark Gurney goto bad; 11527ac139a9SJohn-Mark Gurney if (n->m_flags & M_PKTHDR) 1153c95be8b5SGleb Smirnoff m_move_pkthdr(m, n); 11547ac139a9SJohn-Mark Gurney m->m_data += dstoff; 11557ac139a9SJohn-Mark Gurney space = &m->m_dat[MLEN] - (m->m_data + m->m_len); 11567ac139a9SJohn-Mark Gurney do { 11577ac139a9SJohn-Mark Gurney count = min(min(max(len, max_protohdr), space), n->m_len); 11587ac139a9SJohn-Mark Gurney memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t), 11597ac139a9SJohn-Mark Gurney (unsigned)count); 11607ac139a9SJohn-Mark Gurney len -= count; 11617ac139a9SJohn-Mark Gurney m->m_len += count; 11627ac139a9SJohn-Mark Gurney n->m_len -= count; 11637ac139a9SJohn-Mark Gurney space -= count; 11647ac139a9SJohn-Mark Gurney if (n->m_len) 11657ac139a9SJohn-Mark Gurney n->m_data += count; 11667ac139a9SJohn-Mark Gurney else 11677ac139a9SJohn-Mark Gurney n = m_free(n); 11687ac139a9SJohn-Mark Gurney } while (len > 0 && n); 11697ac139a9SJohn-Mark Gurney if (len > 0) { 11707ac139a9SJohn-Mark Gurney (void) m_free(m); 11717ac139a9SJohn-Mark Gurney goto bad; 11727ac139a9SJohn-Mark Gurney } 11737ac139a9SJohn-Mark Gurney m->m_next = n; 11747ac139a9SJohn-Mark Gurney return (m); 11757ac139a9SJohn-Mark Gurney bad: 11767ac139a9SJohn-Mark Gurney m_freem(n); 11777ac139a9SJohn-Mark Gurney MSFail++; 11787ac139a9SJohn-Mark Gurney return (NULL); 11797ac139a9SJohn-Mark Gurney } 11807ac139a9SJohn-Mark Gurney 11817ac139a9SJohn-Mark Gurney /* 1182df8bae1dSRodney W. Grimes * Partition an mbuf chain in two pieces, returning the tail -- 1183df8bae1dSRodney W. Grimes * all but the first len0 bytes. In case of failure, it returns NULL and 1184df8bae1dSRodney W. Grimes * attempts to restore the chain to its original state. 118548d183faSArchie Cobbs * 118648d183faSArchie Cobbs * Note that the resulting mbufs might be read-only, because the new 118748d183faSArchie Cobbs * mbuf can end up sharing an mbuf cluster with the original mbuf if 118848d183faSArchie Cobbs * the "breaking point" happens to lie within a cluster mbuf. Use the 118948d183faSArchie Cobbs * M_WRITABLE() macro to check for this case. 1190df8bae1dSRodney W. Grimes */ 1191df8bae1dSRodney W. Grimes struct mbuf * 1192122a814aSBosko Milekic m_split(struct mbuf *m0, int len0, int wait) 1193df8bae1dSRodney W. Grimes { 1194122a814aSBosko Milekic struct mbuf *m, *n; 1195bd395ae8SBosko Milekic u_int len = len0, remain; 1196df8bae1dSRodney W. Grimes 1197063d8114SAlfred Perlstein MBUF_CHECKSLEEP(wait); 1198df8bae1dSRodney W. Grimes for (m = m0; m && len > m->m_len; m = m->m_next) 1199df8bae1dSRodney W. Grimes len -= m->m_len; 1200122a814aSBosko Milekic if (m == NULL) 1201122a814aSBosko Milekic return (NULL); 1202df8bae1dSRodney W. Grimes remain = m->m_len - len; 120321f39848SGleb Smirnoff if (m0->m_flags & M_PKTHDR && remain == 0) { 120421f39848SGleb Smirnoff n = m_gethdr(wait, m0->m_type); 120577badb18SGleb Smirnoff if (n == NULL) 120621f39848SGleb Smirnoff return (NULL); 120721f39848SGleb Smirnoff n->m_next = m->m_next; 120821f39848SGleb Smirnoff m->m_next = NULL; 120921f39848SGleb Smirnoff n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 121021f39848SGleb Smirnoff n->m_pkthdr.len = m0->m_pkthdr.len - len0; 121121f39848SGleb Smirnoff m0->m_pkthdr.len = len0; 121221f39848SGleb Smirnoff return (n); 121321f39848SGleb Smirnoff } else if (m0->m_flags & M_PKTHDR) { 1214c95be8b5SGleb Smirnoff n = m_gethdr(wait, m0->m_type); 1215122a814aSBosko Milekic if (n == NULL) 1216122a814aSBosko Milekic return (NULL); 1217df8bae1dSRodney W. Grimes n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif; 1218df8bae1dSRodney W. Grimes n->m_pkthdr.len = m0->m_pkthdr.len - len0; 1219df8bae1dSRodney W. Grimes m0->m_pkthdr.len = len0; 1220df8bae1dSRodney W. Grimes if (m->m_flags & M_EXT) 1221df8bae1dSRodney W. Grimes goto extpacket; 1222df8bae1dSRodney W. Grimes if (remain > MHLEN) { 1223df8bae1dSRodney W. Grimes /* m can't be the lead packet */ 1224*ed6a66caSRobert Watson M_ALIGN(n, 0); 1225df8bae1dSRodney W. Grimes n->m_next = m_split(m, len, wait); 1226122a814aSBosko Milekic if (n->m_next == NULL) { 1227df8bae1dSRodney W. Grimes (void) m_free(n); 1228122a814aSBosko Milekic return (NULL); 122940376987SJeffrey Hsu } else { 123040376987SJeffrey Hsu n->m_len = 0; 1231df8bae1dSRodney W. Grimes return (n); 123240376987SJeffrey Hsu } 1233df8bae1dSRodney W. Grimes } else 1234*ed6a66caSRobert Watson M_ALIGN(n, remain); 1235df8bae1dSRodney W. Grimes } else if (remain == 0) { 1236df8bae1dSRodney W. Grimes n = m->m_next; 1237122a814aSBosko Milekic m->m_next = NULL; 1238df8bae1dSRodney W. Grimes return (n); 1239df8bae1dSRodney W. Grimes } else { 1240c95be8b5SGleb Smirnoff n = m_get(wait, m->m_type); 1241122a814aSBosko Milekic if (n == NULL) 1242122a814aSBosko Milekic return (NULL); 1243df8bae1dSRodney W. Grimes M_ALIGN(n, remain); 1244df8bae1dSRodney W. Grimes } 1245df8bae1dSRodney W. Grimes extpacket: 1246df8bae1dSRodney W. Grimes if (m->m_flags & M_EXT) { 1247df8bae1dSRodney W. Grimes n->m_data = m->m_data + len; 124856a4e45aSAndre Oppermann mb_dupcl(n, m); 1249df8bae1dSRodney W. Grimes } else { 1250df8bae1dSRodney W. Grimes bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain); 1251df8bae1dSRodney W. Grimes } 1252df8bae1dSRodney W. Grimes n->m_len = remain; 1253df8bae1dSRodney W. Grimes m->m_len = len; 1254df8bae1dSRodney W. Grimes n->m_next = m->m_next; 1255122a814aSBosko Milekic m->m_next = NULL; 1256df8bae1dSRodney W. Grimes return (n); 1257df8bae1dSRodney W. Grimes } 1258df8bae1dSRodney W. Grimes /* 1259df8bae1dSRodney W. Grimes * Routine to copy from device local memory into mbufs. 1260f5eece3fSBosko Milekic * Note that `off' argument is offset into first mbuf of target chain from 1261f5eece3fSBosko Milekic * which to begin copying the data to. 1262df8bae1dSRodney W. Grimes */ 1263df8bae1dSRodney W. Grimes struct mbuf * 1264f5eece3fSBosko Milekic m_devget(char *buf, int totlen, int off, struct ifnet *ifp, 1265122a814aSBosko Milekic void (*copy)(char *from, caddr_t to, u_int len)) 1266df8bae1dSRodney W. Grimes { 1267122a814aSBosko Milekic struct mbuf *m; 1268099a0e58SBosko Milekic struct mbuf *top = NULL, **mp = ⊤ 1269f5eece3fSBosko Milekic int len; 1270df8bae1dSRodney W. Grimes 1271f5eece3fSBosko Milekic if (off < 0 || off > MHLEN) 1272f5eece3fSBosko Milekic return (NULL); 1273f5eece3fSBosko Milekic 1274df8bae1dSRodney W. Grimes while (totlen > 0) { 1275099a0e58SBosko Milekic if (top == NULL) { /* First one, must be PKTHDR */ 1276f5eece3fSBosko Milekic if (totlen + off >= MINCLSIZE) { 1277eb1b1807SGleb Smirnoff m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR); 1278f5eece3fSBosko Milekic len = MCLBYTES; 1279df8bae1dSRodney W. Grimes } else { 1280eb1b1807SGleb Smirnoff m = m_gethdr(M_NOWAIT, MT_DATA); 1281099a0e58SBosko Milekic len = MHLEN; 1282099a0e58SBosko Milekic 1283099a0e58SBosko Milekic /* Place initial small packet/header at end of mbuf */ 1284099a0e58SBosko Milekic if (m && totlen + off + max_linkhdr <= MLEN) { 1285df8bae1dSRodney W. Grimes m->m_data += max_linkhdr; 1286f5eece3fSBosko Milekic len -= max_linkhdr; 1287df8bae1dSRodney W. Grimes } 1288f5eece3fSBosko Milekic } 1289099a0e58SBosko Milekic if (m == NULL) 1290099a0e58SBosko Milekic return NULL; 1291099a0e58SBosko Milekic m->m_pkthdr.rcvif = ifp; 1292099a0e58SBosko Milekic m->m_pkthdr.len = totlen; 1293099a0e58SBosko Milekic } else { 1294099a0e58SBosko Milekic if (totlen + off >= MINCLSIZE) { 1295eb1b1807SGleb Smirnoff m = m_getcl(M_NOWAIT, MT_DATA, 0); 1296099a0e58SBosko Milekic len = MCLBYTES; 1297099a0e58SBosko Milekic } else { 1298eb1b1807SGleb Smirnoff m = m_get(M_NOWAIT, MT_DATA); 1299099a0e58SBosko Milekic len = MLEN; 1300099a0e58SBosko Milekic } 1301099a0e58SBosko Milekic if (m == NULL) { 1302099a0e58SBosko Milekic m_freem(top); 1303099a0e58SBosko Milekic return NULL; 1304099a0e58SBosko Milekic } 1305099a0e58SBosko Milekic } 1306f5eece3fSBosko Milekic if (off) { 1307f5eece3fSBosko Milekic m->m_data += off; 1308f5eece3fSBosko Milekic len -= off; 1309f5eece3fSBosko Milekic off = 0; 1310f5eece3fSBosko Milekic } 1311f5eece3fSBosko Milekic m->m_len = len = min(totlen, len); 1312df8bae1dSRodney W. Grimes if (copy) 1313bd395ae8SBosko Milekic copy(buf, mtod(m, caddr_t), (u_int)len); 1314df8bae1dSRodney W. Grimes else 1315bd395ae8SBosko Milekic bcopy(buf, mtod(m, caddr_t), (u_int)len); 1316f5eece3fSBosko Milekic buf += len; 1317df8bae1dSRodney W. Grimes *mp = m; 1318df8bae1dSRodney W. Grimes mp = &m->m_next; 1319df8bae1dSRodney W. Grimes totlen -= len; 1320df8bae1dSRodney W. Grimes } 1321df8bae1dSRodney W. Grimes return (top); 1322df8bae1dSRodney W. Grimes } 1323c5789ba3SPoul-Henning Kamp 1324c5789ba3SPoul-Henning Kamp /* 1325c5789ba3SPoul-Henning Kamp * Copy data from a buffer back into the indicated mbuf chain, 1326c5789ba3SPoul-Henning Kamp * starting "off" bytes from the beginning, extending the mbuf 1327c5789ba3SPoul-Henning Kamp * chain if necessary. 1328c5789ba3SPoul-Henning Kamp */ 1329c5789ba3SPoul-Henning Kamp void 133024665342SLuigi Rizzo m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp) 1331c5789ba3SPoul-Henning Kamp { 1332122a814aSBosko Milekic int mlen; 1333122a814aSBosko Milekic struct mbuf *m = m0, *n; 1334c5789ba3SPoul-Henning Kamp int totlen = 0; 1335c5789ba3SPoul-Henning Kamp 1336122a814aSBosko Milekic if (m0 == NULL) 1337c5789ba3SPoul-Henning Kamp return; 1338c5789ba3SPoul-Henning Kamp while (off > (mlen = m->m_len)) { 1339c5789ba3SPoul-Henning Kamp off -= mlen; 1340c5789ba3SPoul-Henning Kamp totlen += mlen; 1341122a814aSBosko Milekic if (m->m_next == NULL) { 1342eb1b1807SGleb Smirnoff n = m_get(M_NOWAIT, m->m_type); 1343122a814aSBosko Milekic if (n == NULL) 1344c5789ba3SPoul-Henning Kamp goto out; 1345099a0e58SBosko Milekic bzero(mtod(n, caddr_t), MLEN); 1346c5789ba3SPoul-Henning Kamp n->m_len = min(MLEN, len + off); 1347c5789ba3SPoul-Henning Kamp m->m_next = n; 1348c5789ba3SPoul-Henning Kamp } 1349c5789ba3SPoul-Henning Kamp m = m->m_next; 1350c5789ba3SPoul-Henning Kamp } 1351c5789ba3SPoul-Henning Kamp while (len > 0) { 1352129c5c81SAlexander Motin if (m->m_next == NULL && (len > m->m_len - off)) { 1353129c5c81SAlexander Motin m->m_len += min(len - (m->m_len - off), 1354129c5c81SAlexander Motin M_TRAILINGSPACE(m)); 1355129c5c81SAlexander Motin } 1356c5789ba3SPoul-Henning Kamp mlen = min (m->m_len - off, len); 1357bd395ae8SBosko Milekic bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen); 1358c5789ba3SPoul-Henning Kamp cp += mlen; 1359c5789ba3SPoul-Henning Kamp len -= mlen; 1360c5789ba3SPoul-Henning Kamp mlen += off; 1361c5789ba3SPoul-Henning Kamp off = 0; 1362c5789ba3SPoul-Henning Kamp totlen += mlen; 1363c5789ba3SPoul-Henning Kamp if (len == 0) 1364c5789ba3SPoul-Henning Kamp break; 1365122a814aSBosko Milekic if (m->m_next == NULL) { 1366eb1b1807SGleb Smirnoff n = m_get(M_NOWAIT, m->m_type); 1367122a814aSBosko Milekic if (n == NULL) 1368c5789ba3SPoul-Henning Kamp break; 1369c5789ba3SPoul-Henning Kamp n->m_len = min(MLEN, len); 1370c5789ba3SPoul-Henning Kamp m->m_next = n; 1371c5789ba3SPoul-Henning Kamp } 1372c5789ba3SPoul-Henning Kamp m = m->m_next; 1373c5789ba3SPoul-Henning Kamp } 1374c5789ba3SPoul-Henning Kamp out: if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen)) 1375c5789ba3SPoul-Henning Kamp m->m_pkthdr.len = totlen; 1376c5789ba3SPoul-Henning Kamp } 1377ce4a64f7SPoul-Henning Kamp 137837621fd5SBruce M Simpson /* 13794873d175SSam Leffler * Append the specified data to the indicated mbuf chain, 13804873d175SSam Leffler * Extend the mbuf chain if the new data does not fit in 13814873d175SSam Leffler * existing space. 13824873d175SSam Leffler * 13834873d175SSam Leffler * Return 1 if able to complete the job; otherwise 0. 13844873d175SSam Leffler */ 13854873d175SSam Leffler int 13864873d175SSam Leffler m_append(struct mbuf *m0, int len, c_caddr_t cp) 13874873d175SSam Leffler { 13884873d175SSam Leffler struct mbuf *m, *n; 13894873d175SSam Leffler int remainder, space; 13904873d175SSam Leffler 13914873d175SSam Leffler for (m = m0; m->m_next != NULL; m = m->m_next) 13924873d175SSam Leffler ; 13934873d175SSam Leffler remainder = len; 13944873d175SSam Leffler space = M_TRAILINGSPACE(m); 13954873d175SSam Leffler if (space > 0) { 13964873d175SSam Leffler /* 13974873d175SSam Leffler * Copy into available space. 13984873d175SSam Leffler */ 13994873d175SSam Leffler if (space > remainder) 14004873d175SSam Leffler space = remainder; 14014873d175SSam Leffler bcopy(cp, mtod(m, caddr_t) + m->m_len, space); 14024873d175SSam Leffler m->m_len += space; 14034873d175SSam Leffler cp += space, remainder -= space; 14044873d175SSam Leffler } 14054873d175SSam Leffler while (remainder > 0) { 14064873d175SSam Leffler /* 14074873d175SSam Leffler * Allocate a new mbuf; could check space 14084873d175SSam Leffler * and allocate a cluster instead. 14094873d175SSam Leffler */ 1410eb1b1807SGleb Smirnoff n = m_get(M_NOWAIT, m->m_type); 14114873d175SSam Leffler if (n == NULL) 14124873d175SSam Leffler break; 14134873d175SSam Leffler n->m_len = min(MLEN, remainder); 1414a37c415eSSam Leffler bcopy(cp, mtod(n, caddr_t), n->m_len); 1415a37c415eSSam Leffler cp += n->m_len, remainder -= n->m_len; 14164873d175SSam Leffler m->m_next = n; 14174873d175SSam Leffler m = n; 14184873d175SSam Leffler } 14194873d175SSam Leffler if (m0->m_flags & M_PKTHDR) 14204873d175SSam Leffler m0->m_pkthdr.len += len - remainder; 14214873d175SSam Leffler return (remainder == 0); 14224873d175SSam Leffler } 14234873d175SSam Leffler 14244873d175SSam Leffler /* 142537621fd5SBruce M Simpson * Apply function f to the data in an mbuf chain starting "off" bytes from 142637621fd5SBruce M Simpson * the beginning, continuing for "len" bytes. 142737621fd5SBruce M Simpson */ 142837621fd5SBruce M Simpson int 142937621fd5SBruce M Simpson m_apply(struct mbuf *m, int off, int len, 143054065297SBruce M Simpson int (*f)(void *, void *, u_int), void *arg) 143137621fd5SBruce M Simpson { 143254065297SBruce M Simpson u_int count; 143337621fd5SBruce M Simpson int rval; 143437621fd5SBruce M Simpson 143537621fd5SBruce M Simpson KASSERT(off >= 0, ("m_apply, negative off %d", off)); 143637621fd5SBruce M Simpson KASSERT(len >= 0, ("m_apply, negative len %d", len)); 143737621fd5SBruce M Simpson while (off > 0) { 143837621fd5SBruce M Simpson KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 143937621fd5SBruce M Simpson if (off < m->m_len) 144037621fd5SBruce M Simpson break; 144137621fd5SBruce M Simpson off -= m->m_len; 144237621fd5SBruce M Simpson m = m->m_next; 144337621fd5SBruce M Simpson } 144437621fd5SBruce M Simpson while (len > 0) { 144537621fd5SBruce M Simpson KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain")); 144637621fd5SBruce M Simpson count = min(m->m_len - off, len); 144737621fd5SBruce M Simpson rval = (*f)(arg, mtod(m, caddr_t) + off, count); 144837621fd5SBruce M Simpson if (rval) 144937621fd5SBruce M Simpson return (rval); 145037621fd5SBruce M Simpson len -= count; 145137621fd5SBruce M Simpson off = 0; 145237621fd5SBruce M Simpson m = m->m_next; 145337621fd5SBruce M Simpson } 145437621fd5SBruce M Simpson return (0); 145537621fd5SBruce M Simpson } 145637621fd5SBruce M Simpson 145737621fd5SBruce M Simpson /* 145837621fd5SBruce M Simpson * Return a pointer to mbuf/offset of location in mbuf chain. 145937621fd5SBruce M Simpson */ 146037621fd5SBruce M Simpson struct mbuf * 146137621fd5SBruce M Simpson m_getptr(struct mbuf *m, int loc, int *off) 146237621fd5SBruce M Simpson { 146337621fd5SBruce M Simpson 146437621fd5SBruce M Simpson while (loc >= 0) { 146554065297SBruce M Simpson /* Normal end of search. */ 146637621fd5SBruce M Simpson if (m->m_len > loc) { 146737621fd5SBruce M Simpson *off = loc; 146837621fd5SBruce M Simpson return (m); 146937621fd5SBruce M Simpson } else { 147037621fd5SBruce M Simpson loc -= m->m_len; 147137621fd5SBruce M Simpson if (m->m_next == NULL) { 147237621fd5SBruce M Simpson if (loc == 0) { 147354065297SBruce M Simpson /* Point at the end of valid data. */ 147437621fd5SBruce M Simpson *off = m->m_len; 147537621fd5SBruce M Simpson return (m); 147654065297SBruce M Simpson } 147737621fd5SBruce M Simpson return (NULL); 147854065297SBruce M Simpson } 147937621fd5SBruce M Simpson m = m->m_next; 148037621fd5SBruce M Simpson } 148137621fd5SBruce M Simpson } 148237621fd5SBruce M Simpson return (NULL); 148337621fd5SBruce M Simpson } 148437621fd5SBruce M Simpson 1485ce4a64f7SPoul-Henning Kamp void 14867b125090SJohn-Mark Gurney m_print(const struct mbuf *m, int maxlen) 1487ce4a64f7SPoul-Henning Kamp { 1488ce4a64f7SPoul-Henning Kamp int len; 14897b125090SJohn-Mark Gurney int pdata; 14906357e7b5SEivind Eklund const struct mbuf *m2; 1491ce4a64f7SPoul-Henning Kamp 14927e949c46SKenneth D. Merry if (m == NULL) { 14937e949c46SKenneth D. Merry printf("mbuf: %p\n", m); 14947e949c46SKenneth D. Merry return; 14957e949c46SKenneth D. Merry } 14967e949c46SKenneth D. Merry 14977b125090SJohn-Mark Gurney if (m->m_flags & M_PKTHDR) 1498ce4a64f7SPoul-Henning Kamp len = m->m_pkthdr.len; 14997b125090SJohn-Mark Gurney else 15007b125090SJohn-Mark Gurney len = -1; 1501ce4a64f7SPoul-Henning Kamp m2 = m; 15027b125090SJohn-Mark Gurney while (m2 != NULL && (len == -1 || len)) { 15037b125090SJohn-Mark Gurney pdata = m2->m_len; 15047b125090SJohn-Mark Gurney if (maxlen != -1 && pdata > maxlen) 15057b125090SJohn-Mark Gurney pdata = maxlen; 15067b125090SJohn-Mark Gurney printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len, 15077b125090SJohn-Mark Gurney m2->m_next, m2->m_flags, "\20\20freelist\17skipfw" 15087b125090SJohn-Mark Gurney "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly" 15097b125090SJohn-Mark Gurney "\3eor\2pkthdr\1ext", pdata ? "" : "\n"); 15107b125090SJohn-Mark Gurney if (pdata) 151145e0d0aaSJohn-Mark Gurney printf(", %*D\n", pdata, (u_char *)m2->m_data, "-"); 15127b125090SJohn-Mark Gurney if (len != -1) 1513ce4a64f7SPoul-Henning Kamp len -= m2->m_len; 1514ce4a64f7SPoul-Henning Kamp m2 = m2->m_next; 1515ce4a64f7SPoul-Henning Kamp } 15167b125090SJohn-Mark Gurney if (len > 0) 15177b125090SJohn-Mark Gurney printf("%d bytes unaccounted for.\n", len); 1518ce4a64f7SPoul-Henning Kamp return; 1519ce4a64f7SPoul-Henning Kamp } 15203f2e06c5SPoul-Henning Kamp 1521bd395ae8SBosko Milekic u_int 15223f2e06c5SPoul-Henning Kamp m_fixhdr(struct mbuf *m0) 15233f2e06c5SPoul-Henning Kamp { 1524bd395ae8SBosko Milekic u_int len; 15253f2e06c5SPoul-Henning Kamp 1526ac6e585dSPoul-Henning Kamp len = m_length(m0, NULL); 15273f2e06c5SPoul-Henning Kamp m0->m_pkthdr.len = len; 1528ac6e585dSPoul-Henning Kamp return (len); 1529ac6e585dSPoul-Henning Kamp } 1530ac6e585dSPoul-Henning Kamp 1531bd395ae8SBosko Milekic u_int 1532ac6e585dSPoul-Henning Kamp m_length(struct mbuf *m0, struct mbuf **last) 1533ac6e585dSPoul-Henning Kamp { 1534ac6e585dSPoul-Henning Kamp struct mbuf *m; 1535bd395ae8SBosko Milekic u_int len; 1536ac6e585dSPoul-Henning Kamp 1537ac6e585dSPoul-Henning Kamp len = 0; 1538ac6e585dSPoul-Henning Kamp for (m = m0; m != NULL; m = m->m_next) { 1539ac6e585dSPoul-Henning Kamp len += m->m_len; 1540ac6e585dSPoul-Henning Kamp if (m->m_next == NULL) 1541ac6e585dSPoul-Henning Kamp break; 1542ac6e585dSPoul-Henning Kamp } 1543ac6e585dSPoul-Henning Kamp if (last != NULL) 1544ac6e585dSPoul-Henning Kamp *last = m; 1545ac6e585dSPoul-Henning Kamp return (len); 15463f2e06c5SPoul-Henning Kamp } 154755e9f80dSMike Silbersack 154855e9f80dSMike Silbersack /* 154955e9f80dSMike Silbersack * Defragment a mbuf chain, returning the shortest possible 155055e9f80dSMike Silbersack * chain of mbufs and clusters. If allocation fails and 155155e9f80dSMike Silbersack * this cannot be completed, NULL will be returned, but 155255e9f80dSMike Silbersack * the passed in chain will be unchanged. Upon success, 155355e9f80dSMike Silbersack * the original chain will be freed, and the new chain 155455e9f80dSMike Silbersack * will be returned. 155555e9f80dSMike Silbersack * 155655e9f80dSMike Silbersack * If a non-packet header is passed in, the original 155755e9f80dSMike Silbersack * mbuf (chain?) will be returned unharmed. 155855e9f80dSMike Silbersack */ 155955e9f80dSMike Silbersack struct mbuf * 156055e9f80dSMike Silbersack m_defrag(struct mbuf *m0, int how) 156155e9f80dSMike Silbersack { 156255e9f80dSMike Silbersack struct mbuf *m_new = NULL, *m_final = NULL; 156355e9f80dSMike Silbersack int progress = 0, length; 156455e9f80dSMike Silbersack 1565063d8114SAlfred Perlstein MBUF_CHECKSLEEP(how); 156655e9f80dSMike Silbersack if (!(m0->m_flags & M_PKTHDR)) 156755e9f80dSMike Silbersack return (m0); 156855e9f80dSMike Silbersack 1569f8bf8e39SMike Silbersack m_fixhdr(m0); /* Needed sanity check */ 1570f8bf8e39SMike Silbersack 1571352d050eSMike Silbersack #ifdef MBUF_STRESS_TEST 1572352d050eSMike Silbersack if (m_defragrandomfailures) { 1573352d050eSMike Silbersack int temp = arc4random() & 0xff; 1574352d050eSMike Silbersack if (temp == 0xba) 1575352d050eSMike Silbersack goto nospace; 1576352d050eSMike Silbersack } 1577352d050eSMike Silbersack #endif 157855e9f80dSMike Silbersack 157955e9f80dSMike Silbersack if (m0->m_pkthdr.len > MHLEN) 158055e9f80dSMike Silbersack m_final = m_getcl(how, MT_DATA, M_PKTHDR); 158155e9f80dSMike Silbersack else 158255e9f80dSMike Silbersack m_final = m_gethdr(how, MT_DATA); 158355e9f80dSMike Silbersack 158455e9f80dSMike Silbersack if (m_final == NULL) 158555e9f80dSMike Silbersack goto nospace; 158655e9f80dSMike Silbersack 1587a89ec05eSPeter Wemm if (m_dup_pkthdr(m_final, m0, how) == 0) 158855e9f80dSMike Silbersack goto nospace; 158955e9f80dSMike Silbersack 159055e9f80dSMike Silbersack m_new = m_final; 159155e9f80dSMike Silbersack 159255e9f80dSMike Silbersack while (progress < m0->m_pkthdr.len) { 159355e9f80dSMike Silbersack length = m0->m_pkthdr.len - progress; 159455e9f80dSMike Silbersack if (length > MCLBYTES) 159555e9f80dSMike Silbersack length = MCLBYTES; 159655e9f80dSMike Silbersack 159755e9f80dSMike Silbersack if (m_new == NULL) { 159855e9f80dSMike Silbersack if (length > MLEN) 159955e9f80dSMike Silbersack m_new = m_getcl(how, MT_DATA, 0); 160055e9f80dSMike Silbersack else 160155e9f80dSMike Silbersack m_new = m_get(how, MT_DATA); 160255e9f80dSMike Silbersack if (m_new == NULL) 160355e9f80dSMike Silbersack goto nospace; 160455e9f80dSMike Silbersack } 160555e9f80dSMike Silbersack 160655e9f80dSMike Silbersack m_copydata(m0, progress, length, mtod(m_new, caddr_t)); 160755e9f80dSMike Silbersack progress += length; 160855e9f80dSMike Silbersack m_new->m_len = length; 160955e9f80dSMike Silbersack if (m_new != m_final) 161055e9f80dSMike Silbersack m_cat(m_final, m_new); 161155e9f80dSMike Silbersack m_new = NULL; 161255e9f80dSMike Silbersack } 161351710a45SMike Silbersack #ifdef MBUF_STRESS_TEST 161455e9f80dSMike Silbersack if (m0->m_next == NULL) 161555e9f80dSMike Silbersack m_defraguseless++; 161651710a45SMike Silbersack #endif 161755e9f80dSMike Silbersack m_freem(m0); 161855e9f80dSMike Silbersack m0 = m_final; 161951710a45SMike Silbersack #ifdef MBUF_STRESS_TEST 162055e9f80dSMike Silbersack m_defragpackets++; 162155e9f80dSMike Silbersack m_defragbytes += m0->m_pkthdr.len; 162251710a45SMike Silbersack #endif 162355e9f80dSMike Silbersack return (m0); 162455e9f80dSMike Silbersack nospace: 162551710a45SMike Silbersack #ifdef MBUF_STRESS_TEST 162655e9f80dSMike Silbersack m_defragfailure++; 162751710a45SMike Silbersack #endif 162855e9f80dSMike Silbersack if (m_final) 162955e9f80dSMike Silbersack m_freem(m_final); 163055e9f80dSMike Silbersack return (NULL); 163155e9f80dSMike Silbersack } 16323390d476SMike Silbersack 1633eeb76a18SSam Leffler /* 1634eeb76a18SSam Leffler * Defragment an mbuf chain, returning at most maxfrags separate 1635eeb76a18SSam Leffler * mbufs+clusters. If this is not possible NULL is returned and 1636eeb76a18SSam Leffler * the original mbuf chain is left in it's present (potentially 1637eeb76a18SSam Leffler * modified) state. We use two techniques: collapsing consecutive 1638eeb76a18SSam Leffler * mbufs and replacing consecutive mbufs by a cluster. 1639eeb76a18SSam Leffler * 1640eeb76a18SSam Leffler * NB: this should really be named m_defrag but that name is taken 1641eeb76a18SSam Leffler */ 1642eeb76a18SSam Leffler struct mbuf * 1643eeb76a18SSam Leffler m_collapse(struct mbuf *m0, int how, int maxfrags) 1644eeb76a18SSam Leffler { 1645eeb76a18SSam Leffler struct mbuf *m, *n, *n2, **prev; 1646eeb76a18SSam Leffler u_int curfrags; 1647eeb76a18SSam Leffler 1648eeb76a18SSam Leffler /* 1649eeb76a18SSam Leffler * Calculate the current number of frags. 1650eeb76a18SSam Leffler */ 1651eeb76a18SSam Leffler curfrags = 0; 1652eeb76a18SSam Leffler for (m = m0; m != NULL; m = m->m_next) 1653eeb76a18SSam Leffler curfrags++; 1654eeb76a18SSam Leffler /* 1655eeb76a18SSam Leffler * First, try to collapse mbufs. Note that we always collapse 1656eeb76a18SSam Leffler * towards the front so we don't need to deal with moving the 1657eeb76a18SSam Leffler * pkthdr. This may be suboptimal if the first mbuf has much 1658eeb76a18SSam Leffler * less data than the following. 1659eeb76a18SSam Leffler */ 1660eeb76a18SSam Leffler m = m0; 1661eeb76a18SSam Leffler again: 1662eeb76a18SSam Leffler for (;;) { 1663eeb76a18SSam Leffler n = m->m_next; 1664eeb76a18SSam Leffler if (n == NULL) 1665eeb76a18SSam Leffler break; 166614d7c5b1SAndre Oppermann if (M_WRITABLE(m) && 1667eeb76a18SSam Leffler n->m_len < M_TRAILINGSPACE(m)) { 1668eeb76a18SSam Leffler bcopy(mtod(n, void *), mtod(m, char *) + m->m_len, 1669eeb76a18SSam Leffler n->m_len); 1670eeb76a18SSam Leffler m->m_len += n->m_len; 1671eeb76a18SSam Leffler m->m_next = n->m_next; 1672eeb76a18SSam Leffler m_free(n); 1673eeb76a18SSam Leffler if (--curfrags <= maxfrags) 1674eeb76a18SSam Leffler return m0; 1675eeb76a18SSam Leffler } else 1676eeb76a18SSam Leffler m = n; 1677eeb76a18SSam Leffler } 1678eeb76a18SSam Leffler KASSERT(maxfrags > 1, 1679eeb76a18SSam Leffler ("maxfrags %u, but normal collapse failed", maxfrags)); 1680eeb76a18SSam Leffler /* 1681eeb76a18SSam Leffler * Collapse consecutive mbufs to a cluster. 1682eeb76a18SSam Leffler */ 1683eeb76a18SSam Leffler prev = &m0->m_next; /* NB: not the first mbuf */ 1684eeb76a18SSam Leffler while ((n = *prev) != NULL) { 1685eeb76a18SSam Leffler if ((n2 = n->m_next) != NULL && 1686eeb76a18SSam Leffler n->m_len + n2->m_len < MCLBYTES) { 1687eeb76a18SSam Leffler m = m_getcl(how, MT_DATA, 0); 1688eeb76a18SSam Leffler if (m == NULL) 1689eeb76a18SSam Leffler goto bad; 1690eeb76a18SSam Leffler bcopy(mtod(n, void *), mtod(m, void *), n->m_len); 1691eeb76a18SSam Leffler bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len, 1692eeb76a18SSam Leffler n2->m_len); 1693eeb76a18SSam Leffler m->m_len = n->m_len + n2->m_len; 1694eeb76a18SSam Leffler m->m_next = n2->m_next; 1695eeb76a18SSam Leffler *prev = m; 1696eeb76a18SSam Leffler m_free(n); 1697eeb76a18SSam Leffler m_free(n2); 1698eeb76a18SSam Leffler if (--curfrags <= maxfrags) /* +1 cl -2 mbufs */ 1699eeb76a18SSam Leffler return m0; 1700eeb76a18SSam Leffler /* 1701eeb76a18SSam Leffler * Still not there, try the normal collapse 1702eeb76a18SSam Leffler * again before we allocate another cluster. 1703eeb76a18SSam Leffler */ 1704eeb76a18SSam Leffler goto again; 1705eeb76a18SSam Leffler } 1706eeb76a18SSam Leffler prev = &n->m_next; 1707eeb76a18SSam Leffler } 1708eeb76a18SSam Leffler /* 1709eeb76a18SSam Leffler * No place where we can collapse to a cluster; punt. 1710eeb76a18SSam Leffler * This can occur if, for example, you request 2 frags 1711eeb76a18SSam Leffler * but the packet requires that both be clusters (we 1712eeb76a18SSam Leffler * never reallocate the first mbuf to avoid moving the 1713eeb76a18SSam Leffler * packet header). 1714eeb76a18SSam Leffler */ 1715eeb76a18SSam Leffler bad: 1716eeb76a18SSam Leffler return NULL; 1717eeb76a18SSam Leffler } 1718eeb76a18SSam Leffler 17193390d476SMike Silbersack #ifdef MBUF_STRESS_TEST 17203390d476SMike Silbersack 17213390d476SMike Silbersack /* 17223390d476SMike Silbersack * Fragment an mbuf chain. There's no reason you'd ever want to do 17233390d476SMike Silbersack * this in normal usage, but it's great for stress testing various 17243390d476SMike Silbersack * mbuf consumers. 17253390d476SMike Silbersack * 17263390d476SMike Silbersack * If fragmentation is not possible, the original chain will be 17273390d476SMike Silbersack * returned. 17283390d476SMike Silbersack * 17293390d476SMike Silbersack * Possible length values: 17303390d476SMike Silbersack * 0 no fragmentation will occur 17313390d476SMike Silbersack * > 0 each fragment will be of the specified length 17323390d476SMike Silbersack * -1 each fragment will be the same random value in length 17333390d476SMike Silbersack * -2 each fragment's length will be entirely random 17343390d476SMike Silbersack * (Random values range from 1 to 256) 17353390d476SMike Silbersack */ 17363390d476SMike Silbersack struct mbuf * 17373390d476SMike Silbersack m_fragment(struct mbuf *m0, int how, int length) 17383390d476SMike Silbersack { 17393390d476SMike Silbersack struct mbuf *m_new = NULL, *m_final = NULL; 17403390d476SMike Silbersack int progress = 0; 17413390d476SMike Silbersack 17423390d476SMike Silbersack if (!(m0->m_flags & M_PKTHDR)) 17433390d476SMike Silbersack return (m0); 17443390d476SMike Silbersack 17453390d476SMike Silbersack if ((length == 0) || (length < -2)) 17463390d476SMike Silbersack return (m0); 17473390d476SMike Silbersack 17483390d476SMike Silbersack m_fixhdr(m0); /* Needed sanity check */ 17493390d476SMike Silbersack 17503390d476SMike Silbersack m_final = m_getcl(how, MT_DATA, M_PKTHDR); 17513390d476SMike Silbersack 17523390d476SMike Silbersack if (m_final == NULL) 17533390d476SMike Silbersack goto nospace; 17543390d476SMike Silbersack 17558dee2f67SMike Silbersack if (m_dup_pkthdr(m_final, m0, how) == 0) 17563390d476SMike Silbersack goto nospace; 17573390d476SMike Silbersack 17583390d476SMike Silbersack m_new = m_final; 17593390d476SMike Silbersack 17603390d476SMike Silbersack if (length == -1) 17613390d476SMike Silbersack length = 1 + (arc4random() & 255); 17623390d476SMike Silbersack 17633390d476SMike Silbersack while (progress < m0->m_pkthdr.len) { 17643390d476SMike Silbersack int fraglen; 17653390d476SMike Silbersack 17663390d476SMike Silbersack if (length > 0) 17673390d476SMike Silbersack fraglen = length; 17683390d476SMike Silbersack else 17693390d476SMike Silbersack fraglen = 1 + (arc4random() & 255); 17703390d476SMike Silbersack if (fraglen > m0->m_pkthdr.len - progress) 17713390d476SMike Silbersack fraglen = m0->m_pkthdr.len - progress; 17723390d476SMike Silbersack 17733390d476SMike Silbersack if (fraglen > MCLBYTES) 17743390d476SMike Silbersack fraglen = MCLBYTES; 17753390d476SMike Silbersack 17763390d476SMike Silbersack if (m_new == NULL) { 17773390d476SMike Silbersack m_new = m_getcl(how, MT_DATA, 0); 17783390d476SMike Silbersack if (m_new == NULL) 17793390d476SMike Silbersack goto nospace; 17803390d476SMike Silbersack } 17813390d476SMike Silbersack 17823390d476SMike Silbersack m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t)); 17833390d476SMike Silbersack progress += fraglen; 17843390d476SMike Silbersack m_new->m_len = fraglen; 17853390d476SMike Silbersack if (m_new != m_final) 17863390d476SMike Silbersack m_cat(m_final, m_new); 17873390d476SMike Silbersack m_new = NULL; 17883390d476SMike Silbersack } 17893390d476SMike Silbersack m_freem(m0); 17903390d476SMike Silbersack m0 = m_final; 17913390d476SMike Silbersack return (m0); 17923390d476SMike Silbersack nospace: 17933390d476SMike Silbersack if (m_final) 17943390d476SMike Silbersack m_freem(m_final); 17953390d476SMike Silbersack /* Return the original chain on failure */ 17963390d476SMike Silbersack return (m0); 17973390d476SMike Silbersack } 17983390d476SMike Silbersack 17993390d476SMike Silbersack #endif 1800beb699c7SMike Silbersack 18015e20f43dSAndre Oppermann /* 18025e20f43dSAndre Oppermann * Copy the contents of uio into a properly sized mbuf chain. 18035e20f43dSAndre Oppermann */ 1804beb699c7SMike Silbersack struct mbuf * 18055e20f43dSAndre Oppermann m_uiotombuf(struct uio *uio, int how, int len, int align, int flags) 1806beb699c7SMike Silbersack { 18075e20f43dSAndre Oppermann struct mbuf *m, *mb; 1808526d0bd5SKonstantin Belousov int error, length; 1809526d0bd5SKonstantin Belousov ssize_t total; 18105e20f43dSAndre Oppermann int progress = 0; 1811beb699c7SMike Silbersack 18125e20f43dSAndre Oppermann /* 18135e20f43dSAndre Oppermann * len can be zero or an arbitrary large value bound by 18145e20f43dSAndre Oppermann * the total data supplied by the uio. 18155e20f43dSAndre Oppermann */ 1816beb699c7SMike Silbersack if (len > 0) 1817beb699c7SMike Silbersack total = min(uio->uio_resid, len); 1818beb699c7SMike Silbersack else 1819beb699c7SMike Silbersack total = uio->uio_resid; 18205e20f43dSAndre Oppermann 18215e20f43dSAndre Oppermann /* 18225e20f43dSAndre Oppermann * The smallest unit returned by m_getm2() is a single mbuf 18239128ec21SAndrew Thompson * with pkthdr. We can't align past it. 18245e20f43dSAndre Oppermann */ 182575ae2570SMaksim Yevmenkin if (align >= MHLEN) 1826beb699c7SMike Silbersack return (NULL); 18275e20f43dSAndre Oppermann 18287c32173bSAndre Oppermann /* 18297c32173bSAndre Oppermann * Give us the full allocation or nothing. 18307c32173bSAndre Oppermann * If len is zero return the smallest empty mbuf. 18317c32173bSAndre Oppermann */ 18327c32173bSAndre Oppermann m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags); 18335e20f43dSAndre Oppermann if (m == NULL) 18345e20f43dSAndre Oppermann return (NULL); 18355e20f43dSAndre Oppermann m->m_data += align; 18365e20f43dSAndre Oppermann 18375e20f43dSAndre Oppermann /* Fill all mbufs with uio data and update header information. */ 18385e20f43dSAndre Oppermann for (mb = m; mb != NULL; mb = mb->m_next) { 18395e20f43dSAndre Oppermann length = min(M_TRAILINGSPACE(mb), total - progress); 18405e20f43dSAndre Oppermann 18415e20f43dSAndre Oppermann error = uiomove(mtod(mb, void *), length, uio); 18425e20f43dSAndre Oppermann if (error) { 18435e20f43dSAndre Oppermann m_freem(m); 18445e20f43dSAndre Oppermann return (NULL); 18455e20f43dSAndre Oppermann } 18465e20f43dSAndre Oppermann 18475e20f43dSAndre Oppermann mb->m_len = length; 18485e20f43dSAndre Oppermann progress += length; 18495e20f43dSAndre Oppermann if (flags & M_PKTHDR) 18505e20f43dSAndre Oppermann m->m_pkthdr.len += length; 18515e20f43dSAndre Oppermann } 18525e20f43dSAndre Oppermann KASSERT(progress == total, ("%s: progress != total", __func__)); 18535e20f43dSAndre Oppermann 18545e20f43dSAndre Oppermann return (m); 1855beb699c7SMike Silbersack } 1856ab8ab90cSSam Leffler 1857ab8ab90cSSam Leffler /* 1858bc05b2f6SAndre Oppermann * Copy an mbuf chain into a uio limited by len if set. 1859bc05b2f6SAndre Oppermann */ 1860bc05b2f6SAndre Oppermann int 1861bc05b2f6SAndre Oppermann m_mbuftouio(struct uio *uio, struct mbuf *m, int len) 1862bc05b2f6SAndre Oppermann { 1863bc05b2f6SAndre Oppermann int error, length, total; 1864bc05b2f6SAndre Oppermann int progress = 0; 1865bc05b2f6SAndre Oppermann 1866bc05b2f6SAndre Oppermann if (len > 0) 1867bc05b2f6SAndre Oppermann total = min(uio->uio_resid, len); 1868bc05b2f6SAndre Oppermann else 1869bc05b2f6SAndre Oppermann total = uio->uio_resid; 1870bc05b2f6SAndre Oppermann 1871bc05b2f6SAndre Oppermann /* Fill the uio with data from the mbufs. */ 1872bc05b2f6SAndre Oppermann for (; m != NULL; m = m->m_next) { 1873bc05b2f6SAndre Oppermann length = min(m->m_len, total - progress); 1874bc05b2f6SAndre Oppermann 1875bc05b2f6SAndre Oppermann error = uiomove(mtod(m, void *), length, uio); 1876bc05b2f6SAndre Oppermann if (error) 1877bc05b2f6SAndre Oppermann return (error); 1878bc05b2f6SAndre Oppermann 1879bc05b2f6SAndre Oppermann progress += length; 1880bc05b2f6SAndre Oppermann } 1881bc05b2f6SAndre Oppermann 1882bc05b2f6SAndre Oppermann return (0); 1883bc05b2f6SAndre Oppermann } 1884bc05b2f6SAndre Oppermann 1885bc05b2f6SAndre Oppermann /* 188647e2996eSSam Leffler * Create a writable copy of the mbuf chain. While doing this 188747e2996eSSam Leffler * we compact the chain with a goal of producing a chain with 188847e2996eSSam Leffler * at most two mbufs. The second mbuf in this chain is likely 188947e2996eSSam Leffler * to be a cluster. The primary purpose of this work is to create 189047e2996eSSam Leffler * a writable packet for encryption, compression, etc. The 189147e2996eSSam Leffler * secondary goal is to linearize the data so the data can be 189247e2996eSSam Leffler * passed to crypto hardware in the most efficient manner possible. 189347e2996eSSam Leffler */ 189447e2996eSSam Leffler struct mbuf * 189547e2996eSSam Leffler m_unshare(struct mbuf *m0, int how) 189647e2996eSSam Leffler { 189747e2996eSSam Leffler struct mbuf *m, *mprev; 189847e2996eSSam Leffler struct mbuf *n, *mfirst, *mlast; 189947e2996eSSam Leffler int len, off; 190047e2996eSSam Leffler 190147e2996eSSam Leffler mprev = NULL; 190247e2996eSSam Leffler for (m = m0; m != NULL; m = mprev->m_next) { 190347e2996eSSam Leffler /* 190447e2996eSSam Leffler * Regular mbufs are ignored unless there's a cluster 190547e2996eSSam Leffler * in front of it that we can use to coalesce. We do 190647e2996eSSam Leffler * the latter mainly so later clusters can be coalesced 190747e2996eSSam Leffler * also w/o having to handle them specially (i.e. convert 190847e2996eSSam Leffler * mbuf+cluster -> cluster). This optimization is heavily 190947e2996eSSam Leffler * influenced by the assumption that we're running over 191047e2996eSSam Leffler * Ethernet where MCLBYTES is large enough that the max 191147e2996eSSam Leffler * packet size will permit lots of coalescing into a 191247e2996eSSam Leffler * single cluster. This in turn permits efficient 191347e2996eSSam Leffler * crypto operations, especially when using hardware. 191447e2996eSSam Leffler */ 191547e2996eSSam Leffler if ((m->m_flags & M_EXT) == 0) { 191647e2996eSSam Leffler if (mprev && (mprev->m_flags & M_EXT) && 191747e2996eSSam Leffler m->m_len <= M_TRAILINGSPACE(mprev)) { 191847e2996eSSam Leffler /* XXX: this ignores mbuf types */ 191947e2996eSSam Leffler memcpy(mtod(mprev, caddr_t) + mprev->m_len, 192047e2996eSSam Leffler mtod(m, caddr_t), m->m_len); 192147e2996eSSam Leffler mprev->m_len += m->m_len; 192247e2996eSSam Leffler mprev->m_next = m->m_next; /* unlink from chain */ 192347e2996eSSam Leffler m_free(m); /* reclaim mbuf */ 192447e2996eSSam Leffler #if 0 192547e2996eSSam Leffler newipsecstat.ips_mbcoalesced++; 192647e2996eSSam Leffler #endif 192747e2996eSSam Leffler } else { 192847e2996eSSam Leffler mprev = m; 192947e2996eSSam Leffler } 193047e2996eSSam Leffler continue; 193147e2996eSSam Leffler } 193247e2996eSSam Leffler /* 193347e2996eSSam Leffler * Writable mbufs are left alone (for now). 193447e2996eSSam Leffler */ 193547e2996eSSam Leffler if (M_WRITABLE(m)) { 193647e2996eSSam Leffler mprev = m; 193747e2996eSSam Leffler continue; 193847e2996eSSam Leffler } 193947e2996eSSam Leffler 194047e2996eSSam Leffler /* 194147e2996eSSam Leffler * Not writable, replace with a copy or coalesce with 194247e2996eSSam Leffler * the previous mbuf if possible (since we have to copy 194347e2996eSSam Leffler * it anyway, we try to reduce the number of mbufs and 194447e2996eSSam Leffler * clusters so that future work is easier). 194547e2996eSSam Leffler */ 194647e2996eSSam Leffler KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags)); 194747e2996eSSam Leffler /* NB: we only coalesce into a cluster or larger */ 194847e2996eSSam Leffler if (mprev != NULL && (mprev->m_flags & M_EXT) && 194947e2996eSSam Leffler m->m_len <= M_TRAILINGSPACE(mprev)) { 195047e2996eSSam Leffler /* XXX: this ignores mbuf types */ 195147e2996eSSam Leffler memcpy(mtod(mprev, caddr_t) + mprev->m_len, 195247e2996eSSam Leffler mtod(m, caddr_t), m->m_len); 195347e2996eSSam Leffler mprev->m_len += m->m_len; 195447e2996eSSam Leffler mprev->m_next = m->m_next; /* unlink from chain */ 195547e2996eSSam Leffler m_free(m); /* reclaim mbuf */ 195647e2996eSSam Leffler #if 0 195747e2996eSSam Leffler newipsecstat.ips_clcoalesced++; 195847e2996eSSam Leffler #endif 195947e2996eSSam Leffler continue; 196047e2996eSSam Leffler } 196147e2996eSSam Leffler 196247e2996eSSam Leffler /* 19635368b81eSGleb Smirnoff * Allocate new space to hold the copy and copy the data. 19645368b81eSGleb Smirnoff * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by 19655368b81eSGleb Smirnoff * splitting them into clusters. We could just malloc a 19665368b81eSGleb Smirnoff * buffer and make it external but too many device drivers 19675368b81eSGleb Smirnoff * don't know how to break up the non-contiguous memory when 19685368b81eSGleb Smirnoff * doing DMA. 196947e2996eSSam Leffler */ 197047e2996eSSam Leffler n = m_getcl(how, m->m_type, m->m_flags); 197147e2996eSSam Leffler if (n == NULL) { 197247e2996eSSam Leffler m_freem(m0); 197347e2996eSSam Leffler return (NULL); 197447e2996eSSam Leffler } 197547e2996eSSam Leffler len = m->m_len; 197647e2996eSSam Leffler off = 0; 197747e2996eSSam Leffler mfirst = n; 197847e2996eSSam Leffler mlast = NULL; 197947e2996eSSam Leffler for (;;) { 198047e2996eSSam Leffler int cc = min(len, MCLBYTES); 198147e2996eSSam Leffler memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc); 198247e2996eSSam Leffler n->m_len = cc; 198347e2996eSSam Leffler if (mlast != NULL) 198447e2996eSSam Leffler mlast->m_next = n; 198547e2996eSSam Leffler mlast = n; 198647e2996eSSam Leffler #if 0 198747e2996eSSam Leffler newipsecstat.ips_clcopied++; 198847e2996eSSam Leffler #endif 198947e2996eSSam Leffler 199047e2996eSSam Leffler len -= cc; 199147e2996eSSam Leffler if (len <= 0) 199247e2996eSSam Leffler break; 199347e2996eSSam Leffler off += cc; 199447e2996eSSam Leffler 199547e2996eSSam Leffler n = m_getcl(how, m->m_type, m->m_flags); 199647e2996eSSam Leffler if (n == NULL) { 199747e2996eSSam Leffler m_freem(mfirst); 199847e2996eSSam Leffler m_freem(m0); 199947e2996eSSam Leffler return (NULL); 200047e2996eSSam Leffler } 200147e2996eSSam Leffler } 200247e2996eSSam Leffler n->m_next = m->m_next; 200347e2996eSSam Leffler if (mprev == NULL) 200447e2996eSSam Leffler m0 = mfirst; /* new head of chain */ 200547e2996eSSam Leffler else 200647e2996eSSam Leffler mprev->m_next = mfirst; /* replace old mbuf */ 200747e2996eSSam Leffler m_free(m); /* release old mbuf */ 200847e2996eSSam Leffler mprev = mfirst; 200947e2996eSSam Leffler } 201047e2996eSSam Leffler return (m0); 201147e2996eSSam Leffler } 20126eeac1d9SJulian Elischer 20136eeac1d9SJulian Elischer #ifdef MBUF_PROFILING 20146eeac1d9SJulian Elischer 20156eeac1d9SJulian Elischer #define MP_BUCKETS 32 /* don't just change this as things may overflow.*/ 20166eeac1d9SJulian Elischer struct mbufprofile { 20172182c0cfSJulian Elischer uintmax_t wasted[MP_BUCKETS]; 20182182c0cfSJulian Elischer uintmax_t used[MP_BUCKETS]; 20192182c0cfSJulian Elischer uintmax_t segments[MP_BUCKETS]; 20206eeac1d9SJulian Elischer } mbprof; 20216eeac1d9SJulian Elischer 20226eeac1d9SJulian Elischer #define MP_MAXDIGITS 21 /* strlen("16,000,000,000,000,000,000") == 21 */ 20236eeac1d9SJulian Elischer #define MP_NUMLINES 6 20246eeac1d9SJulian Elischer #define MP_NUMSPERLINE 16 20256eeac1d9SJulian Elischer #define MP_EXTRABYTES 64 /* > strlen("used:\nwasted:\nsegments:\n") */ 20266eeac1d9SJulian Elischer /* work out max space needed and add a bit of spare space too */ 20276eeac1d9SJulian Elischer #define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE) 20286eeac1d9SJulian Elischer #define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES) 20296eeac1d9SJulian Elischer 20306eeac1d9SJulian Elischer char mbprofbuf[MP_BUFSIZE]; 20316eeac1d9SJulian Elischer 20326eeac1d9SJulian Elischer void 20336eeac1d9SJulian Elischer m_profile(struct mbuf *m) 20346eeac1d9SJulian Elischer { 20356eeac1d9SJulian Elischer int segments = 0; 20366eeac1d9SJulian Elischer int used = 0; 20376eeac1d9SJulian Elischer int wasted = 0; 20386eeac1d9SJulian Elischer 20396eeac1d9SJulian Elischer while (m) { 20406eeac1d9SJulian Elischer segments++; 20416eeac1d9SJulian Elischer used += m->m_len; 20426eeac1d9SJulian Elischer if (m->m_flags & M_EXT) { 20436eeac1d9SJulian Elischer wasted += MHLEN - sizeof(m->m_ext) + 20446eeac1d9SJulian Elischer m->m_ext.ext_size - m->m_len; 20456eeac1d9SJulian Elischer } else { 20466eeac1d9SJulian Elischer if (m->m_flags & M_PKTHDR) 20476eeac1d9SJulian Elischer wasted += MHLEN - m->m_len; 20486eeac1d9SJulian Elischer else 20496eeac1d9SJulian Elischer wasted += MLEN - m->m_len; 20506eeac1d9SJulian Elischer } 20516eeac1d9SJulian Elischer m = m->m_next; 20526eeac1d9SJulian Elischer } 20536eeac1d9SJulian Elischer /* be paranoid.. it helps */ 20546eeac1d9SJulian Elischer if (segments > MP_BUCKETS - 1) 20556eeac1d9SJulian Elischer segments = MP_BUCKETS - 1; 20566eeac1d9SJulian Elischer if (used > 100000) 20576eeac1d9SJulian Elischer used = 100000; 20586eeac1d9SJulian Elischer if (wasted > 100000) 20596eeac1d9SJulian Elischer wasted = 100000; 20606eeac1d9SJulian Elischer /* store in the appropriate bucket */ 20616eeac1d9SJulian Elischer /* don't bother locking. if it's slightly off, so what? */ 20626eeac1d9SJulian Elischer mbprof.segments[segments]++; 20636eeac1d9SJulian Elischer mbprof.used[fls(used)]++; 20646eeac1d9SJulian Elischer mbprof.wasted[fls(wasted)]++; 20656eeac1d9SJulian Elischer } 20666eeac1d9SJulian Elischer 20676eeac1d9SJulian Elischer static void 20686eeac1d9SJulian Elischer mbprof_textify(void) 20696eeac1d9SJulian Elischer { 20706eeac1d9SJulian Elischer int offset; 20716eeac1d9SJulian Elischer char *c; 207260ae52f7SEd Schouten uint64_t *p; 20736eeac1d9SJulian Elischer 20746eeac1d9SJulian Elischer p = &mbprof.wasted[0]; 20756eeac1d9SJulian Elischer c = mbprofbuf; 20766eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE + 10, 20776eeac1d9SJulian Elischer "wasted:\n" 20782182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 20792182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju\n", 20806eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 20816eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 20826eeac1d9SJulian Elischer #ifdef BIG_ARRAY 20836eeac1d9SJulian Elischer p = &mbprof.wasted[16]; 20846eeac1d9SJulian Elischer c += offset; 20856eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE, 20862182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 20872182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju\n", 20886eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 20896eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 20906eeac1d9SJulian Elischer #endif 20916eeac1d9SJulian Elischer p = &mbprof.used[0]; 20926eeac1d9SJulian Elischer c += offset; 20936eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE + 10, 20946eeac1d9SJulian Elischer "used:\n" 20952182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 20962182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju\n", 20976eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 20986eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 20996eeac1d9SJulian Elischer #ifdef BIG_ARRAY 21006eeac1d9SJulian Elischer p = &mbprof.used[16]; 21016eeac1d9SJulian Elischer c += offset; 21026eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE, 21032182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 21042182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju\n", 21056eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 21066eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 21076eeac1d9SJulian Elischer #endif 21086eeac1d9SJulian Elischer p = &mbprof.segments[0]; 21096eeac1d9SJulian Elischer c += offset; 21106eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE + 10, 21116eeac1d9SJulian Elischer "segments:\n" 21122182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 21132182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju\n", 21146eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 21156eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 21166eeac1d9SJulian Elischer #ifdef BIG_ARRAY 21176eeac1d9SJulian Elischer p = &mbprof.segments[16]; 21186eeac1d9SJulian Elischer c += offset; 21196eeac1d9SJulian Elischer offset = snprintf(c, MP_MAXLINE, 21202182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %ju " 21212182c0cfSJulian Elischer "%ju %ju %ju %ju %ju %ju %ju %jju", 21226eeac1d9SJulian Elischer p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7], 21236eeac1d9SJulian Elischer p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]); 21246eeac1d9SJulian Elischer #endif 21256eeac1d9SJulian Elischer } 21266eeac1d9SJulian Elischer 21276eeac1d9SJulian Elischer static int 21286eeac1d9SJulian Elischer mbprof_handler(SYSCTL_HANDLER_ARGS) 21296eeac1d9SJulian Elischer { 21306eeac1d9SJulian Elischer int error; 21316eeac1d9SJulian Elischer 21326eeac1d9SJulian Elischer mbprof_textify(); 21336eeac1d9SJulian Elischer error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1); 21346eeac1d9SJulian Elischer return (error); 21356eeac1d9SJulian Elischer } 21366eeac1d9SJulian Elischer 21376eeac1d9SJulian Elischer static int 21386eeac1d9SJulian Elischer mbprof_clr_handler(SYSCTL_HANDLER_ARGS) 21396eeac1d9SJulian Elischer { 21406eeac1d9SJulian Elischer int clear, error; 21416eeac1d9SJulian Elischer 21426eeac1d9SJulian Elischer clear = 0; 21436eeac1d9SJulian Elischer error = sysctl_handle_int(oidp, &clear, 0, req); 21446eeac1d9SJulian Elischer if (error || !req->newptr) 21456eeac1d9SJulian Elischer return (error); 21466eeac1d9SJulian Elischer 21476eeac1d9SJulian Elischer if (clear) { 21486eeac1d9SJulian Elischer bzero(&mbprof, sizeof(mbprof)); 21496eeac1d9SJulian Elischer } 21506eeac1d9SJulian Elischer 21516eeac1d9SJulian Elischer return (error); 21526eeac1d9SJulian Elischer } 21536eeac1d9SJulian Elischer 21546eeac1d9SJulian Elischer 21556eeac1d9SJulian Elischer SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD, 21566eeac1d9SJulian Elischer NULL, 0, mbprof_handler, "A", "mbuf profiling statistics"); 21576eeac1d9SJulian Elischer 21586eeac1d9SJulian Elischer SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW, 21596eeac1d9SJulian Elischer NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics"); 21606eeac1d9SJulian Elischer #endif 21616eeac1d9SJulian Elischer 2162