xref: /freebsd/sys/kern/uipc_mbuf.c (revision ed6a66ca6c2065bc1fed93e00805676a5d3b981e)
19454b2d8SWarner Losh /*-
2df8bae1dSRodney W. Grimes  * Copyright (c) 1982, 1986, 1988, 1991, 1993
3df8bae1dSRodney W. Grimes  *	The Regents of the University of California.  All rights reserved.
4df8bae1dSRodney W. Grimes  *
5df8bae1dSRodney W. Grimes  * Redistribution and use in source and binary forms, with or without
6df8bae1dSRodney W. Grimes  * modification, are permitted provided that the following conditions
7df8bae1dSRodney W. Grimes  * are met:
8df8bae1dSRodney W. Grimes  * 1. Redistributions of source code must retain the above copyright
9df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer.
10df8bae1dSRodney W. Grimes  * 2. Redistributions in binary form must reproduce the above copyright
11df8bae1dSRodney W. Grimes  *    notice, this list of conditions and the following disclaimer in the
12df8bae1dSRodney W. Grimes  *    documentation and/or other materials provided with the distribution.
13df8bae1dSRodney W. Grimes  * 4. Neither the name of the University nor the names of its contributors
14df8bae1dSRodney W. Grimes  *    may be used to endorse or promote products derived from this software
15df8bae1dSRodney W. Grimes  *    without specific prior written permission.
16df8bae1dSRodney W. Grimes  *
17df8bae1dSRodney W. Grimes  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18df8bae1dSRodney W. Grimes  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19df8bae1dSRodney W. Grimes  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20df8bae1dSRodney W. Grimes  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21df8bae1dSRodney W. Grimes  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22df8bae1dSRodney W. Grimes  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23df8bae1dSRodney W. Grimes  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24df8bae1dSRodney W. Grimes  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25df8bae1dSRodney W. Grimes  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26df8bae1dSRodney W. Grimes  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27df8bae1dSRodney W. Grimes  * SUCH DAMAGE.
28df8bae1dSRodney W. Grimes  *
29df8bae1dSRodney W. Grimes  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
30df8bae1dSRodney W. Grimes  */
31df8bae1dSRodney W. Grimes 
32677b542eSDavid E. O'Brien #include <sys/cdefs.h>
33677b542eSDavid E. O'Brien __FBSDID("$FreeBSD$");
34677b542eSDavid E. O'Brien 
35240ef842SDavid E. O'Brien #include "opt_param.h"
36352d050eSMike Silbersack #include "opt_mbuf_stress_test.h"
376eeac1d9SJulian Elischer #include "opt_mbuf_profiling.h"
38e32a5b94SRobert Watson 
39df8bae1dSRodney W. Grimes #include <sys/param.h>
40df8bae1dSRodney W. Grimes #include <sys/systm.h>
41fb919e4dSMark Murray #include <sys/kernel.h>
42beb699c7SMike Silbersack #include <sys/limits.h>
43fb919e4dSMark Murray #include <sys/lock.h>
44f9d0d524SRobert Watson #include <sys/malloc.h>
45df8bae1dSRodney W. Grimes #include <sys/mbuf.h>
46639acc13SGarrett Wollman #include <sys/sysctl.h>
47df8bae1dSRodney W. Grimes #include <sys/domain.h>
48df8bae1dSRodney W. Grimes #include <sys/protosw.h>
49beb699c7SMike Silbersack #include <sys/uio.h>
50fb919e4dSMark Murray 
5128f8db14SBruce Evans int	max_linkhdr;
5228f8db14SBruce Evans int	max_protohdr;
5328f8db14SBruce Evans int	max_hdr;
5428f8db14SBruce Evans int	max_datalen;
5551710a45SMike Silbersack #ifdef MBUF_STRESS_TEST
5655e9f80dSMike Silbersack int	m_defragpackets;
5755e9f80dSMike Silbersack int	m_defragbytes;
5855e9f80dSMike Silbersack int	m_defraguseless;
5955e9f80dSMike Silbersack int	m_defragfailure;
60352d050eSMike Silbersack int	m_defragrandomfailures;
61352d050eSMike Silbersack #endif
627d032714SBosko Milekic 
637d032714SBosko Milekic /*
647d032714SBosko Milekic  * sysctl(8) exported objects
657d032714SBosko Milekic  */
6680444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
6780444f88SAndre Oppermann 	   &max_linkhdr, 0, "Size of largest link layer header");
6880444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
6980444f88SAndre Oppermann 	   &max_protohdr, 0, "Size of largest protocol layer header");
7080444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
7180444f88SAndre Oppermann 	   &max_hdr, 0, "Size of largest link plus protocol header");
7280444f88SAndre Oppermann SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD,
7380444f88SAndre Oppermann 	   &max_datalen, 0, "Minimum space left in mbuf after max_hdr");
7451710a45SMike Silbersack #ifdef MBUF_STRESS_TEST
7555e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
7655e9f80dSMike Silbersack 	   &m_defragpackets, 0, "");
7755e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
7855e9f80dSMike Silbersack 	   &m_defragbytes, 0, "");
7955e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
8055e9f80dSMike Silbersack 	   &m_defraguseless, 0, "");
8155e9f80dSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
8255e9f80dSMike Silbersack 	   &m_defragfailure, 0, "");
83352d050eSMike Silbersack SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
84352d050eSMike Silbersack 	   &m_defragrandomfailures, 0, "");
85352d050eSMike Silbersack #endif
86df8bae1dSRodney W. Grimes 
87df8bae1dSRodney W. Grimes /*
88f729ede6SAndre Oppermann  * Ensure the correct size of various mbuf parameters.  It could be off due
89f729ede6SAndre Oppermann  * to compiler-induced padding and alignment artifacts.
90f729ede6SAndre Oppermann  */
91f729ede6SAndre Oppermann CTASSERT(sizeof(struct mbuf) == MSIZE);
92f729ede6SAndre Oppermann CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
93f729ede6SAndre Oppermann CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
94f729ede6SAndre Oppermann 
95f729ede6SAndre Oppermann /*
9629110f87SGleb Smirnoff  * m_get2() allocates minimum mbuf that would fit "size" argument.
9729110f87SGleb Smirnoff  */
9829110f87SGleb Smirnoff struct mbuf *
9941a7572bSGleb Smirnoff m_get2(int size, int how, short type, int flags)
10029110f87SGleb Smirnoff {
10129110f87SGleb Smirnoff 	struct mb_args args;
10229110f87SGleb Smirnoff 	struct mbuf *m, *n;
10329110f87SGleb Smirnoff 
10429110f87SGleb Smirnoff 	args.flags = flags;
10529110f87SGleb Smirnoff 	args.type = type;
10629110f87SGleb Smirnoff 
10729110f87SGleb Smirnoff 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
10829110f87SGleb Smirnoff 		return (uma_zalloc_arg(zone_mbuf, &args, how));
10929110f87SGleb Smirnoff 	if (size <= MCLBYTES)
11029110f87SGleb Smirnoff 		return (uma_zalloc_arg(zone_pack, &args, how));
1113112ae76SGleb Smirnoff 
1123112ae76SGleb Smirnoff 	if (size > MJUMPAGESIZE)
11329110f87SGleb Smirnoff 		return (NULL);
11429110f87SGleb Smirnoff 
11529110f87SGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
11629110f87SGleb Smirnoff 	if (m == NULL)
11729110f87SGleb Smirnoff 		return (NULL);
11829110f87SGleb Smirnoff 
1193112ae76SGleb Smirnoff 	n = uma_zalloc_arg(zone_jumbop, m, how);
12029110f87SGleb Smirnoff 	if (n == NULL) {
12129110f87SGleb Smirnoff 		uma_zfree(zone_mbuf, m);
12229110f87SGleb Smirnoff 		return (NULL);
12329110f87SGleb Smirnoff 	}
12429110f87SGleb Smirnoff 
12529110f87SGleb Smirnoff 	return (m);
12629110f87SGleb Smirnoff }
12729110f87SGleb Smirnoff 
12829110f87SGleb Smirnoff /*
12929110f87SGleb Smirnoff  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
13029110f87SGleb Smirnoff  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
13129110f87SGleb Smirnoff  */
13229110f87SGleb Smirnoff struct mbuf *
13329110f87SGleb Smirnoff m_getjcl(int how, short type, int flags, int size)
13429110f87SGleb Smirnoff {
13529110f87SGleb Smirnoff 	struct mb_args args;
13629110f87SGleb Smirnoff 	struct mbuf *m, *n;
13729110f87SGleb Smirnoff 	uma_zone_t zone;
13829110f87SGleb Smirnoff 
13929110f87SGleb Smirnoff 	if (size == MCLBYTES)
14029110f87SGleb Smirnoff 		return m_getcl(how, type, flags);
14129110f87SGleb Smirnoff 
14229110f87SGleb Smirnoff 	args.flags = flags;
14329110f87SGleb Smirnoff 	args.type = type;
14429110f87SGleb Smirnoff 
14529110f87SGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
14629110f87SGleb Smirnoff 	if (m == NULL)
14729110f87SGleb Smirnoff 		return (NULL);
14829110f87SGleb Smirnoff 
14929110f87SGleb Smirnoff 	zone = m_getzone(size);
15029110f87SGleb Smirnoff 	n = uma_zalloc_arg(zone, m, how);
15129110f87SGleb Smirnoff 	if (n == NULL) {
15229110f87SGleb Smirnoff 		uma_zfree(zone_mbuf, m);
15329110f87SGleb Smirnoff 		return (NULL);
15429110f87SGleb Smirnoff 	}
15529110f87SGleb Smirnoff 	return (m);
15629110f87SGleb Smirnoff }
15729110f87SGleb Smirnoff 
15829110f87SGleb Smirnoff /*
159099a0e58SBosko Milekic  * Allocate a given length worth of mbufs and/or clusters (whatever fits
160099a0e58SBosko Milekic  * best) and return a pointer to the top of the allocated chain.  If an
161099a0e58SBosko Milekic  * existing mbuf chain is provided, then we will append the new chain
162099a0e58SBosko Milekic  * to the existing one but still return the top of the newly allocated
163099a0e58SBosko Milekic  * chain.
164099a0e58SBosko Milekic  */
165099a0e58SBosko Milekic struct mbuf *
1665e20f43dSAndre Oppermann m_getm2(struct mbuf *m, int len, int how, short type, int flags)
167099a0e58SBosko Milekic {
1685e20f43dSAndre Oppermann 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
169099a0e58SBosko Milekic 
1705e20f43dSAndre Oppermann 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
171099a0e58SBosko Milekic 
1725e20f43dSAndre Oppermann 	/* Validate flags. */
1735e20f43dSAndre Oppermann 	flags &= (M_PKTHDR | M_EOR);
1745e20f43dSAndre Oppermann 
1755e20f43dSAndre Oppermann 	/* Packet header mbuf must be first in chain. */
1765e20f43dSAndre Oppermann 	if ((flags & M_PKTHDR) && m != NULL)
1775e20f43dSAndre Oppermann 		flags &= ~M_PKTHDR;
1785e20f43dSAndre Oppermann 
1795e20f43dSAndre Oppermann 	/* Loop and append maximum sized mbufs to the chain tail. */
1805e20f43dSAndre Oppermann 	while (len > 0) {
1815e20f43dSAndre Oppermann 		if (len > MCLBYTES)
1825e20f43dSAndre Oppermann 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
1835e20f43dSAndre Oppermann 			    MJUMPAGESIZE);
1845e20f43dSAndre Oppermann 		else if (len >= MINCLSIZE)
1855e20f43dSAndre Oppermann 			mb = m_getcl(how, type, (flags & M_PKTHDR));
1865e20f43dSAndre Oppermann 		else if (flags & M_PKTHDR)
1875e20f43dSAndre Oppermann 			mb = m_gethdr(how, type);
188099a0e58SBosko Milekic 		else
1895e20f43dSAndre Oppermann 			mb = m_get(how, type);
190099a0e58SBosko Milekic 
1915e20f43dSAndre Oppermann 		/* Fail the whole operation if one mbuf can't be allocated. */
1925e20f43dSAndre Oppermann 		if (mb == NULL) {
1935e20f43dSAndre Oppermann 			if (nm != NULL)
1945e20f43dSAndre Oppermann 				m_freem(nm);
1955e20f43dSAndre Oppermann 			return (NULL);
196099a0e58SBosko Milekic 		}
197099a0e58SBosko Milekic 
1985e20f43dSAndre Oppermann 		/* Book keeping. */
1995e20f43dSAndre Oppermann 		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
2005e20f43dSAndre Oppermann 			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
201099a0e58SBosko Milekic 		if (mtail != NULL)
2025e20f43dSAndre Oppermann 			mtail->m_next = mb;
2035e20f43dSAndre Oppermann 		else
2045e20f43dSAndre Oppermann 			nm = mb;
2055e20f43dSAndre Oppermann 		mtail = mb;
2065e20f43dSAndre Oppermann 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
2075e20f43dSAndre Oppermann 	}
2085e20f43dSAndre Oppermann 	if (flags & M_EOR)
2095e20f43dSAndre Oppermann 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
2105e20f43dSAndre Oppermann 
2115e20f43dSAndre Oppermann 	/* If mbuf was supplied, append new chain to the end of it. */
2125e20f43dSAndre Oppermann 	if (m != NULL) {
2135e20f43dSAndre Oppermann 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
2145e20f43dSAndre Oppermann 			;
2155e20f43dSAndre Oppermann 		mtail->m_next = nm;
2165e20f43dSAndre Oppermann 		mtail->m_flags &= ~M_EOR;
2175e20f43dSAndre Oppermann 	} else
2185e20f43dSAndre Oppermann 		m = nm;
2195e20f43dSAndre Oppermann 
2205e20f43dSAndre Oppermann 	return (m);
221099a0e58SBosko Milekic }
222099a0e58SBosko Milekic 
223099a0e58SBosko Milekic /*
224099a0e58SBosko Milekic  * Free an entire chain of mbufs and associated external buffers, if
225099a0e58SBosko Milekic  * applicable.
226099a0e58SBosko Milekic  */
227099a0e58SBosko Milekic void
228099a0e58SBosko Milekic m_freem(struct mbuf *mb)
229099a0e58SBosko Milekic {
230099a0e58SBosko Milekic 
231099a0e58SBosko Milekic 	while (mb != NULL)
232099a0e58SBosko Milekic 		mb = m_free(mb);
233099a0e58SBosko Milekic }
234099a0e58SBosko Milekic 
2351a996ed1SEdward Tomasz Napierala /*-
236099a0e58SBosko Milekic  * Configure a provided mbuf to refer to the provided external storage
237099a0e58SBosko Milekic  * buffer and setup a reference count for said buffer.  If the setting
238099a0e58SBosko Milekic  * up of the reference count fails, the M_EXT bit will not be set.  If
239099a0e58SBosko Milekic  * successfull, the M_EXT bit is set in the mbuf's flags.
240099a0e58SBosko Milekic  *
241099a0e58SBosko Milekic  * Arguments:
242099a0e58SBosko Milekic  *    mb     The existing mbuf to which to attach the provided buffer.
243099a0e58SBosko Milekic  *    buf    The address of the provided external storage buffer.
244099a0e58SBosko Milekic  *    size   The size of the provided buffer.
245099a0e58SBosko Milekic  *    freef  A pointer to a routine that is responsible for freeing the
246099a0e58SBosko Milekic  *           provided external storage buffer.
247099a0e58SBosko Milekic  *    args   A pointer to an argument structure (of any type) to be passed
248099a0e58SBosko Milekic  *           to the provided freef routine (may be NULL).
249099a0e58SBosko Milekic  *    flags  Any other flags to be passed to the provided mbuf.
250099a0e58SBosko Milekic  *    type   The type that the external storage buffer should be
251099a0e58SBosko Milekic  *           labeled with.
252099a0e58SBosko Milekic  *
253099a0e58SBosko Milekic  * Returns:
254099a0e58SBosko Milekic  *    Nothing.
255099a0e58SBosko Milekic  */
2568c629bdfSGleb Smirnoff int
257099a0e58SBosko Milekic m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
25815c28f87SGleb Smirnoff     void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2,
2599a736876SAndre Oppermann     int flags, int type, int wait)
260099a0e58SBosko Milekic {
26156a4e45aSAndre Oppermann 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
262099a0e58SBosko Milekic 
26356a4e45aSAndre Oppermann 	if (type != EXT_EXTREF)
264fcc34a23SGleb Smirnoff 		mb->m_ext.ext_cnt = uma_zalloc(zone_ext_refcnt, wait);
2658c629bdfSGleb Smirnoff 
266fcc34a23SGleb Smirnoff 	if (mb->m_ext.ext_cnt == NULL)
2678c629bdfSGleb Smirnoff 		return (ENOMEM);
2688c629bdfSGleb Smirnoff 
269fcc34a23SGleb Smirnoff 	*(mb->m_ext.ext_cnt) = 1;
270099a0e58SBosko Milekic 	mb->m_flags |= (M_EXT | flags);
271099a0e58SBosko Milekic 	mb->m_ext.ext_buf = buf;
272099a0e58SBosko Milekic 	mb->m_data = mb->m_ext.ext_buf;
273099a0e58SBosko Milekic 	mb->m_ext.ext_size = size;
274099a0e58SBosko Milekic 	mb->m_ext.ext_free = freef;
275cf827063SPoul-Henning Kamp 	mb->m_ext.ext_arg1 = arg1;
276cf827063SPoul-Henning Kamp 	mb->m_ext.ext_arg2 = arg2;
277099a0e58SBosko Milekic 	mb->m_ext.ext_type = type;
278894734cbSAndre Oppermann 	mb->m_ext.ext_flags = 0;
2798c629bdfSGleb Smirnoff 
2808c629bdfSGleb Smirnoff 	return (0);
281099a0e58SBosko Milekic }
282099a0e58SBosko Milekic 
283099a0e58SBosko Milekic /*
284099a0e58SBosko Milekic  * Non-directly-exported function to clean up after mbufs with M_EXT
28556a4e45aSAndre Oppermann  * storage attached to them if the reference count hits 1.
286099a0e58SBosko Milekic  */
287099a0e58SBosko Milekic void
288099a0e58SBosko Milekic mb_free_ext(struct mbuf *m)
289099a0e58SBosko Milekic {
2901fbe6a82SGleb Smirnoff 	int freembuf;
291457869b9SKip Macy 
2921fbe6a82SGleb Smirnoff 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
2933d2a3ff2SBosko Milekic 
2941227f20dSAndre Oppermann 	/*
2951fbe6a82SGleb Smirnoff 	 * Check if the header is embedded in the cluster.
2961227f20dSAndre Oppermann 	 */
2971fbe6a82SGleb Smirnoff 	freembuf = (m->m_flags & M_NOFREE) ? 0 : 1;
2981227f20dSAndre Oppermann 
2991fbe6a82SGleb Smirnoff 	switch (m->m_ext.ext_type) {
3001fbe6a82SGleb Smirnoff 	case EXT_SFBUF:
3011fbe6a82SGleb Smirnoff 		sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
3021fbe6a82SGleb Smirnoff 		break;
3031fbe6a82SGleb Smirnoff 	default:
3041fbe6a82SGleb Smirnoff 		KASSERT(m->m_ext.ext_cnt != NULL,
3051fbe6a82SGleb Smirnoff 		    ("%s: no refcounting pointer on %p", __func__, m));
3061fbe6a82SGleb Smirnoff 		/*
3071fbe6a82SGleb Smirnoff 		 * Free attached storage if this mbuf is the only
3081fbe6a82SGleb Smirnoff 		 * reference to it.
3091fbe6a82SGleb Smirnoff 		 */
3101fbe6a82SGleb Smirnoff 		if (*(m->m_ext.ext_cnt) != 1) {
3111fbe6a82SGleb Smirnoff 			if (atomic_fetchadd_int(m->m_ext.ext_cnt, -1) != 1)
3121fbe6a82SGleb Smirnoff 				break;
3131fbe6a82SGleb Smirnoff 		}
3141fbe6a82SGleb Smirnoff 
31556a4e45aSAndre Oppermann 		switch (m->m_ext.ext_type) {
316cd5bb63bSAndre Oppermann 		case EXT_PACKET:	/* The packet zone is special. */
317fcc34a23SGleb Smirnoff 			if (*(m->m_ext.ext_cnt) == 0)
318fcc34a23SGleb Smirnoff 				*(m->m_ext.ext_cnt) = 1;
319099a0e58SBosko Milekic 			uma_zfree(zone_pack, m);
32056a4e45aSAndre Oppermann 			return;		/* Job done. */
321cd5bb63bSAndre Oppermann 		case EXT_CLUSTER:
322cd5bb63bSAndre Oppermann 			uma_zfree(zone_clust, m->m_ext.ext_buf);
32356a4e45aSAndre Oppermann 			break;
324ec63cb90SAndre Oppermann 		case EXT_JUMBOP:
325ec63cb90SAndre Oppermann 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
326d5269a63SAndre Oppermann 			break;
32756a4e45aSAndre Oppermann 		case EXT_JUMBO9:
32856a4e45aSAndre Oppermann 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
32956a4e45aSAndre Oppermann 			break;
33056a4e45aSAndre Oppermann 		case EXT_JUMBO16:
33156a4e45aSAndre Oppermann 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
33256a4e45aSAndre Oppermann 			break;
33356a4e45aSAndre Oppermann 		case EXT_NET_DRV:
33456a4e45aSAndre Oppermann 		case EXT_MOD_TYPE:
33556a4e45aSAndre Oppermann 		case EXT_DISPOSABLE:
336fcc34a23SGleb Smirnoff 			*(m->m_ext.ext_cnt) = 0;
33756a4e45aSAndre Oppermann 			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
338fcc34a23SGleb Smirnoff 				m->m_ext.ext_cnt));
33956a4e45aSAndre Oppermann 			/* FALLTHROUGH */
34056a4e45aSAndre Oppermann 		case EXT_EXTREF:
34156a4e45aSAndre Oppermann 			KASSERT(m->m_ext.ext_free != NULL,
34256a4e45aSAndre Oppermann 				("%s: ext_free not set", __func__));
34315c28f87SGleb Smirnoff 			(*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1,
344cf827063SPoul-Henning Kamp 			    m->m_ext.ext_arg2);
34556a4e45aSAndre Oppermann 			break;
34656a4e45aSAndre Oppermann 		default:
34756a4e45aSAndre Oppermann 			KASSERT(m->m_ext.ext_type == 0,
34856a4e45aSAndre Oppermann 				("%s: unknown ext_type", __func__));
3493d2a3ff2SBosko Milekic 		}
35056a4e45aSAndre Oppermann 	}
351457869b9SKip Macy 
3521fbe6a82SGleb Smirnoff 	if (freembuf)
35396e12413SBosko Milekic 		uma_zfree(zone_mbuf, m);
354b5b2ea9aSBosko Milekic }
355099a0e58SBosko Milekic 
356099a0e58SBosko Milekic /*
3576bccea7cSRebecca Cran  * Attach the cluster from *m to *n, set up m_ext in *n
35856a4e45aSAndre Oppermann  * and bump the refcount of the cluster.
35956a4e45aSAndre Oppermann  */
36056a4e45aSAndre Oppermann static void
36156a4e45aSAndre Oppermann mb_dupcl(struct mbuf *n, struct mbuf *m)
36256a4e45aSAndre Oppermann {
36356a4e45aSAndre Oppermann 
3641fbe6a82SGleb Smirnoff 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
3651fbe6a82SGleb Smirnoff 	KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
3661fbe6a82SGleb Smirnoff 
3671fbe6a82SGleb Smirnoff 	switch (m->m_ext.ext_type) {
3681fbe6a82SGleb Smirnoff 	case EXT_SFBUF:
3691fbe6a82SGleb Smirnoff 		sf_ext_ref(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
3701fbe6a82SGleb Smirnoff 		break;
3711fbe6a82SGleb Smirnoff 	default:
3721fbe6a82SGleb Smirnoff 		KASSERT(m->m_ext.ext_cnt != NULL,
3731fbe6a82SGleb Smirnoff 		    ("%s: no refcounting pointer on %p", __func__, m));
374fcc34a23SGleb Smirnoff 		if (*(m->m_ext.ext_cnt) == 1)
375fcc34a23SGleb Smirnoff 			*(m->m_ext.ext_cnt) += 1;
37656a4e45aSAndre Oppermann 		else
377fcc34a23SGleb Smirnoff 			atomic_add_int(m->m_ext.ext_cnt, 1);
3781fbe6a82SGleb Smirnoff 	}
3791fbe6a82SGleb Smirnoff 
380c71b4037SGleb Smirnoff 	n->m_ext = m->m_ext;
38156a4e45aSAndre Oppermann 	n->m_flags |= M_EXT;
38232a8b1d8SColin Percival 	n->m_flags |= m->m_flags & M_RDONLY;
38356a4e45aSAndre Oppermann }
38456a4e45aSAndre Oppermann 
38556a4e45aSAndre Oppermann /*
386ed111688SAndre Oppermann  * Clean up mbuf (chain) from any tags and packet headers.
387e0068c3aSAndre Oppermann  * If "all" is set then the first mbuf in the chain will be
388e0068c3aSAndre Oppermann  * cleaned too.
389ed111688SAndre Oppermann  */
390ed111688SAndre Oppermann void
391651e4e6aSGleb Smirnoff m_demote(struct mbuf *m0, int all, int flags)
392ed111688SAndre Oppermann {
393ed111688SAndre Oppermann 	struct mbuf *m;
394ed111688SAndre Oppermann 
395ed111688SAndre Oppermann 	for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
3967ee2d058SGleb Smirnoff 		KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p",
3977ee2d058SGleb Smirnoff 		    __func__, m, m0));
398ed111688SAndre Oppermann 		if (m->m_flags & M_PKTHDR) {
399ed111688SAndre Oppermann 			m_tag_delete_chain(m, NULL);
400ed111688SAndre Oppermann 			m->m_flags &= ~M_PKTHDR;
401ed111688SAndre Oppermann 			bzero(&m->m_pkthdr, sizeof(struct pkthdr));
402ed111688SAndre Oppermann 		}
403651e4e6aSGleb Smirnoff 		m->m_flags = m->m_flags & (M_EXT | M_RDONLY | M_NOFREE | flags);
404ed111688SAndre Oppermann 	}
405ed111688SAndre Oppermann }
406ed111688SAndre Oppermann 
407ed111688SAndre Oppermann /*
408fdcc028dSAndre Oppermann  * Sanity checks on mbuf (chain) for use in KASSERT() and general
409fdcc028dSAndre Oppermann  * debugging.
410fdcc028dSAndre Oppermann  * Returns 0 or panics when bad and 1 on all tests passed.
411fdcc028dSAndre Oppermann  * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
412fdcc028dSAndre Oppermann  * blow up later.
413a048affbSAndre Oppermann  */
414a048affbSAndre Oppermann int
415a048affbSAndre Oppermann m_sanity(struct mbuf *m0, int sanitize)
416a048affbSAndre Oppermann {
417a048affbSAndre Oppermann 	struct mbuf *m;
418a048affbSAndre Oppermann 	caddr_t a, b;
419a048affbSAndre Oppermann 	int pktlen = 0;
420a048affbSAndre Oppermann 
42121ee3e7aSKip Macy #ifdef INVARIANTS
42221ee3e7aSKip Macy #define	M_SANITY_ACTION(s)	panic("mbuf %p: " s, m)
42321ee3e7aSKip Macy #else
42421ee3e7aSKip Macy #define	M_SANITY_ACTION(s)	printf("mbuf %p: " s, m)
42521ee3e7aSKip Macy #endif
426a048affbSAndre Oppermann 
427fdcc028dSAndre Oppermann 	for (m = m0; m != NULL; m = m->m_next) {
428a048affbSAndre Oppermann 		/*
429a048affbSAndre Oppermann 		 * Basic pointer checks.  If any of these fails then some
430a048affbSAndre Oppermann 		 * unrelated kernel memory before or after us is trashed.
431a048affbSAndre Oppermann 		 * No way to recover from that.
432a048affbSAndre Oppermann 		 */
433fdcc028dSAndre Oppermann 		a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
434fdcc028dSAndre Oppermann 			((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
435a048affbSAndre Oppermann 			 (caddr_t)(&m->m_dat)) );
436a048affbSAndre Oppermann 		b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
437fdcc028dSAndre Oppermann 			((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
438a048affbSAndre Oppermann 		if ((caddr_t)m->m_data < a)
439a048affbSAndre Oppermann 			M_SANITY_ACTION("m_data outside mbuf data range left");
440a048affbSAndre Oppermann 		if ((caddr_t)m->m_data > b)
441a048affbSAndre Oppermann 			M_SANITY_ACTION("m_data outside mbuf data range right");
442a048affbSAndre Oppermann 		if ((caddr_t)m->m_data + m->m_len > b)
443a048affbSAndre Oppermann 			M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
444a048affbSAndre Oppermann 
445a048affbSAndre Oppermann 		/* m->m_nextpkt may only be set on first mbuf in chain. */
446fdcc028dSAndre Oppermann 		if (m != m0 && m->m_nextpkt != NULL) {
447a048affbSAndre Oppermann 			if (sanitize) {
448a048affbSAndre Oppermann 				m_freem(m->m_nextpkt);
449a048affbSAndre Oppermann 				m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
450a048affbSAndre Oppermann 			} else
451a048affbSAndre Oppermann 				M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
452a048affbSAndre Oppermann 		}
453a048affbSAndre Oppermann 
454a048affbSAndre Oppermann 		/* packet length (not mbuf length!) calculation */
455a048affbSAndre Oppermann 		if (m0->m_flags & M_PKTHDR)
456a048affbSAndre Oppermann 			pktlen += m->m_len;
457a048affbSAndre Oppermann 
458a048affbSAndre Oppermann 		/* m_tags may only be attached to first mbuf in chain. */
459a048affbSAndre Oppermann 		if (m != m0 && m->m_flags & M_PKTHDR &&
460a048affbSAndre Oppermann 		    !SLIST_EMPTY(&m->m_pkthdr.tags)) {
461a048affbSAndre Oppermann 			if (sanitize) {
462a048affbSAndre Oppermann 				m_tag_delete_chain(m, NULL);
463a048affbSAndre Oppermann 				/* put in 0xDEADC0DE perhaps? */
464fdcc028dSAndre Oppermann 			} else
465a048affbSAndre Oppermann 				M_SANITY_ACTION("m_tags on in-chain mbuf");
466a048affbSAndre Oppermann 		}
467a048affbSAndre Oppermann 
468a048affbSAndre Oppermann 		/* M_PKTHDR may only be set on first mbuf in chain */
469a048affbSAndre Oppermann 		if (m != m0 && m->m_flags & M_PKTHDR) {
470a048affbSAndre Oppermann 			if (sanitize) {
471a048affbSAndre Oppermann 				bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
472a048affbSAndre Oppermann 				m->m_flags &= ~M_PKTHDR;
473a048affbSAndre Oppermann 				/* put in 0xDEADCODE and leave hdr flag in */
474a048affbSAndre Oppermann 			} else
475a048affbSAndre Oppermann 				M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
476a048affbSAndre Oppermann 		}
477a048affbSAndre Oppermann 	}
478fdcc028dSAndre Oppermann 	m = m0;
479fdcc028dSAndre Oppermann 	if (pktlen && pktlen != m->m_pkthdr.len) {
480a048affbSAndre Oppermann 		if (sanitize)
481fdcc028dSAndre Oppermann 			m->m_pkthdr.len = 0;
482a048affbSAndre Oppermann 		else
483a048affbSAndre Oppermann 			M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
484a048affbSAndre Oppermann 	}
485a048affbSAndre Oppermann 	return 1;
486fdcc028dSAndre Oppermann 
487fdcc028dSAndre Oppermann #undef	M_SANITY_ACTION
488a048affbSAndre Oppermann }
489a048affbSAndre Oppermann 
490a048affbSAndre Oppermann 
491a048affbSAndre Oppermann /*
4929967cafcSSam Leffler  * "Move" mbuf pkthdr from "from" to "to".
493e37b1fcdSRobert Watson  * "from" must have M_PKTHDR set, and "to" must be empty.
494e37b1fcdSRobert Watson  */
495e37b1fcdSRobert Watson void
4969967cafcSSam Leffler m_move_pkthdr(struct mbuf *to, struct mbuf *from)
497e37b1fcdSRobert Watson {
498e37b1fcdSRobert Watson 
499e37b1fcdSRobert Watson #if 0
5009967cafcSSam Leffler 	/* see below for why these are not enabled */
501fe584538SDag-Erling Smørgrav 	M_ASSERTPKTHDR(to);
502225bff6fSRobert Watson 	/* Note: with MAC, this may not be a good assertion. */
5039967cafcSSam Leffler 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
5049967cafcSSam Leffler 	    ("m_move_pkthdr: to has tags"));
505e37b1fcdSRobert Watson #endif
506e32a5b94SRobert Watson #ifdef MAC
507225bff6fSRobert Watson 	/*
508225bff6fSRobert Watson 	 * XXXMAC: It could be this should also occur for non-MAC?
509225bff6fSRobert Watson 	 */
510e32a5b94SRobert Watson 	if (to->m_flags & M_PKTHDR)
511225bff6fSRobert Watson 		m_tag_delete_chain(to, NULL);
512e32a5b94SRobert Watson #endif
513a4e71429SSam Leffler 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
514a4e71429SSam Leffler 	if ((to->m_flags & M_EXT) == 0)
5159967cafcSSam Leffler 		to->m_data = to->m_pktdat;
5169967cafcSSam Leffler 	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
5179967cafcSSam Leffler 	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
5189967cafcSSam Leffler 	from->m_flags &= ~M_PKTHDR;
5199967cafcSSam Leffler }
5209967cafcSSam Leffler 
5219967cafcSSam Leffler /*
5229967cafcSSam Leffler  * Duplicate "from"'s mbuf pkthdr in "to".
5239967cafcSSam Leffler  * "from" must have M_PKTHDR set, and "to" must be empty.
5249967cafcSSam Leffler  * In particular, this does a deep copy of the packet tags.
5259967cafcSSam Leffler  */
5269967cafcSSam Leffler int
5279967cafcSSam Leffler m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
5289967cafcSSam Leffler {
5299967cafcSSam Leffler 
5309967cafcSSam Leffler #if 0
5319967cafcSSam Leffler 	/*
5329967cafcSSam Leffler 	 * The mbuf allocator only initializes the pkthdr
533c95be8b5SGleb Smirnoff 	 * when the mbuf is allocated with m_gethdr(). Many users
534c95be8b5SGleb Smirnoff 	 * (e.g. m_copy*, m_prepend) use m_get() and then
5359967cafcSSam Leffler 	 * smash the pkthdr as needed causing these
5369967cafcSSam Leffler 	 * assertions to trip.  For now just disable them.
5379967cafcSSam Leffler 	 */
538fe584538SDag-Erling Smørgrav 	M_ASSERTPKTHDR(to);
539225bff6fSRobert Watson 	/* Note: with MAC, this may not be a good assertion. */
5409967cafcSSam Leffler 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
5419967cafcSSam Leffler #endif
542063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(how);
5439967cafcSSam Leffler #ifdef MAC
5449967cafcSSam Leffler 	if (to->m_flags & M_PKTHDR)
545225bff6fSRobert Watson 		m_tag_delete_chain(to, NULL);
5469967cafcSSam Leffler #endif
547df8c7fc9SMike Silbersack 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
548df8c7fc9SMike Silbersack 	if ((to->m_flags & M_EXT) == 0)
5499967cafcSSam Leffler 		to->m_data = to->m_pktdat;
550e37b1fcdSRobert Watson 	to->m_pkthdr = from->m_pkthdr;
5519967cafcSSam Leffler 	SLIST_INIT(&to->m_pkthdr.tags);
55294985f74SGleb Smirnoff 	return (m_tag_copy_chain(to, from, how));
553e37b1fcdSRobert Watson }
554e37b1fcdSRobert Watson 
555e37b1fcdSRobert Watson /*
556df8bae1dSRodney W. Grimes  * Lesser-used path for M_PREPEND:
557df8bae1dSRodney W. Grimes  * allocate new mbuf to prepend to chain,
558df8bae1dSRodney W. Grimes  * copy junk along.
559df8bae1dSRodney W. Grimes  */
560df8bae1dSRodney W. Grimes struct mbuf *
561122a814aSBosko Milekic m_prepend(struct mbuf *m, int len, int how)
562df8bae1dSRodney W. Grimes {
563df8bae1dSRodney W. Grimes 	struct mbuf *mn;
564df8bae1dSRodney W. Grimes 
565f8bf8e39SMike Silbersack 	if (m->m_flags & M_PKTHDR)
566c95be8b5SGleb Smirnoff 		mn = m_gethdr(how, m->m_type);
567f8bf8e39SMike Silbersack 	else
568c95be8b5SGleb Smirnoff 		mn = m_get(how, m->m_type);
569122a814aSBosko Milekic 	if (mn == NULL) {
570df8bae1dSRodney W. Grimes 		m_freem(m);
571122a814aSBosko Milekic 		return (NULL);
572df8bae1dSRodney W. Grimes 	}
573225bff6fSRobert Watson 	if (m->m_flags & M_PKTHDR)
574c95be8b5SGleb Smirnoff 		m_move_pkthdr(mn, m);
575df8bae1dSRodney W. Grimes 	mn->m_next = m;
576df8bae1dSRodney W. Grimes 	m = mn;
577*ed6a66caSRobert Watson 	if (len < M_SIZE(m))
5785288989fSRandall Stewart 		M_ALIGN(m, len);
579df8bae1dSRodney W. Grimes 	m->m_len = len;
580df8bae1dSRodney W. Grimes 	return (m);
581df8bae1dSRodney W. Grimes }
582df8bae1dSRodney W. Grimes 
583df8bae1dSRodney W. Grimes /*
584df8bae1dSRodney W. Grimes  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
585df8bae1dSRodney W. Grimes  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
586eb1b1807SGleb Smirnoff  * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
5871c38f2eaSArchie Cobbs  * Note that the copy is read-only, because clusters are not copied,
5881c38f2eaSArchie Cobbs  * only their reference counts are incremented.
589df8bae1dSRodney W. Grimes  */
590df8bae1dSRodney W. Grimes struct mbuf *
591122a814aSBosko Milekic m_copym(struct mbuf *m, int off0, int len, int wait)
592df8bae1dSRodney W. Grimes {
593122a814aSBosko Milekic 	struct mbuf *n, **np;
594122a814aSBosko Milekic 	int off = off0;
595df8bae1dSRodney W. Grimes 	struct mbuf *top;
596df8bae1dSRodney W. Grimes 	int copyhdr = 0;
597df8bae1dSRodney W. Grimes 
598e0a653ddSAlfred Perlstein 	KASSERT(off >= 0, ("m_copym, negative off %d", off));
599e0a653ddSAlfred Perlstein 	KASSERT(len >= 0, ("m_copym, negative len %d", len));
600063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(wait);
601df8bae1dSRodney W. Grimes 	if (off == 0 && m->m_flags & M_PKTHDR)
602df8bae1dSRodney W. Grimes 		copyhdr = 1;
603df8bae1dSRodney W. Grimes 	while (off > 0) {
604e0a653ddSAlfred Perlstein 		KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
605df8bae1dSRodney W. Grimes 		if (off < m->m_len)
606df8bae1dSRodney W. Grimes 			break;
607df8bae1dSRodney W. Grimes 		off -= m->m_len;
608df8bae1dSRodney W. Grimes 		m = m->m_next;
609df8bae1dSRodney W. Grimes 	}
610df8bae1dSRodney W. Grimes 	np = &top;
611df8bae1dSRodney W. Grimes 	top = 0;
612df8bae1dSRodney W. Grimes 	while (len > 0) {
613122a814aSBosko Milekic 		if (m == NULL) {
614e0a653ddSAlfred Perlstein 			KASSERT(len == M_COPYALL,
615e0a653ddSAlfred Perlstein 			    ("m_copym, length > size of mbuf chain"));
616df8bae1dSRodney W. Grimes 			break;
617df8bae1dSRodney W. Grimes 		}
618f8bf8e39SMike Silbersack 		if (copyhdr)
619c95be8b5SGleb Smirnoff 			n = m_gethdr(wait, m->m_type);
620f8bf8e39SMike Silbersack 		else
621c95be8b5SGleb Smirnoff 			n = m_get(wait, m->m_type);
622df8bae1dSRodney W. Grimes 		*np = n;
623122a814aSBosko Milekic 		if (n == NULL)
624df8bae1dSRodney W. Grimes 			goto nospace;
625df8bae1dSRodney W. Grimes 		if (copyhdr) {
6269967cafcSSam Leffler 			if (!m_dup_pkthdr(n, m, wait))
6279967cafcSSam Leffler 				goto nospace;
628df8bae1dSRodney W. Grimes 			if (len == M_COPYALL)
629df8bae1dSRodney W. Grimes 				n->m_pkthdr.len -= off0;
630df8bae1dSRodney W. Grimes 			else
631df8bae1dSRodney W. Grimes 				n->m_pkthdr.len = len;
632df8bae1dSRodney W. Grimes 			copyhdr = 0;
633df8bae1dSRodney W. Grimes 		}
634df8bae1dSRodney W. Grimes 		n->m_len = min(len, m->m_len - off);
635df8bae1dSRodney W. Grimes 		if (m->m_flags & M_EXT) {
636df8bae1dSRodney W. Grimes 			n->m_data = m->m_data + off;
63756a4e45aSAndre Oppermann 			mb_dupcl(n, m);
638df8bae1dSRodney W. Grimes 		} else
639df8bae1dSRodney W. Grimes 			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
640bd395ae8SBosko Milekic 			    (u_int)n->m_len);
641df8bae1dSRodney W. Grimes 		if (len != M_COPYALL)
642df8bae1dSRodney W. Grimes 			len -= n->m_len;
643df8bae1dSRodney W. Grimes 		off = 0;
644df8bae1dSRodney W. Grimes 		m = m->m_next;
645df8bae1dSRodney W. Grimes 		np = &n->m_next;
646df8bae1dSRodney W. Grimes 	}
64708442f8aSBosko Milekic 
648df8bae1dSRodney W. Grimes 	return (top);
649df8bae1dSRodney W. Grimes nospace:
650df8bae1dSRodney W. Grimes 	m_freem(top);
651122a814aSBosko Milekic 	return (NULL);
652df8bae1dSRodney W. Grimes }
653df8bae1dSRodney W. Grimes 
654df8bae1dSRodney W. Grimes /*
6554da84431SAndre Oppermann  * Returns mbuf chain with new head for the prepending case.
6564da84431SAndre Oppermann  * Copies from mbuf (chain) n from off for len to mbuf (chain) m
6574da84431SAndre Oppermann  * either prepending or appending the data.
6584da84431SAndre Oppermann  * The resulting mbuf (chain) m is fully writeable.
6594da84431SAndre Oppermann  * m is destination (is made writeable)
6604da84431SAndre Oppermann  * n is source, off is offset in source, len is len from offset
6614da84431SAndre Oppermann  * dir, 0 append, 1 prepend
6624da84431SAndre Oppermann  * how, wait or nowait
6634da84431SAndre Oppermann  */
6644da84431SAndre Oppermann 
6654da84431SAndre Oppermann static int
6664da84431SAndre Oppermann m_bcopyxxx(void *s, void *t, u_int len)
6674da84431SAndre Oppermann {
6684da84431SAndre Oppermann 	bcopy(s, t, (size_t)len);
6694da84431SAndre Oppermann 	return 0;
6704da84431SAndre Oppermann }
6714da84431SAndre Oppermann 
6724da84431SAndre Oppermann struct mbuf *
6734da84431SAndre Oppermann m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
6744da84431SAndre Oppermann     int prep, int how)
6754da84431SAndre Oppermann {
67656a4e45aSAndre Oppermann 	struct mbuf *mm, *x, *z, *prev = NULL;
6774da84431SAndre Oppermann 	caddr_t p;
67856a4e45aSAndre Oppermann 	int i, nlen = 0;
6794da84431SAndre Oppermann 	caddr_t buf[MLEN];
6804da84431SAndre Oppermann 
6814da84431SAndre Oppermann 	KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
6824da84431SAndre Oppermann 	KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
6834da84431SAndre Oppermann 	KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
6844da84431SAndre Oppermann 	KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
6854da84431SAndre Oppermann 
68656a4e45aSAndre Oppermann 	mm = m;
68756a4e45aSAndre Oppermann 	if (!prep) {
68856a4e45aSAndre Oppermann 		while(mm->m_next) {
68956a4e45aSAndre Oppermann 			prev = mm;
69056a4e45aSAndre Oppermann 			mm = mm->m_next;
6914da84431SAndre Oppermann 		}
6924da84431SAndre Oppermann 	}
6934da84431SAndre Oppermann 	for (z = n; z != NULL; z = z->m_next)
6944da84431SAndre Oppermann 		nlen += z->m_len;
6954da84431SAndre Oppermann 	if (len == M_COPYALL)
6964da84431SAndre Oppermann 		len = nlen - off;
6974da84431SAndre Oppermann 	if (off + len > nlen || len < 1)
6984da84431SAndre Oppermann 		return NULL;
6994da84431SAndre Oppermann 
70056a4e45aSAndre Oppermann 	if (!M_WRITABLE(mm)) {
70156a4e45aSAndre Oppermann 		/* XXX: Use proper m_xxx function instead. */
70256a4e45aSAndre Oppermann 		x = m_getcl(how, MT_DATA, mm->m_flags);
70356a4e45aSAndre Oppermann 		if (x == NULL)
70456a4e45aSAndre Oppermann 			return NULL;
70556a4e45aSAndre Oppermann 		bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
70656a4e45aSAndre Oppermann 		p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
70756a4e45aSAndre Oppermann 		x->m_data = p;
70856a4e45aSAndre Oppermann 		mm->m_next = NULL;
70956a4e45aSAndre Oppermann 		if (mm != m)
71056a4e45aSAndre Oppermann 			prev->m_next = x;
71156a4e45aSAndre Oppermann 		m_free(mm);
71256a4e45aSAndre Oppermann 		mm = x;
71356a4e45aSAndre Oppermann 	}
71456a4e45aSAndre Oppermann 
7154da84431SAndre Oppermann 	/*
7164da84431SAndre Oppermann 	 * Append/prepend the data.  Allocating mbufs as necessary.
7174da84431SAndre Oppermann 	 */
7184da84431SAndre Oppermann 	/* Shortcut if enough free space in first/last mbuf. */
7194da84431SAndre Oppermann 	if (!prep && M_TRAILINGSPACE(mm) >= len) {
7204da84431SAndre Oppermann 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
7214da84431SAndre Oppermann 			 mm->m_len);
7224da84431SAndre Oppermann 		mm->m_len += len;
7234da84431SAndre Oppermann 		mm->m_pkthdr.len += len;
7244da84431SAndre Oppermann 		return m;
7254da84431SAndre Oppermann 	}
7264da84431SAndre Oppermann 	if (prep && M_LEADINGSPACE(mm) >= len) {
7274da84431SAndre Oppermann 		mm->m_data = mtod(mm, caddr_t) - len;
7284da84431SAndre Oppermann 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
7294da84431SAndre Oppermann 		mm->m_len += len;
7304da84431SAndre Oppermann 		mm->m_pkthdr.len += len;
7314da84431SAndre Oppermann 		return mm;
7324da84431SAndre Oppermann 	}
7334da84431SAndre Oppermann 
7344da84431SAndre Oppermann 	/* Expand first/last mbuf to cluster if possible. */
7354da84431SAndre Oppermann 	if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
7364da84431SAndre Oppermann 		bcopy(mm->m_data, &buf, mm->m_len);
7374da84431SAndre Oppermann 		m_clget(mm, how);
7384da84431SAndre Oppermann 		if (!(mm->m_flags & M_EXT))
7394da84431SAndre Oppermann 			return NULL;
7404da84431SAndre Oppermann 		bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
7414da84431SAndre Oppermann 		mm->m_data = mm->m_ext.ext_buf;
7424da84431SAndre Oppermann 	}
7434da84431SAndre Oppermann 	if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
7444da84431SAndre Oppermann 		bcopy(mm->m_data, &buf, mm->m_len);
7454da84431SAndre Oppermann 		m_clget(mm, how);
7464da84431SAndre Oppermann 		if (!(mm->m_flags & M_EXT))
7474da84431SAndre Oppermann 			return NULL;
7484da84431SAndre Oppermann 		bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
7494da84431SAndre Oppermann 		    mm->m_ext.ext_size - mm->m_len, mm->m_len);
7504da84431SAndre Oppermann 		mm->m_data = (caddr_t)mm->m_ext.ext_buf +
7514da84431SAndre Oppermann 		    mm->m_ext.ext_size - mm->m_len;
7524da84431SAndre Oppermann 	}
7534da84431SAndre Oppermann 
7544da84431SAndre Oppermann 	/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
7554da84431SAndre Oppermann 	if (!prep && len > M_TRAILINGSPACE(mm)) {
7564da84431SAndre Oppermann 		if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
7574da84431SAndre Oppermann 			return NULL;
7584da84431SAndre Oppermann 	}
7594da84431SAndre Oppermann 	if (prep && len > M_LEADINGSPACE(mm)) {
7604da84431SAndre Oppermann 		if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
7614da84431SAndre Oppermann 			return NULL;
7624da84431SAndre Oppermann 		i = 0;
7634da84431SAndre Oppermann 		for (x = z; x != NULL; x = x->m_next) {
7644da84431SAndre Oppermann 			i += x->m_flags & M_EXT ? x->m_ext.ext_size :
7654da84431SAndre Oppermann 			    (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
7664da84431SAndre Oppermann 			if (!x->m_next)
7674da84431SAndre Oppermann 				break;
7684da84431SAndre Oppermann 		}
7694da84431SAndre Oppermann 		z->m_data += i - len;
7704da84431SAndre Oppermann 		m_move_pkthdr(mm, z);
7714da84431SAndre Oppermann 		x->m_next = mm;
7724da84431SAndre Oppermann 		mm = z;
7734da84431SAndre Oppermann 	}
7744da84431SAndre Oppermann 
7754da84431SAndre Oppermann 	/* Seek to start position in source mbuf. Optimization for long chains. */
7764da84431SAndre Oppermann 	while (off > 0) {
7774da84431SAndre Oppermann 		if (off < n->m_len)
7784da84431SAndre Oppermann 			break;
7794da84431SAndre Oppermann 		off -= n->m_len;
7804da84431SAndre Oppermann 		n = n->m_next;
7814da84431SAndre Oppermann 	}
7824da84431SAndre Oppermann 
7834da84431SAndre Oppermann 	/* Copy data into target mbuf. */
7844da84431SAndre Oppermann 	z = mm;
7854da84431SAndre Oppermann 	while (len > 0) {
7864da84431SAndre Oppermann 		KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
7874da84431SAndre Oppermann 		i = M_TRAILINGSPACE(z);
7884da84431SAndre Oppermann 		m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
7894da84431SAndre Oppermann 		z->m_len += i;
7904da84431SAndre Oppermann 		/* fixup pkthdr.len if necessary */
7914da84431SAndre Oppermann 		if ((prep ? mm : m)->m_flags & M_PKTHDR)
7924da84431SAndre Oppermann 			(prep ? mm : m)->m_pkthdr.len += i;
7934da84431SAndre Oppermann 		off += i;
7944da84431SAndre Oppermann 		len -= i;
7954da84431SAndre Oppermann 		z = z->m_next;
7964da84431SAndre Oppermann 	}
7974da84431SAndre Oppermann 	return (prep ? mm : m);
7984da84431SAndre Oppermann }
7994da84431SAndre Oppermann 
8004da84431SAndre Oppermann /*
8016a06dea0SGarrett Wollman  * Copy an entire packet, including header (which must be present).
8026a06dea0SGarrett Wollman  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
8031c38f2eaSArchie Cobbs  * Note that the copy is read-only, because clusters are not copied,
8041c38f2eaSArchie Cobbs  * only their reference counts are incremented.
8055fe86675SLuigi Rizzo  * Preserve alignment of the first mbuf so if the creator has left
8065fe86675SLuigi Rizzo  * some room at the beginning (e.g. for inserting protocol headers)
8075fe86675SLuigi Rizzo  * the copies still have the room available.
8086a06dea0SGarrett Wollman  */
8096a06dea0SGarrett Wollman struct mbuf *
810122a814aSBosko Milekic m_copypacket(struct mbuf *m, int how)
8116a06dea0SGarrett Wollman {
8126a06dea0SGarrett Wollman 	struct mbuf *top, *n, *o;
8136a06dea0SGarrett Wollman 
814063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(how);
815c95be8b5SGleb Smirnoff 	n = m_get(how, m->m_type);
8166a06dea0SGarrett Wollman 	top = n;
817122a814aSBosko Milekic 	if (n == NULL)
8186a06dea0SGarrett Wollman 		goto nospace;
8196a06dea0SGarrett Wollman 
8209967cafcSSam Leffler 	if (!m_dup_pkthdr(n, m, how))
8219967cafcSSam Leffler 		goto nospace;
8226a06dea0SGarrett Wollman 	n->m_len = m->m_len;
8236a06dea0SGarrett Wollman 	if (m->m_flags & M_EXT) {
8246a06dea0SGarrett Wollman 		n->m_data = m->m_data;
82556a4e45aSAndre Oppermann 		mb_dupcl(n, m);
8266a06dea0SGarrett Wollman 	} else {
8275fe86675SLuigi Rizzo 		n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
8286a06dea0SGarrett Wollman 		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
8296a06dea0SGarrett Wollman 	}
8306a06dea0SGarrett Wollman 
8316a06dea0SGarrett Wollman 	m = m->m_next;
8326a06dea0SGarrett Wollman 	while (m) {
833c95be8b5SGleb Smirnoff 		o = m_get(how, m->m_type);
834122a814aSBosko Milekic 		if (o == NULL)
8356a06dea0SGarrett Wollman 			goto nospace;
8366a06dea0SGarrett Wollman 
8376a06dea0SGarrett Wollman 		n->m_next = o;
8386a06dea0SGarrett Wollman 		n = n->m_next;
8396a06dea0SGarrett Wollman 
8406a06dea0SGarrett Wollman 		n->m_len = m->m_len;
8416a06dea0SGarrett Wollman 		if (m->m_flags & M_EXT) {
8426a06dea0SGarrett Wollman 			n->m_data = m->m_data;
84356a4e45aSAndre Oppermann 			mb_dupcl(n, m);
8446a06dea0SGarrett Wollman 		} else {
8456a06dea0SGarrett Wollman 			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
8466a06dea0SGarrett Wollman 		}
8476a06dea0SGarrett Wollman 
8486a06dea0SGarrett Wollman 		m = m->m_next;
8496a06dea0SGarrett Wollman 	}
8506a06dea0SGarrett Wollman 	return top;
8516a06dea0SGarrett Wollman nospace:
8526a06dea0SGarrett Wollman 	m_freem(top);
853122a814aSBosko Milekic 	return (NULL);
8546a06dea0SGarrett Wollman }
8556a06dea0SGarrett Wollman 
8566a06dea0SGarrett Wollman /*
857df8bae1dSRodney W. Grimes  * Copy data from an mbuf chain starting "off" bytes from the beginning,
858df8bae1dSRodney W. Grimes  * continuing for "len" bytes, into the indicated buffer.
859df8bae1dSRodney W. Grimes  */
86026f9a767SRodney W. Grimes void
861a8cfc0eeSJulian Elischer m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
862df8bae1dSRodney W. Grimes {
863bd395ae8SBosko Milekic 	u_int count;
864df8bae1dSRodney W. Grimes 
865e0a653ddSAlfred Perlstein 	KASSERT(off >= 0, ("m_copydata, negative off %d", off));
866e0a653ddSAlfred Perlstein 	KASSERT(len >= 0, ("m_copydata, negative len %d", len));
867df8bae1dSRodney W. Grimes 	while (off > 0) {
868e0a653ddSAlfred Perlstein 		KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
869df8bae1dSRodney W. Grimes 		if (off < m->m_len)
870df8bae1dSRodney W. Grimes 			break;
871df8bae1dSRodney W. Grimes 		off -= m->m_len;
872df8bae1dSRodney W. Grimes 		m = m->m_next;
873df8bae1dSRodney W. Grimes 	}
874df8bae1dSRodney W. Grimes 	while (len > 0) {
875e0a653ddSAlfred Perlstein 		KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
876df8bae1dSRodney W. Grimes 		count = min(m->m_len - off, len);
877df8bae1dSRodney W. Grimes 		bcopy(mtod(m, caddr_t) + off, cp, count);
878df8bae1dSRodney W. Grimes 		len -= count;
879df8bae1dSRodney W. Grimes 		cp += count;
880df8bae1dSRodney W. Grimes 		off = 0;
881df8bae1dSRodney W. Grimes 		m = m->m_next;
882df8bae1dSRodney W. Grimes 	}
883df8bae1dSRodney W. Grimes }
884df8bae1dSRodney W. Grimes 
885df8bae1dSRodney W. Grimes /*
8861c38f2eaSArchie Cobbs  * Copy a packet header mbuf chain into a completely new chain, including
8871c38f2eaSArchie Cobbs  * copying any mbuf clusters.  Use this instead of m_copypacket() when
8881c38f2eaSArchie Cobbs  * you need a writable copy of an mbuf chain.
8891c38f2eaSArchie Cobbs  */
8901c38f2eaSArchie Cobbs struct mbuf *
891122a814aSBosko Milekic m_dup(struct mbuf *m, int how)
8921c38f2eaSArchie Cobbs {
8931c38f2eaSArchie Cobbs 	struct mbuf **p, *top = NULL;
8941c38f2eaSArchie Cobbs 	int remain, moff, nsize;
8951c38f2eaSArchie Cobbs 
896063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(how);
8971c38f2eaSArchie Cobbs 	/* Sanity check */
8981c38f2eaSArchie Cobbs 	if (m == NULL)
899122a814aSBosko Milekic 		return (NULL);
900fe584538SDag-Erling Smørgrav 	M_ASSERTPKTHDR(m);
9011c38f2eaSArchie Cobbs 
9021c38f2eaSArchie Cobbs 	/* While there's more data, get a new mbuf, tack it on, and fill it */
9031c38f2eaSArchie Cobbs 	remain = m->m_pkthdr.len;
9041c38f2eaSArchie Cobbs 	moff = 0;
9051c38f2eaSArchie Cobbs 	p = &top;
9061c38f2eaSArchie Cobbs 	while (remain > 0 || top == NULL) {	/* allow m->m_pkthdr.len == 0 */
9071c38f2eaSArchie Cobbs 		struct mbuf *n;
9081c38f2eaSArchie Cobbs 
9091c38f2eaSArchie Cobbs 		/* Get the next new mbuf */
910099a0e58SBosko Milekic 		if (remain >= MINCLSIZE) {
911099a0e58SBosko Milekic 			n = m_getcl(how, m->m_type, 0);
912099a0e58SBosko Milekic 			nsize = MCLBYTES;
913099a0e58SBosko Milekic 		} else {
914099a0e58SBosko Milekic 			n = m_get(how, m->m_type);
915099a0e58SBosko Milekic 			nsize = MLEN;
916099a0e58SBosko Milekic 		}
9171c38f2eaSArchie Cobbs 		if (n == NULL)
9181c38f2eaSArchie Cobbs 			goto nospace;
919099a0e58SBosko Milekic 
920099a0e58SBosko Milekic 		if (top == NULL) {		/* First one, must be PKTHDR */
921099a0e58SBosko Milekic 			if (!m_dup_pkthdr(n, m, how)) {
922099a0e58SBosko Milekic 				m_free(n);
9231c38f2eaSArchie Cobbs 				goto nospace;
9241c38f2eaSArchie Cobbs 			}
92563e6f390SEd Maste 			if ((n->m_flags & M_EXT) == 0)
926099a0e58SBosko Milekic 				nsize = MHLEN;
9271c38f2eaSArchie Cobbs 		}
9281c38f2eaSArchie Cobbs 		n->m_len = 0;
9291c38f2eaSArchie Cobbs 
9301c38f2eaSArchie Cobbs 		/* Link it into the new chain */
9311c38f2eaSArchie Cobbs 		*p = n;
9321c38f2eaSArchie Cobbs 		p = &n->m_next;
9331c38f2eaSArchie Cobbs 
9341c38f2eaSArchie Cobbs 		/* Copy data from original mbuf(s) into new mbuf */
9351c38f2eaSArchie Cobbs 		while (n->m_len < nsize && m != NULL) {
9361c38f2eaSArchie Cobbs 			int chunk = min(nsize - n->m_len, m->m_len - moff);
9371c38f2eaSArchie Cobbs 
9381c38f2eaSArchie Cobbs 			bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
9391c38f2eaSArchie Cobbs 			moff += chunk;
9401c38f2eaSArchie Cobbs 			n->m_len += chunk;
9411c38f2eaSArchie Cobbs 			remain -= chunk;
9421c38f2eaSArchie Cobbs 			if (moff == m->m_len) {
9431c38f2eaSArchie Cobbs 				m = m->m_next;
9441c38f2eaSArchie Cobbs 				moff = 0;
9451c38f2eaSArchie Cobbs 			}
9461c38f2eaSArchie Cobbs 		}
9471c38f2eaSArchie Cobbs 
9481c38f2eaSArchie Cobbs 		/* Check correct total mbuf length */
9491c38f2eaSArchie Cobbs 		KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
950a48740b6SDavid E. O'Brien 		    	("%s: bogus m_pkthdr.len", __func__));
9511c38f2eaSArchie Cobbs 	}
9521c38f2eaSArchie Cobbs 	return (top);
9531c38f2eaSArchie Cobbs 
9541c38f2eaSArchie Cobbs nospace:
9551c38f2eaSArchie Cobbs 	m_freem(top);
956122a814aSBosko Milekic 	return (NULL);
9571c38f2eaSArchie Cobbs }
9581c38f2eaSArchie Cobbs 
9591c38f2eaSArchie Cobbs /*
960df8bae1dSRodney W. Grimes  * Concatenate mbuf chain n to m.
961df8bae1dSRodney W. Grimes  * Both chains must be of the same type (e.g. MT_DATA).
962df8bae1dSRodney W. Grimes  * Any m_pkthdr is not updated.
963df8bae1dSRodney W. Grimes  */
96426f9a767SRodney W. Grimes void
965122a814aSBosko Milekic m_cat(struct mbuf *m, struct mbuf *n)
966df8bae1dSRodney W. Grimes {
967df8bae1dSRodney W. Grimes 	while (m->m_next)
968df8bae1dSRodney W. Grimes 		m = m->m_next;
969df8bae1dSRodney W. Grimes 	while (n) {
97014d7c5b1SAndre Oppermann 		if (!M_WRITABLE(m) ||
97114d7c5b1SAndre Oppermann 		    M_TRAILINGSPACE(m) < n->m_len) {
972df8bae1dSRodney W. Grimes 			/* just join the two chains */
973df8bae1dSRodney W. Grimes 			m->m_next = n;
974df8bae1dSRodney W. Grimes 			return;
975df8bae1dSRodney W. Grimes 		}
976df8bae1dSRodney W. Grimes 		/* splat the data from one into the other */
977df8bae1dSRodney W. Grimes 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
978df8bae1dSRodney W. Grimes 		    (u_int)n->m_len);
979df8bae1dSRodney W. Grimes 		m->m_len += n->m_len;
980df8bae1dSRodney W. Grimes 		n = m_free(n);
981df8bae1dSRodney W. Grimes 	}
982df8bae1dSRodney W. Grimes }
983df8bae1dSRodney W. Grimes 
9841967edbaSGleb Smirnoff /*
9851967edbaSGleb Smirnoff  * Concatenate two pkthdr mbuf chains.
9861967edbaSGleb Smirnoff  */
9871967edbaSGleb Smirnoff void
9881967edbaSGleb Smirnoff m_catpkt(struct mbuf *m, struct mbuf *n)
9891967edbaSGleb Smirnoff {
9901967edbaSGleb Smirnoff 
9911967edbaSGleb Smirnoff 	M_ASSERTPKTHDR(m);
9921967edbaSGleb Smirnoff 	M_ASSERTPKTHDR(n);
9931967edbaSGleb Smirnoff 
9941967edbaSGleb Smirnoff 	m->m_pkthdr.len += n->m_pkthdr.len;
995651e4e6aSGleb Smirnoff 	m_demote(n, 1, 0);
9961967edbaSGleb Smirnoff 
9971967edbaSGleb Smirnoff 	m_cat(m, n);
9981967edbaSGleb Smirnoff }
9991967edbaSGleb Smirnoff 
100026f9a767SRodney W. Grimes void
1001122a814aSBosko Milekic m_adj(struct mbuf *mp, int req_len)
1002df8bae1dSRodney W. Grimes {
1003122a814aSBosko Milekic 	int len = req_len;
1004122a814aSBosko Milekic 	struct mbuf *m;
1005122a814aSBosko Milekic 	int count;
1006df8bae1dSRodney W. Grimes 
1007df8bae1dSRodney W. Grimes 	if ((m = mp) == NULL)
1008df8bae1dSRodney W. Grimes 		return;
1009df8bae1dSRodney W. Grimes 	if (len >= 0) {
1010df8bae1dSRodney W. Grimes 		/*
1011df8bae1dSRodney W. Grimes 		 * Trim from head.
1012df8bae1dSRodney W. Grimes 		 */
1013df8bae1dSRodney W. Grimes 		while (m != NULL && len > 0) {
1014df8bae1dSRodney W. Grimes 			if (m->m_len <= len) {
1015df8bae1dSRodney W. Grimes 				len -= m->m_len;
1016df8bae1dSRodney W. Grimes 				m->m_len = 0;
1017df8bae1dSRodney W. Grimes 				m = m->m_next;
1018df8bae1dSRodney W. Grimes 			} else {
1019df8bae1dSRodney W. Grimes 				m->m_len -= len;
1020df8bae1dSRodney W. Grimes 				m->m_data += len;
1021df8bae1dSRodney W. Grimes 				len = 0;
1022df8bae1dSRodney W. Grimes 			}
1023df8bae1dSRodney W. Grimes 		}
1024df8bae1dSRodney W. Grimes 		if (mp->m_flags & M_PKTHDR)
1025a83baab6SMarko Zec 			mp->m_pkthdr.len -= (req_len - len);
1026df8bae1dSRodney W. Grimes 	} else {
1027df8bae1dSRodney W. Grimes 		/*
1028df8bae1dSRodney W. Grimes 		 * Trim from tail.  Scan the mbuf chain,
1029df8bae1dSRodney W. Grimes 		 * calculating its length and finding the last mbuf.
1030df8bae1dSRodney W. Grimes 		 * If the adjustment only affects this mbuf, then just
1031df8bae1dSRodney W. Grimes 		 * adjust and return.  Otherwise, rescan and truncate
1032df8bae1dSRodney W. Grimes 		 * after the remaining size.
1033df8bae1dSRodney W. Grimes 		 */
1034df8bae1dSRodney W. Grimes 		len = -len;
1035df8bae1dSRodney W. Grimes 		count = 0;
1036df8bae1dSRodney W. Grimes 		for (;;) {
1037df8bae1dSRodney W. Grimes 			count += m->m_len;
1038df8bae1dSRodney W. Grimes 			if (m->m_next == (struct mbuf *)0)
1039df8bae1dSRodney W. Grimes 				break;
1040df8bae1dSRodney W. Grimes 			m = m->m_next;
1041df8bae1dSRodney W. Grimes 		}
1042df8bae1dSRodney W. Grimes 		if (m->m_len >= len) {
1043df8bae1dSRodney W. Grimes 			m->m_len -= len;
1044df8bae1dSRodney W. Grimes 			if (mp->m_flags & M_PKTHDR)
1045df8bae1dSRodney W. Grimes 				mp->m_pkthdr.len -= len;
1046df8bae1dSRodney W. Grimes 			return;
1047df8bae1dSRodney W. Grimes 		}
1048df8bae1dSRodney W. Grimes 		count -= len;
1049df8bae1dSRodney W. Grimes 		if (count < 0)
1050df8bae1dSRodney W. Grimes 			count = 0;
1051df8bae1dSRodney W. Grimes 		/*
1052df8bae1dSRodney W. Grimes 		 * Correct length for chain is "count".
1053df8bae1dSRodney W. Grimes 		 * Find the mbuf with last data, adjust its length,
1054df8bae1dSRodney W. Grimes 		 * and toss data from remaining mbufs on chain.
1055df8bae1dSRodney W. Grimes 		 */
1056df8bae1dSRodney W. Grimes 		m = mp;
1057df8bae1dSRodney W. Grimes 		if (m->m_flags & M_PKTHDR)
1058df8bae1dSRodney W. Grimes 			m->m_pkthdr.len = count;
1059df8bae1dSRodney W. Grimes 		for (; m; m = m->m_next) {
1060df8bae1dSRodney W. Grimes 			if (m->m_len >= count) {
1061df8bae1dSRodney W. Grimes 				m->m_len = count;
106259d8b310SSam Leffler 				if (m->m_next != NULL) {
106359d8b310SSam Leffler 					m_freem(m->m_next);
106459d8b310SSam Leffler 					m->m_next = NULL;
106559d8b310SSam Leffler 				}
1066df8bae1dSRodney W. Grimes 				break;
1067df8bae1dSRodney W. Grimes 			}
1068df8bae1dSRodney W. Grimes 			count -= m->m_len;
1069df8bae1dSRodney W. Grimes 		}
1070df8bae1dSRodney W. Grimes 	}
1071df8bae1dSRodney W. Grimes }
1072df8bae1dSRodney W. Grimes 
1073df8bae1dSRodney W. Grimes /*
1074df8bae1dSRodney W. Grimes  * Rearange an mbuf chain so that len bytes are contiguous
1075a2c36a02SKevin Lo  * and in the data area of an mbuf (so that mtod will work
1076a2c36a02SKevin Lo  * for a structure of size len).  Returns the resulting
1077df8bae1dSRodney W. Grimes  * mbuf chain on success, frees it and returns null on failure.
1078df8bae1dSRodney W. Grimes  * If there is room, it will add up to max_protohdr-len extra bytes to the
1079df8bae1dSRodney W. Grimes  * contiguous region in an attempt to avoid being called next time.
1080df8bae1dSRodney W. Grimes  */
1081df8bae1dSRodney W. Grimes struct mbuf *
1082122a814aSBosko Milekic m_pullup(struct mbuf *n, int len)
1083df8bae1dSRodney W. Grimes {
1084122a814aSBosko Milekic 	struct mbuf *m;
1085122a814aSBosko Milekic 	int count;
1086df8bae1dSRodney W. Grimes 	int space;
1087df8bae1dSRodney W. Grimes 
1088df8bae1dSRodney W. Grimes 	/*
1089df8bae1dSRodney W. Grimes 	 * If first mbuf has no cluster, and has room for len bytes
1090df8bae1dSRodney W. Grimes 	 * without shifting current data, pullup into it,
1091df8bae1dSRodney W. Grimes 	 * otherwise allocate a new mbuf to prepend to the chain.
1092df8bae1dSRodney W. Grimes 	 */
1093df8bae1dSRodney W. Grimes 	if ((n->m_flags & M_EXT) == 0 &&
1094df8bae1dSRodney W. Grimes 	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1095df8bae1dSRodney W. Grimes 		if (n->m_len >= len)
1096df8bae1dSRodney W. Grimes 			return (n);
1097df8bae1dSRodney W. Grimes 		m = n;
1098df8bae1dSRodney W. Grimes 		n = n->m_next;
1099df8bae1dSRodney W. Grimes 		len -= m->m_len;
1100df8bae1dSRodney W. Grimes 	} else {
1101df8bae1dSRodney W. Grimes 		if (len > MHLEN)
1102df8bae1dSRodney W. Grimes 			goto bad;
1103c95be8b5SGleb Smirnoff 		m = m_get(M_NOWAIT, n->m_type);
1104122a814aSBosko Milekic 		if (m == NULL)
1105df8bae1dSRodney W. Grimes 			goto bad;
11069967cafcSSam Leffler 		if (n->m_flags & M_PKTHDR)
1107c95be8b5SGleb Smirnoff 			m_move_pkthdr(m, n);
1108df8bae1dSRodney W. Grimes 	}
1109df8bae1dSRodney W. Grimes 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1110df8bae1dSRodney W. Grimes 	do {
1111df8bae1dSRodney W. Grimes 		count = min(min(max(len, max_protohdr), space), n->m_len);
1112df8bae1dSRodney W. Grimes 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1113bd395ae8SBosko Milekic 		  (u_int)count);
1114df8bae1dSRodney W. Grimes 		len -= count;
1115df8bae1dSRodney W. Grimes 		m->m_len += count;
1116df8bae1dSRodney W. Grimes 		n->m_len -= count;
1117df8bae1dSRodney W. Grimes 		space -= count;
1118df8bae1dSRodney W. Grimes 		if (n->m_len)
1119df8bae1dSRodney W. Grimes 			n->m_data += count;
1120df8bae1dSRodney W. Grimes 		else
1121df8bae1dSRodney W. Grimes 			n = m_free(n);
1122df8bae1dSRodney W. Grimes 	} while (len > 0 && n);
1123df8bae1dSRodney W. Grimes 	if (len > 0) {
1124df8bae1dSRodney W. Grimes 		(void) m_free(m);
1125df8bae1dSRodney W. Grimes 		goto bad;
1126df8bae1dSRodney W. Grimes 	}
1127df8bae1dSRodney W. Grimes 	m->m_next = n;
1128df8bae1dSRodney W. Grimes 	return (m);
1129df8bae1dSRodney W. Grimes bad:
1130df8bae1dSRodney W. Grimes 	m_freem(n);
1131122a814aSBosko Milekic 	return (NULL);
1132df8bae1dSRodney W. Grimes }
1133df8bae1dSRodney W. Grimes 
1134df8bae1dSRodney W. Grimes /*
11357ac139a9SJohn-Mark Gurney  * Like m_pullup(), except a new mbuf is always allocated, and we allow
11367ac139a9SJohn-Mark Gurney  * the amount of empty space before the data in the new mbuf to be specified
11377ac139a9SJohn-Mark Gurney  * (in the event that the caller expects to prepend later).
11387ac139a9SJohn-Mark Gurney  */
11397ac139a9SJohn-Mark Gurney int MSFail;
11407ac139a9SJohn-Mark Gurney 
11417ac139a9SJohn-Mark Gurney struct mbuf *
11427ac139a9SJohn-Mark Gurney m_copyup(struct mbuf *n, int len, int dstoff)
11437ac139a9SJohn-Mark Gurney {
11447ac139a9SJohn-Mark Gurney 	struct mbuf *m;
11457ac139a9SJohn-Mark Gurney 	int count, space;
11467ac139a9SJohn-Mark Gurney 
11477ac139a9SJohn-Mark Gurney 	if (len > (MHLEN - dstoff))
11487ac139a9SJohn-Mark Gurney 		goto bad;
1149c95be8b5SGleb Smirnoff 	m = m_get(M_NOWAIT, n->m_type);
11507ac139a9SJohn-Mark Gurney 	if (m == NULL)
11517ac139a9SJohn-Mark Gurney 		goto bad;
11527ac139a9SJohn-Mark Gurney 	if (n->m_flags & M_PKTHDR)
1153c95be8b5SGleb Smirnoff 		m_move_pkthdr(m, n);
11547ac139a9SJohn-Mark Gurney 	m->m_data += dstoff;
11557ac139a9SJohn-Mark Gurney 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
11567ac139a9SJohn-Mark Gurney 	do {
11577ac139a9SJohn-Mark Gurney 		count = min(min(max(len, max_protohdr), space), n->m_len);
11587ac139a9SJohn-Mark Gurney 		memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
11597ac139a9SJohn-Mark Gurney 		    (unsigned)count);
11607ac139a9SJohn-Mark Gurney 		len -= count;
11617ac139a9SJohn-Mark Gurney 		m->m_len += count;
11627ac139a9SJohn-Mark Gurney 		n->m_len -= count;
11637ac139a9SJohn-Mark Gurney 		space -= count;
11647ac139a9SJohn-Mark Gurney 		if (n->m_len)
11657ac139a9SJohn-Mark Gurney 			n->m_data += count;
11667ac139a9SJohn-Mark Gurney 		else
11677ac139a9SJohn-Mark Gurney 			n = m_free(n);
11687ac139a9SJohn-Mark Gurney 	} while (len > 0 && n);
11697ac139a9SJohn-Mark Gurney 	if (len > 0) {
11707ac139a9SJohn-Mark Gurney 		(void) m_free(m);
11717ac139a9SJohn-Mark Gurney 		goto bad;
11727ac139a9SJohn-Mark Gurney 	}
11737ac139a9SJohn-Mark Gurney 	m->m_next = n;
11747ac139a9SJohn-Mark Gurney 	return (m);
11757ac139a9SJohn-Mark Gurney  bad:
11767ac139a9SJohn-Mark Gurney 	m_freem(n);
11777ac139a9SJohn-Mark Gurney 	MSFail++;
11787ac139a9SJohn-Mark Gurney 	return (NULL);
11797ac139a9SJohn-Mark Gurney }
11807ac139a9SJohn-Mark Gurney 
11817ac139a9SJohn-Mark Gurney /*
1182df8bae1dSRodney W. Grimes  * Partition an mbuf chain in two pieces, returning the tail --
1183df8bae1dSRodney W. Grimes  * all but the first len0 bytes.  In case of failure, it returns NULL and
1184df8bae1dSRodney W. Grimes  * attempts to restore the chain to its original state.
118548d183faSArchie Cobbs  *
118648d183faSArchie Cobbs  * Note that the resulting mbufs might be read-only, because the new
118748d183faSArchie Cobbs  * mbuf can end up sharing an mbuf cluster with the original mbuf if
118848d183faSArchie Cobbs  * the "breaking point" happens to lie within a cluster mbuf. Use the
118948d183faSArchie Cobbs  * M_WRITABLE() macro to check for this case.
1190df8bae1dSRodney W. Grimes  */
1191df8bae1dSRodney W. Grimes struct mbuf *
1192122a814aSBosko Milekic m_split(struct mbuf *m0, int len0, int wait)
1193df8bae1dSRodney W. Grimes {
1194122a814aSBosko Milekic 	struct mbuf *m, *n;
1195bd395ae8SBosko Milekic 	u_int len = len0, remain;
1196df8bae1dSRodney W. Grimes 
1197063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(wait);
1198df8bae1dSRodney W. Grimes 	for (m = m0; m && len > m->m_len; m = m->m_next)
1199df8bae1dSRodney W. Grimes 		len -= m->m_len;
1200122a814aSBosko Milekic 	if (m == NULL)
1201122a814aSBosko Milekic 		return (NULL);
1202df8bae1dSRodney W. Grimes 	remain = m->m_len - len;
120321f39848SGleb Smirnoff 	if (m0->m_flags & M_PKTHDR && remain == 0) {
120421f39848SGleb Smirnoff 		n = m_gethdr(wait, m0->m_type);
120577badb18SGleb Smirnoff 		if (n == NULL)
120621f39848SGleb Smirnoff 			return (NULL);
120721f39848SGleb Smirnoff 		n->m_next = m->m_next;
120821f39848SGleb Smirnoff 		m->m_next = NULL;
120921f39848SGleb Smirnoff 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
121021f39848SGleb Smirnoff 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
121121f39848SGleb Smirnoff 		m0->m_pkthdr.len = len0;
121221f39848SGleb Smirnoff 		return (n);
121321f39848SGleb Smirnoff 	} else if (m0->m_flags & M_PKTHDR) {
1214c95be8b5SGleb Smirnoff 		n = m_gethdr(wait, m0->m_type);
1215122a814aSBosko Milekic 		if (n == NULL)
1216122a814aSBosko Milekic 			return (NULL);
1217df8bae1dSRodney W. Grimes 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1218df8bae1dSRodney W. Grimes 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1219df8bae1dSRodney W. Grimes 		m0->m_pkthdr.len = len0;
1220df8bae1dSRodney W. Grimes 		if (m->m_flags & M_EXT)
1221df8bae1dSRodney W. Grimes 			goto extpacket;
1222df8bae1dSRodney W. Grimes 		if (remain > MHLEN) {
1223df8bae1dSRodney W. Grimes 			/* m can't be the lead packet */
1224*ed6a66caSRobert Watson 			M_ALIGN(n, 0);
1225df8bae1dSRodney W. Grimes 			n->m_next = m_split(m, len, wait);
1226122a814aSBosko Milekic 			if (n->m_next == NULL) {
1227df8bae1dSRodney W. Grimes 				(void) m_free(n);
1228122a814aSBosko Milekic 				return (NULL);
122940376987SJeffrey Hsu 			} else {
123040376987SJeffrey Hsu 				n->m_len = 0;
1231df8bae1dSRodney W. Grimes 				return (n);
123240376987SJeffrey Hsu 			}
1233df8bae1dSRodney W. Grimes 		} else
1234*ed6a66caSRobert Watson 			M_ALIGN(n, remain);
1235df8bae1dSRodney W. Grimes 	} else if (remain == 0) {
1236df8bae1dSRodney W. Grimes 		n = m->m_next;
1237122a814aSBosko Milekic 		m->m_next = NULL;
1238df8bae1dSRodney W. Grimes 		return (n);
1239df8bae1dSRodney W. Grimes 	} else {
1240c95be8b5SGleb Smirnoff 		n = m_get(wait, m->m_type);
1241122a814aSBosko Milekic 		if (n == NULL)
1242122a814aSBosko Milekic 			return (NULL);
1243df8bae1dSRodney W. Grimes 		M_ALIGN(n, remain);
1244df8bae1dSRodney W. Grimes 	}
1245df8bae1dSRodney W. Grimes extpacket:
1246df8bae1dSRodney W. Grimes 	if (m->m_flags & M_EXT) {
1247df8bae1dSRodney W. Grimes 		n->m_data = m->m_data + len;
124856a4e45aSAndre Oppermann 		mb_dupcl(n, m);
1249df8bae1dSRodney W. Grimes 	} else {
1250df8bae1dSRodney W. Grimes 		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1251df8bae1dSRodney W. Grimes 	}
1252df8bae1dSRodney W. Grimes 	n->m_len = remain;
1253df8bae1dSRodney W. Grimes 	m->m_len = len;
1254df8bae1dSRodney W. Grimes 	n->m_next = m->m_next;
1255122a814aSBosko Milekic 	m->m_next = NULL;
1256df8bae1dSRodney W. Grimes 	return (n);
1257df8bae1dSRodney W. Grimes }
1258df8bae1dSRodney W. Grimes /*
1259df8bae1dSRodney W. Grimes  * Routine to copy from device local memory into mbufs.
1260f5eece3fSBosko Milekic  * Note that `off' argument is offset into first mbuf of target chain from
1261f5eece3fSBosko Milekic  * which to begin copying the data to.
1262df8bae1dSRodney W. Grimes  */
1263df8bae1dSRodney W. Grimes struct mbuf *
1264f5eece3fSBosko Milekic m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1265122a814aSBosko Milekic     void (*copy)(char *from, caddr_t to, u_int len))
1266df8bae1dSRodney W. Grimes {
1267122a814aSBosko Milekic 	struct mbuf *m;
1268099a0e58SBosko Milekic 	struct mbuf *top = NULL, **mp = &top;
1269f5eece3fSBosko Milekic 	int len;
1270df8bae1dSRodney W. Grimes 
1271f5eece3fSBosko Milekic 	if (off < 0 || off > MHLEN)
1272f5eece3fSBosko Milekic 		return (NULL);
1273f5eece3fSBosko Milekic 
1274df8bae1dSRodney W. Grimes 	while (totlen > 0) {
1275099a0e58SBosko Milekic 		if (top == NULL) {	/* First one, must be PKTHDR */
1276f5eece3fSBosko Milekic 			if (totlen + off >= MINCLSIZE) {
1277eb1b1807SGleb Smirnoff 				m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1278f5eece3fSBosko Milekic 				len = MCLBYTES;
1279df8bae1dSRodney W. Grimes 			} else {
1280eb1b1807SGleb Smirnoff 				m = m_gethdr(M_NOWAIT, MT_DATA);
1281099a0e58SBosko Milekic 				len = MHLEN;
1282099a0e58SBosko Milekic 
1283099a0e58SBosko Milekic 				/* Place initial small packet/header at end of mbuf */
1284099a0e58SBosko Milekic 				if (m && totlen + off + max_linkhdr <= MLEN) {
1285df8bae1dSRodney W. Grimes 					m->m_data += max_linkhdr;
1286f5eece3fSBosko Milekic 					len -= max_linkhdr;
1287df8bae1dSRodney W. Grimes 				}
1288f5eece3fSBosko Milekic 			}
1289099a0e58SBosko Milekic 			if (m == NULL)
1290099a0e58SBosko Milekic 				return NULL;
1291099a0e58SBosko Milekic 			m->m_pkthdr.rcvif = ifp;
1292099a0e58SBosko Milekic 			m->m_pkthdr.len = totlen;
1293099a0e58SBosko Milekic 		} else {
1294099a0e58SBosko Milekic 			if (totlen + off >= MINCLSIZE) {
1295eb1b1807SGleb Smirnoff 				m = m_getcl(M_NOWAIT, MT_DATA, 0);
1296099a0e58SBosko Milekic 				len = MCLBYTES;
1297099a0e58SBosko Milekic 			} else {
1298eb1b1807SGleb Smirnoff 				m = m_get(M_NOWAIT, MT_DATA);
1299099a0e58SBosko Milekic 				len = MLEN;
1300099a0e58SBosko Milekic 			}
1301099a0e58SBosko Milekic 			if (m == NULL) {
1302099a0e58SBosko Milekic 				m_freem(top);
1303099a0e58SBosko Milekic 				return NULL;
1304099a0e58SBosko Milekic 			}
1305099a0e58SBosko Milekic 		}
1306f5eece3fSBosko Milekic 		if (off) {
1307f5eece3fSBosko Milekic 			m->m_data += off;
1308f5eece3fSBosko Milekic 			len -= off;
1309f5eece3fSBosko Milekic 			off = 0;
1310f5eece3fSBosko Milekic 		}
1311f5eece3fSBosko Milekic 		m->m_len = len = min(totlen, len);
1312df8bae1dSRodney W. Grimes 		if (copy)
1313bd395ae8SBosko Milekic 			copy(buf, mtod(m, caddr_t), (u_int)len);
1314df8bae1dSRodney W. Grimes 		else
1315bd395ae8SBosko Milekic 			bcopy(buf, mtod(m, caddr_t), (u_int)len);
1316f5eece3fSBosko Milekic 		buf += len;
1317df8bae1dSRodney W. Grimes 		*mp = m;
1318df8bae1dSRodney W. Grimes 		mp = &m->m_next;
1319df8bae1dSRodney W. Grimes 		totlen -= len;
1320df8bae1dSRodney W. Grimes 	}
1321df8bae1dSRodney W. Grimes 	return (top);
1322df8bae1dSRodney W. Grimes }
1323c5789ba3SPoul-Henning Kamp 
1324c5789ba3SPoul-Henning Kamp /*
1325c5789ba3SPoul-Henning Kamp  * Copy data from a buffer back into the indicated mbuf chain,
1326c5789ba3SPoul-Henning Kamp  * starting "off" bytes from the beginning, extending the mbuf
1327c5789ba3SPoul-Henning Kamp  * chain if necessary.
1328c5789ba3SPoul-Henning Kamp  */
1329c5789ba3SPoul-Henning Kamp void
133024665342SLuigi Rizzo m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
1331c5789ba3SPoul-Henning Kamp {
1332122a814aSBosko Milekic 	int mlen;
1333122a814aSBosko Milekic 	struct mbuf *m = m0, *n;
1334c5789ba3SPoul-Henning Kamp 	int totlen = 0;
1335c5789ba3SPoul-Henning Kamp 
1336122a814aSBosko Milekic 	if (m0 == NULL)
1337c5789ba3SPoul-Henning Kamp 		return;
1338c5789ba3SPoul-Henning Kamp 	while (off > (mlen = m->m_len)) {
1339c5789ba3SPoul-Henning Kamp 		off -= mlen;
1340c5789ba3SPoul-Henning Kamp 		totlen += mlen;
1341122a814aSBosko Milekic 		if (m->m_next == NULL) {
1342eb1b1807SGleb Smirnoff 			n = m_get(M_NOWAIT, m->m_type);
1343122a814aSBosko Milekic 			if (n == NULL)
1344c5789ba3SPoul-Henning Kamp 				goto out;
1345099a0e58SBosko Milekic 			bzero(mtod(n, caddr_t), MLEN);
1346c5789ba3SPoul-Henning Kamp 			n->m_len = min(MLEN, len + off);
1347c5789ba3SPoul-Henning Kamp 			m->m_next = n;
1348c5789ba3SPoul-Henning Kamp 		}
1349c5789ba3SPoul-Henning Kamp 		m = m->m_next;
1350c5789ba3SPoul-Henning Kamp 	}
1351c5789ba3SPoul-Henning Kamp 	while (len > 0) {
1352129c5c81SAlexander Motin 		if (m->m_next == NULL && (len > m->m_len - off)) {
1353129c5c81SAlexander Motin 			m->m_len += min(len - (m->m_len - off),
1354129c5c81SAlexander Motin 			    M_TRAILINGSPACE(m));
1355129c5c81SAlexander Motin 		}
1356c5789ba3SPoul-Henning Kamp 		mlen = min (m->m_len - off, len);
1357bd395ae8SBosko Milekic 		bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
1358c5789ba3SPoul-Henning Kamp 		cp += mlen;
1359c5789ba3SPoul-Henning Kamp 		len -= mlen;
1360c5789ba3SPoul-Henning Kamp 		mlen += off;
1361c5789ba3SPoul-Henning Kamp 		off = 0;
1362c5789ba3SPoul-Henning Kamp 		totlen += mlen;
1363c5789ba3SPoul-Henning Kamp 		if (len == 0)
1364c5789ba3SPoul-Henning Kamp 			break;
1365122a814aSBosko Milekic 		if (m->m_next == NULL) {
1366eb1b1807SGleb Smirnoff 			n = m_get(M_NOWAIT, m->m_type);
1367122a814aSBosko Milekic 			if (n == NULL)
1368c5789ba3SPoul-Henning Kamp 				break;
1369c5789ba3SPoul-Henning Kamp 			n->m_len = min(MLEN, len);
1370c5789ba3SPoul-Henning Kamp 			m->m_next = n;
1371c5789ba3SPoul-Henning Kamp 		}
1372c5789ba3SPoul-Henning Kamp 		m = m->m_next;
1373c5789ba3SPoul-Henning Kamp 	}
1374c5789ba3SPoul-Henning Kamp out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1375c5789ba3SPoul-Henning Kamp 		m->m_pkthdr.len = totlen;
1376c5789ba3SPoul-Henning Kamp }
1377ce4a64f7SPoul-Henning Kamp 
137837621fd5SBruce M Simpson /*
13794873d175SSam Leffler  * Append the specified data to the indicated mbuf chain,
13804873d175SSam Leffler  * Extend the mbuf chain if the new data does not fit in
13814873d175SSam Leffler  * existing space.
13824873d175SSam Leffler  *
13834873d175SSam Leffler  * Return 1 if able to complete the job; otherwise 0.
13844873d175SSam Leffler  */
13854873d175SSam Leffler int
13864873d175SSam Leffler m_append(struct mbuf *m0, int len, c_caddr_t cp)
13874873d175SSam Leffler {
13884873d175SSam Leffler 	struct mbuf *m, *n;
13894873d175SSam Leffler 	int remainder, space;
13904873d175SSam Leffler 
13914873d175SSam Leffler 	for (m = m0; m->m_next != NULL; m = m->m_next)
13924873d175SSam Leffler 		;
13934873d175SSam Leffler 	remainder = len;
13944873d175SSam Leffler 	space = M_TRAILINGSPACE(m);
13954873d175SSam Leffler 	if (space > 0) {
13964873d175SSam Leffler 		/*
13974873d175SSam Leffler 		 * Copy into available space.
13984873d175SSam Leffler 		 */
13994873d175SSam Leffler 		if (space > remainder)
14004873d175SSam Leffler 			space = remainder;
14014873d175SSam Leffler 		bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
14024873d175SSam Leffler 		m->m_len += space;
14034873d175SSam Leffler 		cp += space, remainder -= space;
14044873d175SSam Leffler 	}
14054873d175SSam Leffler 	while (remainder > 0) {
14064873d175SSam Leffler 		/*
14074873d175SSam Leffler 		 * Allocate a new mbuf; could check space
14084873d175SSam Leffler 		 * and allocate a cluster instead.
14094873d175SSam Leffler 		 */
1410eb1b1807SGleb Smirnoff 		n = m_get(M_NOWAIT, m->m_type);
14114873d175SSam Leffler 		if (n == NULL)
14124873d175SSam Leffler 			break;
14134873d175SSam Leffler 		n->m_len = min(MLEN, remainder);
1414a37c415eSSam Leffler 		bcopy(cp, mtod(n, caddr_t), n->m_len);
1415a37c415eSSam Leffler 		cp += n->m_len, remainder -= n->m_len;
14164873d175SSam Leffler 		m->m_next = n;
14174873d175SSam Leffler 		m = n;
14184873d175SSam Leffler 	}
14194873d175SSam Leffler 	if (m0->m_flags & M_PKTHDR)
14204873d175SSam Leffler 		m0->m_pkthdr.len += len - remainder;
14214873d175SSam Leffler 	return (remainder == 0);
14224873d175SSam Leffler }
14234873d175SSam Leffler 
14244873d175SSam Leffler /*
142537621fd5SBruce M Simpson  * Apply function f to the data in an mbuf chain starting "off" bytes from
142637621fd5SBruce M Simpson  * the beginning, continuing for "len" bytes.
142737621fd5SBruce M Simpson  */
142837621fd5SBruce M Simpson int
142937621fd5SBruce M Simpson m_apply(struct mbuf *m, int off, int len,
143054065297SBruce M Simpson     int (*f)(void *, void *, u_int), void *arg)
143137621fd5SBruce M Simpson {
143254065297SBruce M Simpson 	u_int count;
143337621fd5SBruce M Simpson 	int rval;
143437621fd5SBruce M Simpson 
143537621fd5SBruce M Simpson 	KASSERT(off >= 0, ("m_apply, negative off %d", off));
143637621fd5SBruce M Simpson 	KASSERT(len >= 0, ("m_apply, negative len %d", len));
143737621fd5SBruce M Simpson 	while (off > 0) {
143837621fd5SBruce M Simpson 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
143937621fd5SBruce M Simpson 		if (off < m->m_len)
144037621fd5SBruce M Simpson 			break;
144137621fd5SBruce M Simpson 		off -= m->m_len;
144237621fd5SBruce M Simpson 		m = m->m_next;
144337621fd5SBruce M Simpson 	}
144437621fd5SBruce M Simpson 	while (len > 0) {
144537621fd5SBruce M Simpson 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
144637621fd5SBruce M Simpson 		count = min(m->m_len - off, len);
144737621fd5SBruce M Simpson 		rval = (*f)(arg, mtod(m, caddr_t) + off, count);
144837621fd5SBruce M Simpson 		if (rval)
144937621fd5SBruce M Simpson 			return (rval);
145037621fd5SBruce M Simpson 		len -= count;
145137621fd5SBruce M Simpson 		off = 0;
145237621fd5SBruce M Simpson 		m = m->m_next;
145337621fd5SBruce M Simpson 	}
145437621fd5SBruce M Simpson 	return (0);
145537621fd5SBruce M Simpson }
145637621fd5SBruce M Simpson 
145737621fd5SBruce M Simpson /*
145837621fd5SBruce M Simpson  * Return a pointer to mbuf/offset of location in mbuf chain.
145937621fd5SBruce M Simpson  */
146037621fd5SBruce M Simpson struct mbuf *
146137621fd5SBruce M Simpson m_getptr(struct mbuf *m, int loc, int *off)
146237621fd5SBruce M Simpson {
146337621fd5SBruce M Simpson 
146437621fd5SBruce M Simpson 	while (loc >= 0) {
146554065297SBruce M Simpson 		/* Normal end of search. */
146637621fd5SBruce M Simpson 		if (m->m_len > loc) {
146737621fd5SBruce M Simpson 			*off = loc;
146837621fd5SBruce M Simpson 			return (m);
146937621fd5SBruce M Simpson 		} else {
147037621fd5SBruce M Simpson 			loc -= m->m_len;
147137621fd5SBruce M Simpson 			if (m->m_next == NULL) {
147237621fd5SBruce M Simpson 				if (loc == 0) {
147354065297SBruce M Simpson 					/* Point at the end of valid data. */
147437621fd5SBruce M Simpson 					*off = m->m_len;
147537621fd5SBruce M Simpson 					return (m);
147654065297SBruce M Simpson 				}
147737621fd5SBruce M Simpson 				return (NULL);
147854065297SBruce M Simpson 			}
147937621fd5SBruce M Simpson 			m = m->m_next;
148037621fd5SBruce M Simpson 		}
148137621fd5SBruce M Simpson 	}
148237621fd5SBruce M Simpson 	return (NULL);
148337621fd5SBruce M Simpson }
148437621fd5SBruce M Simpson 
1485ce4a64f7SPoul-Henning Kamp void
14867b125090SJohn-Mark Gurney m_print(const struct mbuf *m, int maxlen)
1487ce4a64f7SPoul-Henning Kamp {
1488ce4a64f7SPoul-Henning Kamp 	int len;
14897b125090SJohn-Mark Gurney 	int pdata;
14906357e7b5SEivind Eklund 	const struct mbuf *m2;
1491ce4a64f7SPoul-Henning Kamp 
14927e949c46SKenneth D. Merry 	if (m == NULL) {
14937e949c46SKenneth D. Merry 		printf("mbuf: %p\n", m);
14947e949c46SKenneth D. Merry 		return;
14957e949c46SKenneth D. Merry 	}
14967e949c46SKenneth D. Merry 
14977b125090SJohn-Mark Gurney 	if (m->m_flags & M_PKTHDR)
1498ce4a64f7SPoul-Henning Kamp 		len = m->m_pkthdr.len;
14997b125090SJohn-Mark Gurney 	else
15007b125090SJohn-Mark Gurney 		len = -1;
1501ce4a64f7SPoul-Henning Kamp 	m2 = m;
15027b125090SJohn-Mark Gurney 	while (m2 != NULL && (len == -1 || len)) {
15037b125090SJohn-Mark Gurney 		pdata = m2->m_len;
15047b125090SJohn-Mark Gurney 		if (maxlen != -1 && pdata > maxlen)
15057b125090SJohn-Mark Gurney 			pdata = maxlen;
15067b125090SJohn-Mark Gurney 		printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
15077b125090SJohn-Mark Gurney 		    m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
15087b125090SJohn-Mark Gurney 		    "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
15097b125090SJohn-Mark Gurney 		    "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
15107b125090SJohn-Mark Gurney 		if (pdata)
151145e0d0aaSJohn-Mark Gurney 			printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
15127b125090SJohn-Mark Gurney 		if (len != -1)
1513ce4a64f7SPoul-Henning Kamp 			len -= m2->m_len;
1514ce4a64f7SPoul-Henning Kamp 		m2 = m2->m_next;
1515ce4a64f7SPoul-Henning Kamp 	}
15167b125090SJohn-Mark Gurney 	if (len > 0)
15177b125090SJohn-Mark Gurney 		printf("%d bytes unaccounted for.\n", len);
1518ce4a64f7SPoul-Henning Kamp 	return;
1519ce4a64f7SPoul-Henning Kamp }
15203f2e06c5SPoul-Henning Kamp 
1521bd395ae8SBosko Milekic u_int
15223f2e06c5SPoul-Henning Kamp m_fixhdr(struct mbuf *m0)
15233f2e06c5SPoul-Henning Kamp {
1524bd395ae8SBosko Milekic 	u_int len;
15253f2e06c5SPoul-Henning Kamp 
1526ac6e585dSPoul-Henning Kamp 	len = m_length(m0, NULL);
15273f2e06c5SPoul-Henning Kamp 	m0->m_pkthdr.len = len;
1528ac6e585dSPoul-Henning Kamp 	return (len);
1529ac6e585dSPoul-Henning Kamp }
1530ac6e585dSPoul-Henning Kamp 
1531bd395ae8SBosko Milekic u_int
1532ac6e585dSPoul-Henning Kamp m_length(struct mbuf *m0, struct mbuf **last)
1533ac6e585dSPoul-Henning Kamp {
1534ac6e585dSPoul-Henning Kamp 	struct mbuf *m;
1535bd395ae8SBosko Milekic 	u_int len;
1536ac6e585dSPoul-Henning Kamp 
1537ac6e585dSPoul-Henning Kamp 	len = 0;
1538ac6e585dSPoul-Henning Kamp 	for (m = m0; m != NULL; m = m->m_next) {
1539ac6e585dSPoul-Henning Kamp 		len += m->m_len;
1540ac6e585dSPoul-Henning Kamp 		if (m->m_next == NULL)
1541ac6e585dSPoul-Henning Kamp 			break;
1542ac6e585dSPoul-Henning Kamp 	}
1543ac6e585dSPoul-Henning Kamp 	if (last != NULL)
1544ac6e585dSPoul-Henning Kamp 		*last = m;
1545ac6e585dSPoul-Henning Kamp 	return (len);
15463f2e06c5SPoul-Henning Kamp }
154755e9f80dSMike Silbersack 
154855e9f80dSMike Silbersack /*
154955e9f80dSMike Silbersack  * Defragment a mbuf chain, returning the shortest possible
155055e9f80dSMike Silbersack  * chain of mbufs and clusters.  If allocation fails and
155155e9f80dSMike Silbersack  * this cannot be completed, NULL will be returned, but
155255e9f80dSMike Silbersack  * the passed in chain will be unchanged.  Upon success,
155355e9f80dSMike Silbersack  * the original chain will be freed, and the new chain
155455e9f80dSMike Silbersack  * will be returned.
155555e9f80dSMike Silbersack  *
155655e9f80dSMike Silbersack  * If a non-packet header is passed in, the original
155755e9f80dSMike Silbersack  * mbuf (chain?) will be returned unharmed.
155855e9f80dSMike Silbersack  */
155955e9f80dSMike Silbersack struct mbuf *
156055e9f80dSMike Silbersack m_defrag(struct mbuf *m0, int how)
156155e9f80dSMike Silbersack {
156255e9f80dSMike Silbersack 	struct mbuf *m_new = NULL, *m_final = NULL;
156355e9f80dSMike Silbersack 	int progress = 0, length;
156455e9f80dSMike Silbersack 
1565063d8114SAlfred Perlstein 	MBUF_CHECKSLEEP(how);
156655e9f80dSMike Silbersack 	if (!(m0->m_flags & M_PKTHDR))
156755e9f80dSMike Silbersack 		return (m0);
156855e9f80dSMike Silbersack 
1569f8bf8e39SMike Silbersack 	m_fixhdr(m0); /* Needed sanity check */
1570f8bf8e39SMike Silbersack 
1571352d050eSMike Silbersack #ifdef MBUF_STRESS_TEST
1572352d050eSMike Silbersack 	if (m_defragrandomfailures) {
1573352d050eSMike Silbersack 		int temp = arc4random() & 0xff;
1574352d050eSMike Silbersack 		if (temp == 0xba)
1575352d050eSMike Silbersack 			goto nospace;
1576352d050eSMike Silbersack 	}
1577352d050eSMike Silbersack #endif
157855e9f80dSMike Silbersack 
157955e9f80dSMike Silbersack 	if (m0->m_pkthdr.len > MHLEN)
158055e9f80dSMike Silbersack 		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
158155e9f80dSMike Silbersack 	else
158255e9f80dSMike Silbersack 		m_final = m_gethdr(how, MT_DATA);
158355e9f80dSMike Silbersack 
158455e9f80dSMike Silbersack 	if (m_final == NULL)
158555e9f80dSMike Silbersack 		goto nospace;
158655e9f80dSMike Silbersack 
1587a89ec05eSPeter Wemm 	if (m_dup_pkthdr(m_final, m0, how) == 0)
158855e9f80dSMike Silbersack 		goto nospace;
158955e9f80dSMike Silbersack 
159055e9f80dSMike Silbersack 	m_new = m_final;
159155e9f80dSMike Silbersack 
159255e9f80dSMike Silbersack 	while (progress < m0->m_pkthdr.len) {
159355e9f80dSMike Silbersack 		length = m0->m_pkthdr.len - progress;
159455e9f80dSMike Silbersack 		if (length > MCLBYTES)
159555e9f80dSMike Silbersack 			length = MCLBYTES;
159655e9f80dSMike Silbersack 
159755e9f80dSMike Silbersack 		if (m_new == NULL) {
159855e9f80dSMike Silbersack 			if (length > MLEN)
159955e9f80dSMike Silbersack 				m_new = m_getcl(how, MT_DATA, 0);
160055e9f80dSMike Silbersack 			else
160155e9f80dSMike Silbersack 				m_new = m_get(how, MT_DATA);
160255e9f80dSMike Silbersack 			if (m_new == NULL)
160355e9f80dSMike Silbersack 				goto nospace;
160455e9f80dSMike Silbersack 		}
160555e9f80dSMike Silbersack 
160655e9f80dSMike Silbersack 		m_copydata(m0, progress, length, mtod(m_new, caddr_t));
160755e9f80dSMike Silbersack 		progress += length;
160855e9f80dSMike Silbersack 		m_new->m_len = length;
160955e9f80dSMike Silbersack 		if (m_new != m_final)
161055e9f80dSMike Silbersack 			m_cat(m_final, m_new);
161155e9f80dSMike Silbersack 		m_new = NULL;
161255e9f80dSMike Silbersack 	}
161351710a45SMike Silbersack #ifdef MBUF_STRESS_TEST
161455e9f80dSMike Silbersack 	if (m0->m_next == NULL)
161555e9f80dSMike Silbersack 		m_defraguseless++;
161651710a45SMike Silbersack #endif
161755e9f80dSMike Silbersack 	m_freem(m0);
161855e9f80dSMike Silbersack 	m0 = m_final;
161951710a45SMike Silbersack #ifdef MBUF_STRESS_TEST
162055e9f80dSMike Silbersack 	m_defragpackets++;
162155e9f80dSMike Silbersack 	m_defragbytes += m0->m_pkthdr.len;
162251710a45SMike Silbersack #endif
162355e9f80dSMike Silbersack 	return (m0);
162455e9f80dSMike Silbersack nospace:
162551710a45SMike Silbersack #ifdef MBUF_STRESS_TEST
162655e9f80dSMike Silbersack 	m_defragfailure++;
162751710a45SMike Silbersack #endif
162855e9f80dSMike Silbersack 	if (m_final)
162955e9f80dSMike Silbersack 		m_freem(m_final);
163055e9f80dSMike Silbersack 	return (NULL);
163155e9f80dSMike Silbersack }
16323390d476SMike Silbersack 
1633eeb76a18SSam Leffler /*
1634eeb76a18SSam Leffler  * Defragment an mbuf chain, returning at most maxfrags separate
1635eeb76a18SSam Leffler  * mbufs+clusters.  If this is not possible NULL is returned and
1636eeb76a18SSam Leffler  * the original mbuf chain is left in it's present (potentially
1637eeb76a18SSam Leffler  * modified) state.  We use two techniques: collapsing consecutive
1638eeb76a18SSam Leffler  * mbufs and replacing consecutive mbufs by a cluster.
1639eeb76a18SSam Leffler  *
1640eeb76a18SSam Leffler  * NB: this should really be named m_defrag but that name is taken
1641eeb76a18SSam Leffler  */
1642eeb76a18SSam Leffler struct mbuf *
1643eeb76a18SSam Leffler m_collapse(struct mbuf *m0, int how, int maxfrags)
1644eeb76a18SSam Leffler {
1645eeb76a18SSam Leffler 	struct mbuf *m, *n, *n2, **prev;
1646eeb76a18SSam Leffler 	u_int curfrags;
1647eeb76a18SSam Leffler 
1648eeb76a18SSam Leffler 	/*
1649eeb76a18SSam Leffler 	 * Calculate the current number of frags.
1650eeb76a18SSam Leffler 	 */
1651eeb76a18SSam Leffler 	curfrags = 0;
1652eeb76a18SSam Leffler 	for (m = m0; m != NULL; m = m->m_next)
1653eeb76a18SSam Leffler 		curfrags++;
1654eeb76a18SSam Leffler 	/*
1655eeb76a18SSam Leffler 	 * First, try to collapse mbufs.  Note that we always collapse
1656eeb76a18SSam Leffler 	 * towards the front so we don't need to deal with moving the
1657eeb76a18SSam Leffler 	 * pkthdr.  This may be suboptimal if the first mbuf has much
1658eeb76a18SSam Leffler 	 * less data than the following.
1659eeb76a18SSam Leffler 	 */
1660eeb76a18SSam Leffler 	m = m0;
1661eeb76a18SSam Leffler again:
1662eeb76a18SSam Leffler 	for (;;) {
1663eeb76a18SSam Leffler 		n = m->m_next;
1664eeb76a18SSam Leffler 		if (n == NULL)
1665eeb76a18SSam Leffler 			break;
166614d7c5b1SAndre Oppermann 		if (M_WRITABLE(m) &&
1667eeb76a18SSam Leffler 		    n->m_len < M_TRAILINGSPACE(m)) {
1668eeb76a18SSam Leffler 			bcopy(mtod(n, void *), mtod(m, char *) + m->m_len,
1669eeb76a18SSam Leffler 				n->m_len);
1670eeb76a18SSam Leffler 			m->m_len += n->m_len;
1671eeb76a18SSam Leffler 			m->m_next = n->m_next;
1672eeb76a18SSam Leffler 			m_free(n);
1673eeb76a18SSam Leffler 			if (--curfrags <= maxfrags)
1674eeb76a18SSam Leffler 				return m0;
1675eeb76a18SSam Leffler 		} else
1676eeb76a18SSam Leffler 			m = n;
1677eeb76a18SSam Leffler 	}
1678eeb76a18SSam Leffler 	KASSERT(maxfrags > 1,
1679eeb76a18SSam Leffler 		("maxfrags %u, but normal collapse failed", maxfrags));
1680eeb76a18SSam Leffler 	/*
1681eeb76a18SSam Leffler 	 * Collapse consecutive mbufs to a cluster.
1682eeb76a18SSam Leffler 	 */
1683eeb76a18SSam Leffler 	prev = &m0->m_next;		/* NB: not the first mbuf */
1684eeb76a18SSam Leffler 	while ((n = *prev) != NULL) {
1685eeb76a18SSam Leffler 		if ((n2 = n->m_next) != NULL &&
1686eeb76a18SSam Leffler 		    n->m_len + n2->m_len < MCLBYTES) {
1687eeb76a18SSam Leffler 			m = m_getcl(how, MT_DATA, 0);
1688eeb76a18SSam Leffler 			if (m == NULL)
1689eeb76a18SSam Leffler 				goto bad;
1690eeb76a18SSam Leffler 			bcopy(mtod(n, void *), mtod(m, void *), n->m_len);
1691eeb76a18SSam Leffler 			bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len,
1692eeb76a18SSam Leffler 				n2->m_len);
1693eeb76a18SSam Leffler 			m->m_len = n->m_len + n2->m_len;
1694eeb76a18SSam Leffler 			m->m_next = n2->m_next;
1695eeb76a18SSam Leffler 			*prev = m;
1696eeb76a18SSam Leffler 			m_free(n);
1697eeb76a18SSam Leffler 			m_free(n2);
1698eeb76a18SSam Leffler 			if (--curfrags <= maxfrags)	/* +1 cl -2 mbufs */
1699eeb76a18SSam Leffler 				return m0;
1700eeb76a18SSam Leffler 			/*
1701eeb76a18SSam Leffler 			 * Still not there, try the normal collapse
1702eeb76a18SSam Leffler 			 * again before we allocate another cluster.
1703eeb76a18SSam Leffler 			 */
1704eeb76a18SSam Leffler 			goto again;
1705eeb76a18SSam Leffler 		}
1706eeb76a18SSam Leffler 		prev = &n->m_next;
1707eeb76a18SSam Leffler 	}
1708eeb76a18SSam Leffler 	/*
1709eeb76a18SSam Leffler 	 * No place where we can collapse to a cluster; punt.
1710eeb76a18SSam Leffler 	 * This can occur if, for example, you request 2 frags
1711eeb76a18SSam Leffler 	 * but the packet requires that both be clusters (we
1712eeb76a18SSam Leffler 	 * never reallocate the first mbuf to avoid moving the
1713eeb76a18SSam Leffler 	 * packet header).
1714eeb76a18SSam Leffler 	 */
1715eeb76a18SSam Leffler bad:
1716eeb76a18SSam Leffler 	return NULL;
1717eeb76a18SSam Leffler }
1718eeb76a18SSam Leffler 
17193390d476SMike Silbersack #ifdef MBUF_STRESS_TEST
17203390d476SMike Silbersack 
17213390d476SMike Silbersack /*
17223390d476SMike Silbersack  * Fragment an mbuf chain.  There's no reason you'd ever want to do
17233390d476SMike Silbersack  * this in normal usage, but it's great for stress testing various
17243390d476SMike Silbersack  * mbuf consumers.
17253390d476SMike Silbersack  *
17263390d476SMike Silbersack  * If fragmentation is not possible, the original chain will be
17273390d476SMike Silbersack  * returned.
17283390d476SMike Silbersack  *
17293390d476SMike Silbersack  * Possible length values:
17303390d476SMike Silbersack  * 0	 no fragmentation will occur
17313390d476SMike Silbersack  * > 0	each fragment will be of the specified length
17323390d476SMike Silbersack  * -1	each fragment will be the same random value in length
17333390d476SMike Silbersack  * -2	each fragment's length will be entirely random
17343390d476SMike Silbersack  * (Random values range from 1 to 256)
17353390d476SMike Silbersack  */
17363390d476SMike Silbersack struct mbuf *
17373390d476SMike Silbersack m_fragment(struct mbuf *m0, int how, int length)
17383390d476SMike Silbersack {
17393390d476SMike Silbersack 	struct mbuf *m_new = NULL, *m_final = NULL;
17403390d476SMike Silbersack 	int progress = 0;
17413390d476SMike Silbersack 
17423390d476SMike Silbersack 	if (!(m0->m_flags & M_PKTHDR))
17433390d476SMike Silbersack 		return (m0);
17443390d476SMike Silbersack 
17453390d476SMike Silbersack 	if ((length == 0) || (length < -2))
17463390d476SMike Silbersack 		return (m0);
17473390d476SMike Silbersack 
17483390d476SMike Silbersack 	m_fixhdr(m0); /* Needed sanity check */
17493390d476SMike Silbersack 
17503390d476SMike Silbersack 	m_final = m_getcl(how, MT_DATA, M_PKTHDR);
17513390d476SMike Silbersack 
17523390d476SMike Silbersack 	if (m_final == NULL)
17533390d476SMike Silbersack 		goto nospace;
17543390d476SMike Silbersack 
17558dee2f67SMike Silbersack 	if (m_dup_pkthdr(m_final, m0, how) == 0)
17563390d476SMike Silbersack 		goto nospace;
17573390d476SMike Silbersack 
17583390d476SMike Silbersack 	m_new = m_final;
17593390d476SMike Silbersack 
17603390d476SMike Silbersack 	if (length == -1)
17613390d476SMike Silbersack 		length = 1 + (arc4random() & 255);
17623390d476SMike Silbersack 
17633390d476SMike Silbersack 	while (progress < m0->m_pkthdr.len) {
17643390d476SMike Silbersack 		int fraglen;
17653390d476SMike Silbersack 
17663390d476SMike Silbersack 		if (length > 0)
17673390d476SMike Silbersack 			fraglen = length;
17683390d476SMike Silbersack 		else
17693390d476SMike Silbersack 			fraglen = 1 + (arc4random() & 255);
17703390d476SMike Silbersack 		if (fraglen > m0->m_pkthdr.len - progress)
17713390d476SMike Silbersack 			fraglen = m0->m_pkthdr.len - progress;
17723390d476SMike Silbersack 
17733390d476SMike Silbersack 		if (fraglen > MCLBYTES)
17743390d476SMike Silbersack 			fraglen = MCLBYTES;
17753390d476SMike Silbersack 
17763390d476SMike Silbersack 		if (m_new == NULL) {
17773390d476SMike Silbersack 			m_new = m_getcl(how, MT_DATA, 0);
17783390d476SMike Silbersack 			if (m_new == NULL)
17793390d476SMike Silbersack 				goto nospace;
17803390d476SMike Silbersack 		}
17813390d476SMike Silbersack 
17823390d476SMike Silbersack 		m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
17833390d476SMike Silbersack 		progress += fraglen;
17843390d476SMike Silbersack 		m_new->m_len = fraglen;
17853390d476SMike Silbersack 		if (m_new != m_final)
17863390d476SMike Silbersack 			m_cat(m_final, m_new);
17873390d476SMike Silbersack 		m_new = NULL;
17883390d476SMike Silbersack 	}
17893390d476SMike Silbersack 	m_freem(m0);
17903390d476SMike Silbersack 	m0 = m_final;
17913390d476SMike Silbersack 	return (m0);
17923390d476SMike Silbersack nospace:
17933390d476SMike Silbersack 	if (m_final)
17943390d476SMike Silbersack 		m_freem(m_final);
17953390d476SMike Silbersack 	/* Return the original chain on failure */
17963390d476SMike Silbersack 	return (m0);
17973390d476SMike Silbersack }
17983390d476SMike Silbersack 
17993390d476SMike Silbersack #endif
1800beb699c7SMike Silbersack 
18015e20f43dSAndre Oppermann /*
18025e20f43dSAndre Oppermann  * Copy the contents of uio into a properly sized mbuf chain.
18035e20f43dSAndre Oppermann  */
1804beb699c7SMike Silbersack struct mbuf *
18055e20f43dSAndre Oppermann m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
1806beb699c7SMike Silbersack {
18075e20f43dSAndre Oppermann 	struct mbuf *m, *mb;
1808526d0bd5SKonstantin Belousov 	int error, length;
1809526d0bd5SKonstantin Belousov 	ssize_t total;
18105e20f43dSAndre Oppermann 	int progress = 0;
1811beb699c7SMike Silbersack 
18125e20f43dSAndre Oppermann 	/*
18135e20f43dSAndre Oppermann 	 * len can be zero or an arbitrary large value bound by
18145e20f43dSAndre Oppermann 	 * the total data supplied by the uio.
18155e20f43dSAndre Oppermann 	 */
1816beb699c7SMike Silbersack 	if (len > 0)
1817beb699c7SMike Silbersack 		total = min(uio->uio_resid, len);
1818beb699c7SMike Silbersack 	else
1819beb699c7SMike Silbersack 		total = uio->uio_resid;
18205e20f43dSAndre Oppermann 
18215e20f43dSAndre Oppermann 	/*
18225e20f43dSAndre Oppermann 	 * The smallest unit returned by m_getm2() is a single mbuf
18239128ec21SAndrew Thompson 	 * with pkthdr.  We can't align past it.
18245e20f43dSAndre Oppermann 	 */
182575ae2570SMaksim Yevmenkin 	if (align >= MHLEN)
1826beb699c7SMike Silbersack 		return (NULL);
18275e20f43dSAndre Oppermann 
18287c32173bSAndre Oppermann 	/*
18297c32173bSAndre Oppermann 	 * Give us the full allocation or nothing.
18307c32173bSAndre Oppermann 	 * If len is zero return the smallest empty mbuf.
18317c32173bSAndre Oppermann 	 */
18327c32173bSAndre Oppermann 	m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
18335e20f43dSAndre Oppermann 	if (m == NULL)
18345e20f43dSAndre Oppermann 		return (NULL);
18355e20f43dSAndre Oppermann 	m->m_data += align;
18365e20f43dSAndre Oppermann 
18375e20f43dSAndre Oppermann 	/* Fill all mbufs with uio data and update header information. */
18385e20f43dSAndre Oppermann 	for (mb = m; mb != NULL; mb = mb->m_next) {
18395e20f43dSAndre Oppermann 		length = min(M_TRAILINGSPACE(mb), total - progress);
18405e20f43dSAndre Oppermann 
18415e20f43dSAndre Oppermann 		error = uiomove(mtod(mb, void *), length, uio);
18425e20f43dSAndre Oppermann 		if (error) {
18435e20f43dSAndre Oppermann 			m_freem(m);
18445e20f43dSAndre Oppermann 			return (NULL);
18455e20f43dSAndre Oppermann 		}
18465e20f43dSAndre Oppermann 
18475e20f43dSAndre Oppermann 		mb->m_len = length;
18485e20f43dSAndre Oppermann 		progress += length;
18495e20f43dSAndre Oppermann 		if (flags & M_PKTHDR)
18505e20f43dSAndre Oppermann 			m->m_pkthdr.len += length;
18515e20f43dSAndre Oppermann 	}
18525e20f43dSAndre Oppermann 	KASSERT(progress == total, ("%s: progress != total", __func__));
18535e20f43dSAndre Oppermann 
18545e20f43dSAndre Oppermann 	return (m);
1855beb699c7SMike Silbersack }
1856ab8ab90cSSam Leffler 
1857ab8ab90cSSam Leffler /*
1858bc05b2f6SAndre Oppermann  * Copy an mbuf chain into a uio limited by len if set.
1859bc05b2f6SAndre Oppermann  */
1860bc05b2f6SAndre Oppermann int
1861bc05b2f6SAndre Oppermann m_mbuftouio(struct uio *uio, struct mbuf *m, int len)
1862bc05b2f6SAndre Oppermann {
1863bc05b2f6SAndre Oppermann 	int error, length, total;
1864bc05b2f6SAndre Oppermann 	int progress = 0;
1865bc05b2f6SAndre Oppermann 
1866bc05b2f6SAndre Oppermann 	if (len > 0)
1867bc05b2f6SAndre Oppermann 		total = min(uio->uio_resid, len);
1868bc05b2f6SAndre Oppermann 	else
1869bc05b2f6SAndre Oppermann 		total = uio->uio_resid;
1870bc05b2f6SAndre Oppermann 
1871bc05b2f6SAndre Oppermann 	/* Fill the uio with data from the mbufs. */
1872bc05b2f6SAndre Oppermann 	for (; m != NULL; m = m->m_next) {
1873bc05b2f6SAndre Oppermann 		length = min(m->m_len, total - progress);
1874bc05b2f6SAndre Oppermann 
1875bc05b2f6SAndre Oppermann 		error = uiomove(mtod(m, void *), length, uio);
1876bc05b2f6SAndre Oppermann 		if (error)
1877bc05b2f6SAndre Oppermann 			return (error);
1878bc05b2f6SAndre Oppermann 
1879bc05b2f6SAndre Oppermann 		progress += length;
1880bc05b2f6SAndre Oppermann 	}
1881bc05b2f6SAndre Oppermann 
1882bc05b2f6SAndre Oppermann 	return (0);
1883bc05b2f6SAndre Oppermann }
1884bc05b2f6SAndre Oppermann 
1885bc05b2f6SAndre Oppermann /*
188647e2996eSSam Leffler  * Create a writable copy of the mbuf chain.  While doing this
188747e2996eSSam Leffler  * we compact the chain with a goal of producing a chain with
188847e2996eSSam Leffler  * at most two mbufs.  The second mbuf in this chain is likely
188947e2996eSSam Leffler  * to be a cluster.  The primary purpose of this work is to create
189047e2996eSSam Leffler  * a writable packet for encryption, compression, etc.  The
189147e2996eSSam Leffler  * secondary goal is to linearize the data so the data can be
189247e2996eSSam Leffler  * passed to crypto hardware in the most efficient manner possible.
189347e2996eSSam Leffler  */
189447e2996eSSam Leffler struct mbuf *
189547e2996eSSam Leffler m_unshare(struct mbuf *m0, int how)
189647e2996eSSam Leffler {
189747e2996eSSam Leffler 	struct mbuf *m, *mprev;
189847e2996eSSam Leffler 	struct mbuf *n, *mfirst, *mlast;
189947e2996eSSam Leffler 	int len, off;
190047e2996eSSam Leffler 
190147e2996eSSam Leffler 	mprev = NULL;
190247e2996eSSam Leffler 	for (m = m0; m != NULL; m = mprev->m_next) {
190347e2996eSSam Leffler 		/*
190447e2996eSSam Leffler 		 * Regular mbufs are ignored unless there's a cluster
190547e2996eSSam Leffler 		 * in front of it that we can use to coalesce.  We do
190647e2996eSSam Leffler 		 * the latter mainly so later clusters can be coalesced
190747e2996eSSam Leffler 		 * also w/o having to handle them specially (i.e. convert
190847e2996eSSam Leffler 		 * mbuf+cluster -> cluster).  This optimization is heavily
190947e2996eSSam Leffler 		 * influenced by the assumption that we're running over
191047e2996eSSam Leffler 		 * Ethernet where MCLBYTES is large enough that the max
191147e2996eSSam Leffler 		 * packet size will permit lots of coalescing into a
191247e2996eSSam Leffler 		 * single cluster.  This in turn permits efficient
191347e2996eSSam Leffler 		 * crypto operations, especially when using hardware.
191447e2996eSSam Leffler 		 */
191547e2996eSSam Leffler 		if ((m->m_flags & M_EXT) == 0) {
191647e2996eSSam Leffler 			if (mprev && (mprev->m_flags & M_EXT) &&
191747e2996eSSam Leffler 			    m->m_len <= M_TRAILINGSPACE(mprev)) {
191847e2996eSSam Leffler 				/* XXX: this ignores mbuf types */
191947e2996eSSam Leffler 				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
192047e2996eSSam Leffler 				    mtod(m, caddr_t), m->m_len);
192147e2996eSSam Leffler 				mprev->m_len += m->m_len;
192247e2996eSSam Leffler 				mprev->m_next = m->m_next;	/* unlink from chain */
192347e2996eSSam Leffler 				m_free(m);			/* reclaim mbuf */
192447e2996eSSam Leffler #if 0
192547e2996eSSam Leffler 				newipsecstat.ips_mbcoalesced++;
192647e2996eSSam Leffler #endif
192747e2996eSSam Leffler 			} else {
192847e2996eSSam Leffler 				mprev = m;
192947e2996eSSam Leffler 			}
193047e2996eSSam Leffler 			continue;
193147e2996eSSam Leffler 		}
193247e2996eSSam Leffler 		/*
193347e2996eSSam Leffler 		 * Writable mbufs are left alone (for now).
193447e2996eSSam Leffler 		 */
193547e2996eSSam Leffler 		if (M_WRITABLE(m)) {
193647e2996eSSam Leffler 			mprev = m;
193747e2996eSSam Leffler 			continue;
193847e2996eSSam Leffler 		}
193947e2996eSSam Leffler 
194047e2996eSSam Leffler 		/*
194147e2996eSSam Leffler 		 * Not writable, replace with a copy or coalesce with
194247e2996eSSam Leffler 		 * the previous mbuf if possible (since we have to copy
194347e2996eSSam Leffler 		 * it anyway, we try to reduce the number of mbufs and
194447e2996eSSam Leffler 		 * clusters so that future work is easier).
194547e2996eSSam Leffler 		 */
194647e2996eSSam Leffler 		KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
194747e2996eSSam Leffler 		/* NB: we only coalesce into a cluster or larger */
194847e2996eSSam Leffler 		if (mprev != NULL && (mprev->m_flags & M_EXT) &&
194947e2996eSSam Leffler 		    m->m_len <= M_TRAILINGSPACE(mprev)) {
195047e2996eSSam Leffler 			/* XXX: this ignores mbuf types */
195147e2996eSSam Leffler 			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
195247e2996eSSam Leffler 			    mtod(m, caddr_t), m->m_len);
195347e2996eSSam Leffler 			mprev->m_len += m->m_len;
195447e2996eSSam Leffler 			mprev->m_next = m->m_next;	/* unlink from chain */
195547e2996eSSam Leffler 			m_free(m);			/* reclaim mbuf */
195647e2996eSSam Leffler #if 0
195747e2996eSSam Leffler 			newipsecstat.ips_clcoalesced++;
195847e2996eSSam Leffler #endif
195947e2996eSSam Leffler 			continue;
196047e2996eSSam Leffler 		}
196147e2996eSSam Leffler 
196247e2996eSSam Leffler 		/*
19635368b81eSGleb Smirnoff 		 * Allocate new space to hold the copy and copy the data.
19645368b81eSGleb Smirnoff 		 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
19655368b81eSGleb Smirnoff 		 * splitting them into clusters.  We could just malloc a
19665368b81eSGleb Smirnoff 		 * buffer and make it external but too many device drivers
19675368b81eSGleb Smirnoff 		 * don't know how to break up the non-contiguous memory when
19685368b81eSGleb Smirnoff 		 * doing DMA.
196947e2996eSSam Leffler 		 */
197047e2996eSSam Leffler 		n = m_getcl(how, m->m_type, m->m_flags);
197147e2996eSSam Leffler 		if (n == NULL) {
197247e2996eSSam Leffler 			m_freem(m0);
197347e2996eSSam Leffler 			return (NULL);
197447e2996eSSam Leffler 		}
197547e2996eSSam Leffler 		len = m->m_len;
197647e2996eSSam Leffler 		off = 0;
197747e2996eSSam Leffler 		mfirst = n;
197847e2996eSSam Leffler 		mlast = NULL;
197947e2996eSSam Leffler 		for (;;) {
198047e2996eSSam Leffler 			int cc = min(len, MCLBYTES);
198147e2996eSSam Leffler 			memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
198247e2996eSSam Leffler 			n->m_len = cc;
198347e2996eSSam Leffler 			if (mlast != NULL)
198447e2996eSSam Leffler 				mlast->m_next = n;
198547e2996eSSam Leffler 			mlast = n;
198647e2996eSSam Leffler #if 0
198747e2996eSSam Leffler 			newipsecstat.ips_clcopied++;
198847e2996eSSam Leffler #endif
198947e2996eSSam Leffler 
199047e2996eSSam Leffler 			len -= cc;
199147e2996eSSam Leffler 			if (len <= 0)
199247e2996eSSam Leffler 				break;
199347e2996eSSam Leffler 			off += cc;
199447e2996eSSam Leffler 
199547e2996eSSam Leffler 			n = m_getcl(how, m->m_type, m->m_flags);
199647e2996eSSam Leffler 			if (n == NULL) {
199747e2996eSSam Leffler 				m_freem(mfirst);
199847e2996eSSam Leffler 				m_freem(m0);
199947e2996eSSam Leffler 				return (NULL);
200047e2996eSSam Leffler 			}
200147e2996eSSam Leffler 		}
200247e2996eSSam Leffler 		n->m_next = m->m_next;
200347e2996eSSam Leffler 		if (mprev == NULL)
200447e2996eSSam Leffler 			m0 = mfirst;		/* new head of chain */
200547e2996eSSam Leffler 		else
200647e2996eSSam Leffler 			mprev->m_next = mfirst;	/* replace old mbuf */
200747e2996eSSam Leffler 		m_free(m);			/* release old mbuf */
200847e2996eSSam Leffler 		mprev = mfirst;
200947e2996eSSam Leffler 	}
201047e2996eSSam Leffler 	return (m0);
201147e2996eSSam Leffler }
20126eeac1d9SJulian Elischer 
20136eeac1d9SJulian Elischer #ifdef MBUF_PROFILING
20146eeac1d9SJulian Elischer 
20156eeac1d9SJulian Elischer #define MP_BUCKETS 32 /* don't just change this as things may overflow.*/
20166eeac1d9SJulian Elischer struct mbufprofile {
20172182c0cfSJulian Elischer 	uintmax_t wasted[MP_BUCKETS];
20182182c0cfSJulian Elischer 	uintmax_t used[MP_BUCKETS];
20192182c0cfSJulian Elischer 	uintmax_t segments[MP_BUCKETS];
20206eeac1d9SJulian Elischer } mbprof;
20216eeac1d9SJulian Elischer 
20226eeac1d9SJulian Elischer #define MP_MAXDIGITS 21	/* strlen("16,000,000,000,000,000,000") == 21 */
20236eeac1d9SJulian Elischer #define MP_NUMLINES 6
20246eeac1d9SJulian Elischer #define MP_NUMSPERLINE 16
20256eeac1d9SJulian Elischer #define MP_EXTRABYTES 64	/* > strlen("used:\nwasted:\nsegments:\n") */
20266eeac1d9SJulian Elischer /* work out max space needed and add a bit of spare space too */
20276eeac1d9SJulian Elischer #define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE)
20286eeac1d9SJulian Elischer #define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES)
20296eeac1d9SJulian Elischer 
20306eeac1d9SJulian Elischer char mbprofbuf[MP_BUFSIZE];
20316eeac1d9SJulian Elischer 
20326eeac1d9SJulian Elischer void
20336eeac1d9SJulian Elischer m_profile(struct mbuf *m)
20346eeac1d9SJulian Elischer {
20356eeac1d9SJulian Elischer 	int segments = 0;
20366eeac1d9SJulian Elischer 	int used = 0;
20376eeac1d9SJulian Elischer 	int wasted = 0;
20386eeac1d9SJulian Elischer 
20396eeac1d9SJulian Elischer 	while (m) {
20406eeac1d9SJulian Elischer 		segments++;
20416eeac1d9SJulian Elischer 		used += m->m_len;
20426eeac1d9SJulian Elischer 		if (m->m_flags & M_EXT) {
20436eeac1d9SJulian Elischer 			wasted += MHLEN - sizeof(m->m_ext) +
20446eeac1d9SJulian Elischer 			    m->m_ext.ext_size - m->m_len;
20456eeac1d9SJulian Elischer 		} else {
20466eeac1d9SJulian Elischer 			if (m->m_flags & M_PKTHDR)
20476eeac1d9SJulian Elischer 				wasted += MHLEN - m->m_len;
20486eeac1d9SJulian Elischer 			else
20496eeac1d9SJulian Elischer 				wasted += MLEN - m->m_len;
20506eeac1d9SJulian Elischer 		}
20516eeac1d9SJulian Elischer 		m = m->m_next;
20526eeac1d9SJulian Elischer 	}
20536eeac1d9SJulian Elischer 	/* be paranoid.. it helps */
20546eeac1d9SJulian Elischer 	if (segments > MP_BUCKETS - 1)
20556eeac1d9SJulian Elischer 		segments = MP_BUCKETS - 1;
20566eeac1d9SJulian Elischer 	if (used > 100000)
20576eeac1d9SJulian Elischer 		used = 100000;
20586eeac1d9SJulian Elischer 	if (wasted > 100000)
20596eeac1d9SJulian Elischer 		wasted = 100000;
20606eeac1d9SJulian Elischer 	/* store in the appropriate bucket */
20616eeac1d9SJulian Elischer 	/* don't bother locking. if it's slightly off, so what? */
20626eeac1d9SJulian Elischer 	mbprof.segments[segments]++;
20636eeac1d9SJulian Elischer 	mbprof.used[fls(used)]++;
20646eeac1d9SJulian Elischer 	mbprof.wasted[fls(wasted)]++;
20656eeac1d9SJulian Elischer }
20666eeac1d9SJulian Elischer 
20676eeac1d9SJulian Elischer static void
20686eeac1d9SJulian Elischer mbprof_textify(void)
20696eeac1d9SJulian Elischer {
20706eeac1d9SJulian Elischer 	int offset;
20716eeac1d9SJulian Elischer 	char *c;
207260ae52f7SEd Schouten 	uint64_t *p;
20736eeac1d9SJulian Elischer 
20746eeac1d9SJulian Elischer 	p = &mbprof.wasted[0];
20756eeac1d9SJulian Elischer 	c = mbprofbuf;
20766eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE + 10,
20776eeac1d9SJulian Elischer 	    "wasted:\n"
20782182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
20792182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
20806eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
20816eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
20826eeac1d9SJulian Elischer #ifdef BIG_ARRAY
20836eeac1d9SJulian Elischer 	p = &mbprof.wasted[16];
20846eeac1d9SJulian Elischer 	c += offset;
20856eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE,
20862182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
20872182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
20886eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
20896eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
20906eeac1d9SJulian Elischer #endif
20916eeac1d9SJulian Elischer 	p = &mbprof.used[0];
20926eeac1d9SJulian Elischer 	c += offset;
20936eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE + 10,
20946eeac1d9SJulian Elischer 	    "used:\n"
20952182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
20962182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
20976eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
20986eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
20996eeac1d9SJulian Elischer #ifdef BIG_ARRAY
21006eeac1d9SJulian Elischer 	p = &mbprof.used[16];
21016eeac1d9SJulian Elischer 	c += offset;
21026eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE,
21032182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
21042182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
21056eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
21066eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
21076eeac1d9SJulian Elischer #endif
21086eeac1d9SJulian Elischer 	p = &mbprof.segments[0];
21096eeac1d9SJulian Elischer 	c += offset;
21106eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE + 10,
21116eeac1d9SJulian Elischer 	    "segments:\n"
21122182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
21132182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
21146eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
21156eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
21166eeac1d9SJulian Elischer #ifdef BIG_ARRAY
21176eeac1d9SJulian Elischer 	p = &mbprof.segments[16];
21186eeac1d9SJulian Elischer 	c += offset;
21196eeac1d9SJulian Elischer 	offset = snprintf(c, MP_MAXLINE,
21202182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
21212182c0cfSJulian Elischer 	    "%ju %ju %ju %ju %ju %ju %ju %jju",
21226eeac1d9SJulian Elischer 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
21236eeac1d9SJulian Elischer 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
21246eeac1d9SJulian Elischer #endif
21256eeac1d9SJulian Elischer }
21266eeac1d9SJulian Elischer 
21276eeac1d9SJulian Elischer static int
21286eeac1d9SJulian Elischer mbprof_handler(SYSCTL_HANDLER_ARGS)
21296eeac1d9SJulian Elischer {
21306eeac1d9SJulian Elischer 	int error;
21316eeac1d9SJulian Elischer 
21326eeac1d9SJulian Elischer 	mbprof_textify();
21336eeac1d9SJulian Elischer 	error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1);
21346eeac1d9SJulian Elischer 	return (error);
21356eeac1d9SJulian Elischer }
21366eeac1d9SJulian Elischer 
21376eeac1d9SJulian Elischer static int
21386eeac1d9SJulian Elischer mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
21396eeac1d9SJulian Elischer {
21406eeac1d9SJulian Elischer 	int clear, error;
21416eeac1d9SJulian Elischer 
21426eeac1d9SJulian Elischer 	clear = 0;
21436eeac1d9SJulian Elischer 	error = sysctl_handle_int(oidp, &clear, 0, req);
21446eeac1d9SJulian Elischer 	if (error || !req->newptr)
21456eeac1d9SJulian Elischer 		return (error);
21466eeac1d9SJulian Elischer 
21476eeac1d9SJulian Elischer 	if (clear) {
21486eeac1d9SJulian Elischer 		bzero(&mbprof, sizeof(mbprof));
21496eeac1d9SJulian Elischer 	}
21506eeac1d9SJulian Elischer 
21516eeac1d9SJulian Elischer 	return (error);
21526eeac1d9SJulian Elischer }
21536eeac1d9SJulian Elischer 
21546eeac1d9SJulian Elischer 
21556eeac1d9SJulian Elischer SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD,
21566eeac1d9SJulian Elischer 	    NULL, 0, mbprof_handler, "A", "mbuf profiling statistics");
21576eeac1d9SJulian Elischer 
21586eeac1d9SJulian Elischer SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW,
21596eeac1d9SJulian Elischer 	    NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics");
21606eeac1d9SJulian Elischer #endif
21616eeac1d9SJulian Elischer 
2162