xref: /freebsd/sys/kern/uipc_mbuf.c (revision 38f0b757fd84d17d0fc24739a7cda160c4516d81)
1 /*-
2  * Copyright (c) 1982, 1986, 1988, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 4. Neither the name of the University nor the names of its contributors
14  *    may be used to endorse or promote products derived from this software
15  *    without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include "opt_param.h"
36 #include "opt_mbuf_stress_test.h"
37 #include "opt_mbuf_profiling.h"
38 
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/sysctl.h>
47 #include <sys/domain.h>
48 #include <sys/protosw.h>
49 #include <sys/uio.h>
50 
51 int	max_linkhdr;
52 int	max_protohdr;
53 int	max_hdr;
54 int	max_datalen;
55 #ifdef MBUF_STRESS_TEST
56 int	m_defragpackets;
57 int	m_defragbytes;
58 int	m_defraguseless;
59 int	m_defragfailure;
60 int	m_defragrandomfailures;
61 #endif
62 
63 /*
64  * sysctl(8) exported objects
65  */
66 SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
67 	   &max_linkhdr, 0, "Size of largest link layer header");
68 SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
69 	   &max_protohdr, 0, "Size of largest protocol layer header");
70 SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
71 	   &max_hdr, 0, "Size of largest link plus protocol header");
72 SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD,
73 	   &max_datalen, 0, "Minimum space left in mbuf after max_hdr");
74 #ifdef MBUF_STRESS_TEST
75 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
76 	   &m_defragpackets, 0, "");
77 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
78 	   &m_defragbytes, 0, "");
79 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
80 	   &m_defraguseless, 0, "");
81 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
82 	   &m_defragfailure, 0, "");
83 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
84 	   &m_defragrandomfailures, 0, "");
85 #endif
86 
87 /*
88  * Ensure the correct size of various mbuf parameters.  It could be off due
89  * to compiler-induced padding and alignment artifacts.
90  */
91 CTASSERT(sizeof(struct mbuf) == MSIZE);
92 CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
93 CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
94 
95 /*
96  * m_get2() allocates minimum mbuf that would fit "size" argument.
97  */
98 struct mbuf *
99 m_get2(int size, int how, short type, int flags)
100 {
101 	struct mb_args args;
102 	struct mbuf *m, *n;
103 
104 	args.flags = flags;
105 	args.type = type;
106 
107 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
108 		return (uma_zalloc_arg(zone_mbuf, &args, how));
109 	if (size <= MCLBYTES)
110 		return (uma_zalloc_arg(zone_pack, &args, how));
111 
112 	if (size > MJUMPAGESIZE)
113 		return (NULL);
114 
115 	m = uma_zalloc_arg(zone_mbuf, &args, how);
116 	if (m == NULL)
117 		return (NULL);
118 
119 	n = uma_zalloc_arg(zone_jumbop, m, how);
120 	if (n == NULL) {
121 		uma_zfree(zone_mbuf, m);
122 		return (NULL);
123 	}
124 
125 	return (m);
126 }
127 
128 /*
129  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
130  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
131  */
132 struct mbuf *
133 m_getjcl(int how, short type, int flags, int size)
134 {
135 	struct mb_args args;
136 	struct mbuf *m, *n;
137 	uma_zone_t zone;
138 
139 	if (size == MCLBYTES)
140 		return m_getcl(how, type, flags);
141 
142 	args.flags = flags;
143 	args.type = type;
144 
145 	m = uma_zalloc_arg(zone_mbuf, &args, how);
146 	if (m == NULL)
147 		return (NULL);
148 
149 	zone = m_getzone(size);
150 	n = uma_zalloc_arg(zone, m, how);
151 	if (n == NULL) {
152 		uma_zfree(zone_mbuf, m);
153 		return (NULL);
154 	}
155 	return (m);
156 }
157 
158 /*
159  * Allocate a given length worth of mbufs and/or clusters (whatever fits
160  * best) and return a pointer to the top of the allocated chain.  If an
161  * existing mbuf chain is provided, then we will append the new chain
162  * to the existing one but still return the top of the newly allocated
163  * chain.
164  */
165 struct mbuf *
166 m_getm2(struct mbuf *m, int len, int how, short type, int flags)
167 {
168 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
169 
170 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
171 
172 	/* Validate flags. */
173 	flags &= (M_PKTHDR | M_EOR);
174 
175 	/* Packet header mbuf must be first in chain. */
176 	if ((flags & M_PKTHDR) && m != NULL)
177 		flags &= ~M_PKTHDR;
178 
179 	/* Loop and append maximum sized mbufs to the chain tail. */
180 	while (len > 0) {
181 		if (len > MCLBYTES)
182 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
183 			    MJUMPAGESIZE);
184 		else if (len >= MINCLSIZE)
185 			mb = m_getcl(how, type, (flags & M_PKTHDR));
186 		else if (flags & M_PKTHDR)
187 			mb = m_gethdr(how, type);
188 		else
189 			mb = m_get(how, type);
190 
191 		/* Fail the whole operation if one mbuf can't be allocated. */
192 		if (mb == NULL) {
193 			if (nm != NULL)
194 				m_freem(nm);
195 			return (NULL);
196 		}
197 
198 		/* Book keeping. */
199 		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
200 			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
201 		if (mtail != NULL)
202 			mtail->m_next = mb;
203 		else
204 			nm = mb;
205 		mtail = mb;
206 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
207 	}
208 	if (flags & M_EOR)
209 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
210 
211 	/* If mbuf was supplied, append new chain to the end of it. */
212 	if (m != NULL) {
213 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
214 			;
215 		mtail->m_next = nm;
216 		mtail->m_flags &= ~M_EOR;
217 	} else
218 		m = nm;
219 
220 	return (m);
221 }
222 
223 /*
224  * Free an entire chain of mbufs and associated external buffers, if
225  * applicable.
226  */
227 void
228 m_freem(struct mbuf *mb)
229 {
230 
231 	while (mb != NULL)
232 		mb = m_free(mb);
233 }
234 
235 /*-
236  * Configure a provided mbuf to refer to the provided external storage
237  * buffer and setup a reference count for said buffer.  If the setting
238  * up of the reference count fails, the M_EXT bit will not be set.  If
239  * successfull, the M_EXT bit is set in the mbuf's flags.
240  *
241  * Arguments:
242  *    mb     The existing mbuf to which to attach the provided buffer.
243  *    buf    The address of the provided external storage buffer.
244  *    size   The size of the provided buffer.
245  *    freef  A pointer to a routine that is responsible for freeing the
246  *           provided external storage buffer.
247  *    args   A pointer to an argument structure (of any type) to be passed
248  *           to the provided freef routine (may be NULL).
249  *    flags  Any other flags to be passed to the provided mbuf.
250  *    type   The type that the external storage buffer should be
251  *           labeled with.
252  *
253  * Returns:
254  *    Nothing.
255  */
256 int
257 m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
258     int (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2,
259     int flags, int type, int wait)
260 {
261 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
262 
263 	if (type != EXT_EXTREF)
264 		mb->m_ext.ref_cnt = uma_zalloc(zone_ext_refcnt, wait);
265 
266 	if (mb->m_ext.ref_cnt == NULL)
267 		return (ENOMEM);
268 
269 	*(mb->m_ext.ref_cnt) = 1;
270 	mb->m_flags |= (M_EXT | flags);
271 	mb->m_ext.ext_buf = buf;
272 	mb->m_data = mb->m_ext.ext_buf;
273 	mb->m_ext.ext_size = size;
274 	mb->m_ext.ext_free = freef;
275 	mb->m_ext.ext_arg1 = arg1;
276 	mb->m_ext.ext_arg2 = arg2;
277 	mb->m_ext.ext_type = type;
278 	mb->m_ext.ext_flags = 0;
279 
280 	return (0);
281 }
282 
283 /*
284  * Non-directly-exported function to clean up after mbufs with M_EXT
285  * storage attached to them if the reference count hits 1.
286  */
287 void
288 mb_free_ext(struct mbuf *m)
289 {
290 	int skipmbuf;
291 
292 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
293 	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
294 
295 	/*
296 	 * check if the header is embedded in the cluster
297 	 */
298 	skipmbuf = (m->m_flags & M_NOFREE);
299 
300 	/* Free attached storage if this mbuf is the only reference to it. */
301 	if (*(m->m_ext.ref_cnt) == 1 ||
302 	    atomic_fetchadd_int(m->m_ext.ref_cnt, -1) == 1) {
303 		switch (m->m_ext.ext_type) {
304 		case EXT_PACKET:	/* The packet zone is special. */
305 			if (*(m->m_ext.ref_cnt) == 0)
306 				*(m->m_ext.ref_cnt) = 1;
307 			uma_zfree(zone_pack, m);
308 			return;		/* Job done. */
309 		case EXT_CLUSTER:
310 			uma_zfree(zone_clust, m->m_ext.ext_buf);
311 			break;
312 		case EXT_JUMBOP:
313 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
314 			break;
315 		case EXT_JUMBO9:
316 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
317 			break;
318 		case EXT_JUMBO16:
319 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
320 			break;
321 		case EXT_SFBUF:
322 		case EXT_NET_DRV:
323 		case EXT_MOD_TYPE:
324 		case EXT_DISPOSABLE:
325 			*(m->m_ext.ref_cnt) = 0;
326 			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
327 				m->m_ext.ref_cnt));
328 			/* FALLTHROUGH */
329 		case EXT_EXTREF:
330 			KASSERT(m->m_ext.ext_free != NULL,
331 				("%s: ext_free not set", __func__));
332 			(void)(*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1,
333 			    m->m_ext.ext_arg2);
334 			break;
335 		default:
336 			KASSERT(m->m_ext.ext_type == 0,
337 				("%s: unknown ext_type", __func__));
338 		}
339 	}
340 	if (skipmbuf)
341 		return;
342 
343 	/*
344 	 * Free this mbuf back to the mbuf zone with all m_ext
345 	 * information purged.
346 	 */
347 	m->m_ext.ext_buf = NULL;
348 	m->m_ext.ext_free = NULL;
349 	m->m_ext.ext_arg1 = NULL;
350 	m->m_ext.ext_arg2 = NULL;
351 	m->m_ext.ref_cnt = NULL;
352 	m->m_ext.ext_size = 0;
353 	m->m_ext.ext_type = 0;
354 	m->m_ext.ext_flags = 0;
355 	m->m_flags &= ~M_EXT;
356 	uma_zfree(zone_mbuf, m);
357 }
358 
359 /*
360  * Attach the cluster from *m to *n, set up m_ext in *n
361  * and bump the refcount of the cluster.
362  */
363 static void
364 mb_dupcl(struct mbuf *n, struct mbuf *m)
365 {
366 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
367 	KASSERT(m->m_ext.ref_cnt != NULL, ("%s: ref_cnt not set", __func__));
368 	KASSERT((n->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
369 
370 	if (*(m->m_ext.ref_cnt) == 1)
371 		*(m->m_ext.ref_cnt) += 1;
372 	else
373 		atomic_add_int(m->m_ext.ref_cnt, 1);
374 	n->m_ext.ext_buf = m->m_ext.ext_buf;
375 	n->m_ext.ext_free = m->m_ext.ext_free;
376 	n->m_ext.ext_arg1 = m->m_ext.ext_arg1;
377 	n->m_ext.ext_arg2 = m->m_ext.ext_arg2;
378 	n->m_ext.ext_size = m->m_ext.ext_size;
379 	n->m_ext.ref_cnt = m->m_ext.ref_cnt;
380 	n->m_ext.ext_type = m->m_ext.ext_type;
381 	n->m_ext.ext_flags = m->m_ext.ext_flags;
382 	n->m_flags |= M_EXT;
383 	n->m_flags |= m->m_flags & M_RDONLY;
384 }
385 
386 /*
387  * Clean up mbuf (chain) from any tags and packet headers.
388  * If "all" is set then the first mbuf in the chain will be
389  * cleaned too.
390  */
391 void
392 m_demote(struct mbuf *m0, int all)
393 {
394 	struct mbuf *m;
395 
396 	for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
397 		if (m->m_flags & M_PKTHDR) {
398 			m_tag_delete_chain(m, NULL);
399 			m->m_flags &= ~M_PKTHDR;
400 			bzero(&m->m_pkthdr, sizeof(struct pkthdr));
401 		}
402 		if (m != m0 && m->m_nextpkt != NULL) {
403 			KASSERT(m->m_nextpkt == NULL,
404 			    ("%s: m_nextpkt not NULL", __func__));
405 			m_freem(m->m_nextpkt);
406 			m->m_nextpkt = NULL;
407 		}
408 		m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_NOFREE);
409 	}
410 }
411 
412 /*
413  * Sanity checks on mbuf (chain) for use in KASSERT() and general
414  * debugging.
415  * Returns 0 or panics when bad and 1 on all tests passed.
416  * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
417  * blow up later.
418  */
419 int
420 m_sanity(struct mbuf *m0, int sanitize)
421 {
422 	struct mbuf *m;
423 	caddr_t a, b;
424 	int pktlen = 0;
425 
426 #ifdef INVARIANTS
427 #define	M_SANITY_ACTION(s)	panic("mbuf %p: " s, m)
428 #else
429 #define	M_SANITY_ACTION(s)	printf("mbuf %p: " s, m)
430 #endif
431 
432 	for (m = m0; m != NULL; m = m->m_next) {
433 		/*
434 		 * Basic pointer checks.  If any of these fails then some
435 		 * unrelated kernel memory before or after us is trashed.
436 		 * No way to recover from that.
437 		 */
438 		a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
439 			((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
440 			 (caddr_t)(&m->m_dat)) );
441 		b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
442 			((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
443 		if ((caddr_t)m->m_data < a)
444 			M_SANITY_ACTION("m_data outside mbuf data range left");
445 		if ((caddr_t)m->m_data > b)
446 			M_SANITY_ACTION("m_data outside mbuf data range right");
447 		if ((caddr_t)m->m_data + m->m_len > b)
448 			M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
449 
450 		/* m->m_nextpkt may only be set on first mbuf in chain. */
451 		if (m != m0 && m->m_nextpkt != NULL) {
452 			if (sanitize) {
453 				m_freem(m->m_nextpkt);
454 				m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
455 			} else
456 				M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
457 		}
458 
459 		/* packet length (not mbuf length!) calculation */
460 		if (m0->m_flags & M_PKTHDR)
461 			pktlen += m->m_len;
462 
463 		/* m_tags may only be attached to first mbuf in chain. */
464 		if (m != m0 && m->m_flags & M_PKTHDR &&
465 		    !SLIST_EMPTY(&m->m_pkthdr.tags)) {
466 			if (sanitize) {
467 				m_tag_delete_chain(m, NULL);
468 				/* put in 0xDEADC0DE perhaps? */
469 			} else
470 				M_SANITY_ACTION("m_tags on in-chain mbuf");
471 		}
472 
473 		/* M_PKTHDR may only be set on first mbuf in chain */
474 		if (m != m0 && m->m_flags & M_PKTHDR) {
475 			if (sanitize) {
476 				bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
477 				m->m_flags &= ~M_PKTHDR;
478 				/* put in 0xDEADCODE and leave hdr flag in */
479 			} else
480 				M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
481 		}
482 	}
483 	m = m0;
484 	if (pktlen && pktlen != m->m_pkthdr.len) {
485 		if (sanitize)
486 			m->m_pkthdr.len = 0;
487 		else
488 			M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
489 	}
490 	return 1;
491 
492 #undef	M_SANITY_ACTION
493 }
494 
495 
496 /*
497  * "Move" mbuf pkthdr from "from" to "to".
498  * "from" must have M_PKTHDR set, and "to" must be empty.
499  */
500 void
501 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
502 {
503 
504 #if 0
505 	/* see below for why these are not enabled */
506 	M_ASSERTPKTHDR(to);
507 	/* Note: with MAC, this may not be a good assertion. */
508 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
509 	    ("m_move_pkthdr: to has tags"));
510 #endif
511 #ifdef MAC
512 	/*
513 	 * XXXMAC: It could be this should also occur for non-MAC?
514 	 */
515 	if (to->m_flags & M_PKTHDR)
516 		m_tag_delete_chain(to, NULL);
517 #endif
518 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
519 	if ((to->m_flags & M_EXT) == 0)
520 		to->m_data = to->m_pktdat;
521 	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
522 	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
523 	from->m_flags &= ~M_PKTHDR;
524 }
525 
526 /*
527  * Duplicate "from"'s mbuf pkthdr in "to".
528  * "from" must have M_PKTHDR set, and "to" must be empty.
529  * In particular, this does a deep copy of the packet tags.
530  */
531 int
532 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
533 {
534 
535 #if 0
536 	/*
537 	 * The mbuf allocator only initializes the pkthdr
538 	 * when the mbuf is allocated with m_gethdr(). Many users
539 	 * (e.g. m_copy*, m_prepend) use m_get() and then
540 	 * smash the pkthdr as needed causing these
541 	 * assertions to trip.  For now just disable them.
542 	 */
543 	M_ASSERTPKTHDR(to);
544 	/* Note: with MAC, this may not be a good assertion. */
545 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
546 #endif
547 	MBUF_CHECKSLEEP(how);
548 #ifdef MAC
549 	if (to->m_flags & M_PKTHDR)
550 		m_tag_delete_chain(to, NULL);
551 #endif
552 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
553 	if ((to->m_flags & M_EXT) == 0)
554 		to->m_data = to->m_pktdat;
555 	to->m_pkthdr = from->m_pkthdr;
556 	SLIST_INIT(&to->m_pkthdr.tags);
557 	return (m_tag_copy_chain(to, from, how));
558 }
559 
560 /*
561  * Lesser-used path for M_PREPEND:
562  * allocate new mbuf to prepend to chain,
563  * copy junk along.
564  */
565 struct mbuf *
566 m_prepend(struct mbuf *m, int len, int how)
567 {
568 	struct mbuf *mn;
569 
570 	if (m->m_flags & M_PKTHDR)
571 		mn = m_gethdr(how, m->m_type);
572 	else
573 		mn = m_get(how, m->m_type);
574 	if (mn == NULL) {
575 		m_freem(m);
576 		return (NULL);
577 	}
578 	if (m->m_flags & M_PKTHDR)
579 		m_move_pkthdr(mn, m);
580 	mn->m_next = m;
581 	m = mn;
582 	if(m->m_flags & M_PKTHDR) {
583 		if (len < MHLEN)
584 			MH_ALIGN(m, len);
585 	} else {
586 		if (len < MLEN)
587 			M_ALIGN(m, len);
588 	}
589 	m->m_len = len;
590 	return (m);
591 }
592 
593 /*
594  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
595  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
596  * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
597  * Note that the copy is read-only, because clusters are not copied,
598  * only their reference counts are incremented.
599  */
600 struct mbuf *
601 m_copym(struct mbuf *m, int off0, int len, int wait)
602 {
603 	struct mbuf *n, **np;
604 	int off = off0;
605 	struct mbuf *top;
606 	int copyhdr = 0;
607 
608 	KASSERT(off >= 0, ("m_copym, negative off %d", off));
609 	KASSERT(len >= 0, ("m_copym, negative len %d", len));
610 	MBUF_CHECKSLEEP(wait);
611 	if (off == 0 && m->m_flags & M_PKTHDR)
612 		copyhdr = 1;
613 	while (off > 0) {
614 		KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
615 		if (off < m->m_len)
616 			break;
617 		off -= m->m_len;
618 		m = m->m_next;
619 	}
620 	np = &top;
621 	top = 0;
622 	while (len > 0) {
623 		if (m == NULL) {
624 			KASSERT(len == M_COPYALL,
625 			    ("m_copym, length > size of mbuf chain"));
626 			break;
627 		}
628 		if (copyhdr)
629 			n = m_gethdr(wait, m->m_type);
630 		else
631 			n = m_get(wait, m->m_type);
632 		*np = n;
633 		if (n == NULL)
634 			goto nospace;
635 		if (copyhdr) {
636 			if (!m_dup_pkthdr(n, m, wait))
637 				goto nospace;
638 			if (len == M_COPYALL)
639 				n->m_pkthdr.len -= off0;
640 			else
641 				n->m_pkthdr.len = len;
642 			copyhdr = 0;
643 		}
644 		n->m_len = min(len, m->m_len - off);
645 		if (m->m_flags & M_EXT) {
646 			n->m_data = m->m_data + off;
647 			mb_dupcl(n, m);
648 		} else
649 			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
650 			    (u_int)n->m_len);
651 		if (len != M_COPYALL)
652 			len -= n->m_len;
653 		off = 0;
654 		m = m->m_next;
655 		np = &n->m_next;
656 	}
657 
658 	return (top);
659 nospace:
660 	m_freem(top);
661 	return (NULL);
662 }
663 
664 /*
665  * Returns mbuf chain with new head for the prepending case.
666  * Copies from mbuf (chain) n from off for len to mbuf (chain) m
667  * either prepending or appending the data.
668  * The resulting mbuf (chain) m is fully writeable.
669  * m is destination (is made writeable)
670  * n is source, off is offset in source, len is len from offset
671  * dir, 0 append, 1 prepend
672  * how, wait or nowait
673  */
674 
675 static int
676 m_bcopyxxx(void *s, void *t, u_int len)
677 {
678 	bcopy(s, t, (size_t)len);
679 	return 0;
680 }
681 
682 struct mbuf *
683 m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
684     int prep, int how)
685 {
686 	struct mbuf *mm, *x, *z, *prev = NULL;
687 	caddr_t p;
688 	int i, nlen = 0;
689 	caddr_t buf[MLEN];
690 
691 	KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
692 	KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
693 	KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
694 	KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
695 
696 	mm = m;
697 	if (!prep) {
698 		while(mm->m_next) {
699 			prev = mm;
700 			mm = mm->m_next;
701 		}
702 	}
703 	for (z = n; z != NULL; z = z->m_next)
704 		nlen += z->m_len;
705 	if (len == M_COPYALL)
706 		len = nlen - off;
707 	if (off + len > nlen || len < 1)
708 		return NULL;
709 
710 	if (!M_WRITABLE(mm)) {
711 		/* XXX: Use proper m_xxx function instead. */
712 		x = m_getcl(how, MT_DATA, mm->m_flags);
713 		if (x == NULL)
714 			return NULL;
715 		bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
716 		p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
717 		x->m_data = p;
718 		mm->m_next = NULL;
719 		if (mm != m)
720 			prev->m_next = x;
721 		m_free(mm);
722 		mm = x;
723 	}
724 
725 	/*
726 	 * Append/prepend the data.  Allocating mbufs as necessary.
727 	 */
728 	/* Shortcut if enough free space in first/last mbuf. */
729 	if (!prep && M_TRAILINGSPACE(mm) >= len) {
730 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
731 			 mm->m_len);
732 		mm->m_len += len;
733 		mm->m_pkthdr.len += len;
734 		return m;
735 	}
736 	if (prep && M_LEADINGSPACE(mm) >= len) {
737 		mm->m_data = mtod(mm, caddr_t) - len;
738 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
739 		mm->m_len += len;
740 		mm->m_pkthdr.len += len;
741 		return mm;
742 	}
743 
744 	/* Expand first/last mbuf to cluster if possible. */
745 	if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
746 		bcopy(mm->m_data, &buf, mm->m_len);
747 		m_clget(mm, how);
748 		if (!(mm->m_flags & M_EXT))
749 			return NULL;
750 		bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
751 		mm->m_data = mm->m_ext.ext_buf;
752 	}
753 	if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
754 		bcopy(mm->m_data, &buf, mm->m_len);
755 		m_clget(mm, how);
756 		if (!(mm->m_flags & M_EXT))
757 			return NULL;
758 		bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
759 		       mm->m_ext.ext_size - mm->m_len, mm->m_len);
760 		mm->m_data = (caddr_t)mm->m_ext.ext_buf +
761 			      mm->m_ext.ext_size - mm->m_len;
762 	}
763 
764 	/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
765 	if (!prep && len > M_TRAILINGSPACE(mm)) {
766 		if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
767 			return NULL;
768 	}
769 	if (prep && len > M_LEADINGSPACE(mm)) {
770 		if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
771 			return NULL;
772 		i = 0;
773 		for (x = z; x != NULL; x = x->m_next) {
774 			i += x->m_flags & M_EXT ? x->m_ext.ext_size :
775 			      (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
776 			if (!x->m_next)
777 				break;
778 		}
779 		z->m_data += i - len;
780 		m_move_pkthdr(mm, z);
781 		x->m_next = mm;
782 		mm = z;
783 	}
784 
785 	/* Seek to start position in source mbuf. Optimization for long chains. */
786 	while (off > 0) {
787 		if (off < n->m_len)
788 			break;
789 		off -= n->m_len;
790 		n = n->m_next;
791 	}
792 
793 	/* Copy data into target mbuf. */
794 	z = mm;
795 	while (len > 0) {
796 		KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
797 		i = M_TRAILINGSPACE(z);
798 		m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
799 		z->m_len += i;
800 		/* fixup pkthdr.len if necessary */
801 		if ((prep ? mm : m)->m_flags & M_PKTHDR)
802 			(prep ? mm : m)->m_pkthdr.len += i;
803 		off += i;
804 		len -= i;
805 		z = z->m_next;
806 	}
807 	return (prep ? mm : m);
808 }
809 
810 /*
811  * Copy an entire packet, including header (which must be present).
812  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
813  * Note that the copy is read-only, because clusters are not copied,
814  * only their reference counts are incremented.
815  * Preserve alignment of the first mbuf so if the creator has left
816  * some room at the beginning (e.g. for inserting protocol headers)
817  * the copies still have the room available.
818  */
819 struct mbuf *
820 m_copypacket(struct mbuf *m, int how)
821 {
822 	struct mbuf *top, *n, *o;
823 
824 	MBUF_CHECKSLEEP(how);
825 	n = m_get(how, m->m_type);
826 	top = n;
827 	if (n == NULL)
828 		goto nospace;
829 
830 	if (!m_dup_pkthdr(n, m, how))
831 		goto nospace;
832 	n->m_len = m->m_len;
833 	if (m->m_flags & M_EXT) {
834 		n->m_data = m->m_data;
835 		mb_dupcl(n, m);
836 	} else {
837 		n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
838 		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
839 	}
840 
841 	m = m->m_next;
842 	while (m) {
843 		o = m_get(how, m->m_type);
844 		if (o == NULL)
845 			goto nospace;
846 
847 		n->m_next = o;
848 		n = n->m_next;
849 
850 		n->m_len = m->m_len;
851 		if (m->m_flags & M_EXT) {
852 			n->m_data = m->m_data;
853 			mb_dupcl(n, m);
854 		} else {
855 			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
856 		}
857 
858 		m = m->m_next;
859 	}
860 	return top;
861 nospace:
862 	m_freem(top);
863 	return (NULL);
864 }
865 
866 /*
867  * Copy data from an mbuf chain starting "off" bytes from the beginning,
868  * continuing for "len" bytes, into the indicated buffer.
869  */
870 void
871 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
872 {
873 	u_int count;
874 
875 	KASSERT(off >= 0, ("m_copydata, negative off %d", off));
876 	KASSERT(len >= 0, ("m_copydata, negative len %d", len));
877 	while (off > 0) {
878 		KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
879 		if (off < m->m_len)
880 			break;
881 		off -= m->m_len;
882 		m = m->m_next;
883 	}
884 	while (len > 0) {
885 		KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
886 		count = min(m->m_len - off, len);
887 		bcopy(mtod(m, caddr_t) + off, cp, count);
888 		len -= count;
889 		cp += count;
890 		off = 0;
891 		m = m->m_next;
892 	}
893 }
894 
895 /*
896  * Copy a packet header mbuf chain into a completely new chain, including
897  * copying any mbuf clusters.  Use this instead of m_copypacket() when
898  * you need a writable copy of an mbuf chain.
899  */
900 struct mbuf *
901 m_dup(struct mbuf *m, int how)
902 {
903 	struct mbuf **p, *top = NULL;
904 	int remain, moff, nsize;
905 
906 	MBUF_CHECKSLEEP(how);
907 	/* Sanity check */
908 	if (m == NULL)
909 		return (NULL);
910 	M_ASSERTPKTHDR(m);
911 
912 	/* While there's more data, get a new mbuf, tack it on, and fill it */
913 	remain = m->m_pkthdr.len;
914 	moff = 0;
915 	p = &top;
916 	while (remain > 0 || top == NULL) {	/* allow m->m_pkthdr.len == 0 */
917 		struct mbuf *n;
918 
919 		/* Get the next new mbuf */
920 		if (remain >= MINCLSIZE) {
921 			n = m_getcl(how, m->m_type, 0);
922 			nsize = MCLBYTES;
923 		} else {
924 			n = m_get(how, m->m_type);
925 			nsize = MLEN;
926 		}
927 		if (n == NULL)
928 			goto nospace;
929 
930 		if (top == NULL) {		/* First one, must be PKTHDR */
931 			if (!m_dup_pkthdr(n, m, how)) {
932 				m_free(n);
933 				goto nospace;
934 			}
935 			if ((n->m_flags & M_EXT) == 0)
936 				nsize = MHLEN;
937 		}
938 		n->m_len = 0;
939 
940 		/* Link it into the new chain */
941 		*p = n;
942 		p = &n->m_next;
943 
944 		/* Copy data from original mbuf(s) into new mbuf */
945 		while (n->m_len < nsize && m != NULL) {
946 			int chunk = min(nsize - n->m_len, m->m_len - moff);
947 
948 			bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
949 			moff += chunk;
950 			n->m_len += chunk;
951 			remain -= chunk;
952 			if (moff == m->m_len) {
953 				m = m->m_next;
954 				moff = 0;
955 			}
956 		}
957 
958 		/* Check correct total mbuf length */
959 		KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
960 		    	("%s: bogus m_pkthdr.len", __func__));
961 	}
962 	return (top);
963 
964 nospace:
965 	m_freem(top);
966 	return (NULL);
967 }
968 
969 /*
970  * Concatenate mbuf chain n to m.
971  * Both chains must be of the same type (e.g. MT_DATA).
972  * Any m_pkthdr is not updated.
973  */
974 void
975 m_cat(struct mbuf *m, struct mbuf *n)
976 {
977 	while (m->m_next)
978 		m = m->m_next;
979 	while (n) {
980 		if (!M_WRITABLE(m) ||
981 		    M_TRAILINGSPACE(m) < n->m_len) {
982 			/* just join the two chains */
983 			m->m_next = n;
984 			return;
985 		}
986 		/* splat the data from one into the other */
987 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
988 		    (u_int)n->m_len);
989 		m->m_len += n->m_len;
990 		n = m_free(n);
991 	}
992 }
993 
994 void
995 m_adj(struct mbuf *mp, int req_len)
996 {
997 	int len = req_len;
998 	struct mbuf *m;
999 	int count;
1000 
1001 	if ((m = mp) == NULL)
1002 		return;
1003 	if (len >= 0) {
1004 		/*
1005 		 * Trim from head.
1006 		 */
1007 		while (m != NULL && len > 0) {
1008 			if (m->m_len <= len) {
1009 				len -= m->m_len;
1010 				m->m_len = 0;
1011 				m = m->m_next;
1012 			} else {
1013 				m->m_len -= len;
1014 				m->m_data += len;
1015 				len = 0;
1016 			}
1017 		}
1018 		if (mp->m_flags & M_PKTHDR)
1019 			mp->m_pkthdr.len -= (req_len - len);
1020 	} else {
1021 		/*
1022 		 * Trim from tail.  Scan the mbuf chain,
1023 		 * calculating its length and finding the last mbuf.
1024 		 * If the adjustment only affects this mbuf, then just
1025 		 * adjust and return.  Otherwise, rescan and truncate
1026 		 * after the remaining size.
1027 		 */
1028 		len = -len;
1029 		count = 0;
1030 		for (;;) {
1031 			count += m->m_len;
1032 			if (m->m_next == (struct mbuf *)0)
1033 				break;
1034 			m = m->m_next;
1035 		}
1036 		if (m->m_len >= len) {
1037 			m->m_len -= len;
1038 			if (mp->m_flags & M_PKTHDR)
1039 				mp->m_pkthdr.len -= len;
1040 			return;
1041 		}
1042 		count -= len;
1043 		if (count < 0)
1044 			count = 0;
1045 		/*
1046 		 * Correct length for chain is "count".
1047 		 * Find the mbuf with last data, adjust its length,
1048 		 * and toss data from remaining mbufs on chain.
1049 		 */
1050 		m = mp;
1051 		if (m->m_flags & M_PKTHDR)
1052 			m->m_pkthdr.len = count;
1053 		for (; m; m = m->m_next) {
1054 			if (m->m_len >= count) {
1055 				m->m_len = count;
1056 				if (m->m_next != NULL) {
1057 					m_freem(m->m_next);
1058 					m->m_next = NULL;
1059 				}
1060 				break;
1061 			}
1062 			count -= m->m_len;
1063 		}
1064 	}
1065 }
1066 
1067 /*
1068  * Rearange an mbuf chain so that len bytes are contiguous
1069  * and in the data area of an mbuf (so that mtod will work
1070  * for a structure of size len).  Returns the resulting
1071  * mbuf chain on success, frees it and returns null on failure.
1072  * If there is room, it will add up to max_protohdr-len extra bytes to the
1073  * contiguous region in an attempt to avoid being called next time.
1074  */
1075 struct mbuf *
1076 m_pullup(struct mbuf *n, int len)
1077 {
1078 	struct mbuf *m;
1079 	int count;
1080 	int space;
1081 
1082 	/*
1083 	 * If first mbuf has no cluster, and has room for len bytes
1084 	 * without shifting current data, pullup into it,
1085 	 * otherwise allocate a new mbuf to prepend to the chain.
1086 	 */
1087 	if ((n->m_flags & M_EXT) == 0 &&
1088 	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1089 		if (n->m_len >= len)
1090 			return (n);
1091 		m = n;
1092 		n = n->m_next;
1093 		len -= m->m_len;
1094 	} else {
1095 		if (len > MHLEN)
1096 			goto bad;
1097 		m = m_get(M_NOWAIT, n->m_type);
1098 		if (m == NULL)
1099 			goto bad;
1100 		if (n->m_flags & M_PKTHDR)
1101 			m_move_pkthdr(m, n);
1102 	}
1103 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1104 	do {
1105 		count = min(min(max(len, max_protohdr), space), n->m_len);
1106 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1107 		  (u_int)count);
1108 		len -= count;
1109 		m->m_len += count;
1110 		n->m_len -= count;
1111 		space -= count;
1112 		if (n->m_len)
1113 			n->m_data += count;
1114 		else
1115 			n = m_free(n);
1116 	} while (len > 0 && n);
1117 	if (len > 0) {
1118 		(void) m_free(m);
1119 		goto bad;
1120 	}
1121 	m->m_next = n;
1122 	return (m);
1123 bad:
1124 	m_freem(n);
1125 	return (NULL);
1126 }
1127 
1128 /*
1129  * Like m_pullup(), except a new mbuf is always allocated, and we allow
1130  * the amount of empty space before the data in the new mbuf to be specified
1131  * (in the event that the caller expects to prepend later).
1132  */
1133 int MSFail;
1134 
1135 struct mbuf *
1136 m_copyup(struct mbuf *n, int len, int dstoff)
1137 {
1138 	struct mbuf *m;
1139 	int count, space;
1140 
1141 	if (len > (MHLEN - dstoff))
1142 		goto bad;
1143 	m = m_get(M_NOWAIT, n->m_type);
1144 	if (m == NULL)
1145 		goto bad;
1146 	if (n->m_flags & M_PKTHDR)
1147 		m_move_pkthdr(m, n);
1148 	m->m_data += dstoff;
1149 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1150 	do {
1151 		count = min(min(max(len, max_protohdr), space), n->m_len);
1152 		memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
1153 		    (unsigned)count);
1154 		len -= count;
1155 		m->m_len += count;
1156 		n->m_len -= count;
1157 		space -= count;
1158 		if (n->m_len)
1159 			n->m_data += count;
1160 		else
1161 			n = m_free(n);
1162 	} while (len > 0 && n);
1163 	if (len > 0) {
1164 		(void) m_free(m);
1165 		goto bad;
1166 	}
1167 	m->m_next = n;
1168 	return (m);
1169  bad:
1170 	m_freem(n);
1171 	MSFail++;
1172 	return (NULL);
1173 }
1174 
1175 /*
1176  * Partition an mbuf chain in two pieces, returning the tail --
1177  * all but the first len0 bytes.  In case of failure, it returns NULL and
1178  * attempts to restore the chain to its original state.
1179  *
1180  * Note that the resulting mbufs might be read-only, because the new
1181  * mbuf can end up sharing an mbuf cluster with the original mbuf if
1182  * the "breaking point" happens to lie within a cluster mbuf. Use the
1183  * M_WRITABLE() macro to check for this case.
1184  */
1185 struct mbuf *
1186 m_split(struct mbuf *m0, int len0, int wait)
1187 {
1188 	struct mbuf *m, *n;
1189 	u_int len = len0, remain;
1190 
1191 	MBUF_CHECKSLEEP(wait);
1192 	for (m = m0; m && len > m->m_len; m = m->m_next)
1193 		len -= m->m_len;
1194 	if (m == NULL)
1195 		return (NULL);
1196 	remain = m->m_len - len;
1197 	if (m0->m_flags & M_PKTHDR && remain == 0) {
1198 		n = m_gethdr(wait, m0->m_type);
1199 		if (n == NULL)
1200 			return (NULL);
1201 		n->m_next = m->m_next;
1202 		m->m_next = NULL;
1203 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1204 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1205 		m0->m_pkthdr.len = len0;
1206 		return (n);
1207 	} else if (m0->m_flags & M_PKTHDR) {
1208 		n = m_gethdr(wait, m0->m_type);
1209 		if (n == NULL)
1210 			return (NULL);
1211 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1212 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1213 		m0->m_pkthdr.len = len0;
1214 		if (m->m_flags & M_EXT)
1215 			goto extpacket;
1216 		if (remain > MHLEN) {
1217 			/* m can't be the lead packet */
1218 			MH_ALIGN(n, 0);
1219 			n->m_next = m_split(m, len, wait);
1220 			if (n->m_next == NULL) {
1221 				(void) m_free(n);
1222 				return (NULL);
1223 			} else {
1224 				n->m_len = 0;
1225 				return (n);
1226 			}
1227 		} else
1228 			MH_ALIGN(n, remain);
1229 	} else if (remain == 0) {
1230 		n = m->m_next;
1231 		m->m_next = NULL;
1232 		return (n);
1233 	} else {
1234 		n = m_get(wait, m->m_type);
1235 		if (n == NULL)
1236 			return (NULL);
1237 		M_ALIGN(n, remain);
1238 	}
1239 extpacket:
1240 	if (m->m_flags & M_EXT) {
1241 		n->m_data = m->m_data + len;
1242 		mb_dupcl(n, m);
1243 	} else {
1244 		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1245 	}
1246 	n->m_len = remain;
1247 	m->m_len = len;
1248 	n->m_next = m->m_next;
1249 	m->m_next = NULL;
1250 	return (n);
1251 }
1252 /*
1253  * Routine to copy from device local memory into mbufs.
1254  * Note that `off' argument is offset into first mbuf of target chain from
1255  * which to begin copying the data to.
1256  */
1257 struct mbuf *
1258 m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1259     void (*copy)(char *from, caddr_t to, u_int len))
1260 {
1261 	struct mbuf *m;
1262 	struct mbuf *top = NULL, **mp = &top;
1263 	int len;
1264 
1265 	if (off < 0 || off > MHLEN)
1266 		return (NULL);
1267 
1268 	while (totlen > 0) {
1269 		if (top == NULL) {	/* First one, must be PKTHDR */
1270 			if (totlen + off >= MINCLSIZE) {
1271 				m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1272 				len = MCLBYTES;
1273 			} else {
1274 				m = m_gethdr(M_NOWAIT, MT_DATA);
1275 				len = MHLEN;
1276 
1277 				/* Place initial small packet/header at end of mbuf */
1278 				if (m && totlen + off + max_linkhdr <= MLEN) {
1279 					m->m_data += max_linkhdr;
1280 					len -= max_linkhdr;
1281 				}
1282 			}
1283 			if (m == NULL)
1284 				return NULL;
1285 			m->m_pkthdr.rcvif = ifp;
1286 			m->m_pkthdr.len = totlen;
1287 		} else {
1288 			if (totlen + off >= MINCLSIZE) {
1289 				m = m_getcl(M_NOWAIT, MT_DATA, 0);
1290 				len = MCLBYTES;
1291 			} else {
1292 				m = m_get(M_NOWAIT, MT_DATA);
1293 				len = MLEN;
1294 			}
1295 			if (m == NULL) {
1296 				m_freem(top);
1297 				return NULL;
1298 			}
1299 		}
1300 		if (off) {
1301 			m->m_data += off;
1302 			len -= off;
1303 			off = 0;
1304 		}
1305 		m->m_len = len = min(totlen, len);
1306 		if (copy)
1307 			copy(buf, mtod(m, caddr_t), (u_int)len);
1308 		else
1309 			bcopy(buf, mtod(m, caddr_t), (u_int)len);
1310 		buf += len;
1311 		*mp = m;
1312 		mp = &m->m_next;
1313 		totlen -= len;
1314 	}
1315 	return (top);
1316 }
1317 
1318 /*
1319  * Copy data from a buffer back into the indicated mbuf chain,
1320  * starting "off" bytes from the beginning, extending the mbuf
1321  * chain if necessary.
1322  */
1323 void
1324 m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
1325 {
1326 	int mlen;
1327 	struct mbuf *m = m0, *n;
1328 	int totlen = 0;
1329 
1330 	if (m0 == NULL)
1331 		return;
1332 	while (off > (mlen = m->m_len)) {
1333 		off -= mlen;
1334 		totlen += mlen;
1335 		if (m->m_next == NULL) {
1336 			n = m_get(M_NOWAIT, m->m_type);
1337 			if (n == NULL)
1338 				goto out;
1339 			bzero(mtod(n, caddr_t), MLEN);
1340 			n->m_len = min(MLEN, len + off);
1341 			m->m_next = n;
1342 		}
1343 		m = m->m_next;
1344 	}
1345 	while (len > 0) {
1346 		if (m->m_next == NULL && (len > m->m_len - off)) {
1347 			m->m_len += min(len - (m->m_len - off),
1348 			    M_TRAILINGSPACE(m));
1349 		}
1350 		mlen = min (m->m_len - off, len);
1351 		bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
1352 		cp += mlen;
1353 		len -= mlen;
1354 		mlen += off;
1355 		off = 0;
1356 		totlen += mlen;
1357 		if (len == 0)
1358 			break;
1359 		if (m->m_next == NULL) {
1360 			n = m_get(M_NOWAIT, m->m_type);
1361 			if (n == NULL)
1362 				break;
1363 			n->m_len = min(MLEN, len);
1364 			m->m_next = n;
1365 		}
1366 		m = m->m_next;
1367 	}
1368 out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1369 		m->m_pkthdr.len = totlen;
1370 }
1371 
1372 /*
1373  * Append the specified data to the indicated mbuf chain,
1374  * Extend the mbuf chain if the new data does not fit in
1375  * existing space.
1376  *
1377  * Return 1 if able to complete the job; otherwise 0.
1378  */
1379 int
1380 m_append(struct mbuf *m0, int len, c_caddr_t cp)
1381 {
1382 	struct mbuf *m, *n;
1383 	int remainder, space;
1384 
1385 	for (m = m0; m->m_next != NULL; m = m->m_next)
1386 		;
1387 	remainder = len;
1388 	space = M_TRAILINGSPACE(m);
1389 	if (space > 0) {
1390 		/*
1391 		 * Copy into available space.
1392 		 */
1393 		if (space > remainder)
1394 			space = remainder;
1395 		bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
1396 		m->m_len += space;
1397 		cp += space, remainder -= space;
1398 	}
1399 	while (remainder > 0) {
1400 		/*
1401 		 * Allocate a new mbuf; could check space
1402 		 * and allocate a cluster instead.
1403 		 */
1404 		n = m_get(M_NOWAIT, m->m_type);
1405 		if (n == NULL)
1406 			break;
1407 		n->m_len = min(MLEN, remainder);
1408 		bcopy(cp, mtod(n, caddr_t), n->m_len);
1409 		cp += n->m_len, remainder -= n->m_len;
1410 		m->m_next = n;
1411 		m = n;
1412 	}
1413 	if (m0->m_flags & M_PKTHDR)
1414 		m0->m_pkthdr.len += len - remainder;
1415 	return (remainder == 0);
1416 }
1417 
1418 /*
1419  * Apply function f to the data in an mbuf chain starting "off" bytes from
1420  * the beginning, continuing for "len" bytes.
1421  */
1422 int
1423 m_apply(struct mbuf *m, int off, int len,
1424     int (*f)(void *, void *, u_int), void *arg)
1425 {
1426 	u_int count;
1427 	int rval;
1428 
1429 	KASSERT(off >= 0, ("m_apply, negative off %d", off));
1430 	KASSERT(len >= 0, ("m_apply, negative len %d", len));
1431 	while (off > 0) {
1432 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1433 		if (off < m->m_len)
1434 			break;
1435 		off -= m->m_len;
1436 		m = m->m_next;
1437 	}
1438 	while (len > 0) {
1439 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1440 		count = min(m->m_len - off, len);
1441 		rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1442 		if (rval)
1443 			return (rval);
1444 		len -= count;
1445 		off = 0;
1446 		m = m->m_next;
1447 	}
1448 	return (0);
1449 }
1450 
1451 /*
1452  * Return a pointer to mbuf/offset of location in mbuf chain.
1453  */
1454 struct mbuf *
1455 m_getptr(struct mbuf *m, int loc, int *off)
1456 {
1457 
1458 	while (loc >= 0) {
1459 		/* Normal end of search. */
1460 		if (m->m_len > loc) {
1461 			*off = loc;
1462 			return (m);
1463 		} else {
1464 			loc -= m->m_len;
1465 			if (m->m_next == NULL) {
1466 				if (loc == 0) {
1467 					/* Point at the end of valid data. */
1468 					*off = m->m_len;
1469 					return (m);
1470 				}
1471 				return (NULL);
1472 			}
1473 			m = m->m_next;
1474 		}
1475 	}
1476 	return (NULL);
1477 }
1478 
1479 void
1480 m_print(const struct mbuf *m, int maxlen)
1481 {
1482 	int len;
1483 	int pdata;
1484 	const struct mbuf *m2;
1485 
1486 	if (m == NULL) {
1487 		printf("mbuf: %p\n", m);
1488 		return;
1489 	}
1490 
1491 	if (m->m_flags & M_PKTHDR)
1492 		len = m->m_pkthdr.len;
1493 	else
1494 		len = -1;
1495 	m2 = m;
1496 	while (m2 != NULL && (len == -1 || len)) {
1497 		pdata = m2->m_len;
1498 		if (maxlen != -1 && pdata > maxlen)
1499 			pdata = maxlen;
1500 		printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
1501 		    m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
1502 		    "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
1503 		    "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
1504 		if (pdata)
1505 			printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
1506 		if (len != -1)
1507 			len -= m2->m_len;
1508 		m2 = m2->m_next;
1509 	}
1510 	if (len > 0)
1511 		printf("%d bytes unaccounted for.\n", len);
1512 	return;
1513 }
1514 
1515 u_int
1516 m_fixhdr(struct mbuf *m0)
1517 {
1518 	u_int len;
1519 
1520 	len = m_length(m0, NULL);
1521 	m0->m_pkthdr.len = len;
1522 	return (len);
1523 }
1524 
1525 u_int
1526 m_length(struct mbuf *m0, struct mbuf **last)
1527 {
1528 	struct mbuf *m;
1529 	u_int len;
1530 
1531 	len = 0;
1532 	for (m = m0; m != NULL; m = m->m_next) {
1533 		len += m->m_len;
1534 		if (m->m_next == NULL)
1535 			break;
1536 	}
1537 	if (last != NULL)
1538 		*last = m;
1539 	return (len);
1540 }
1541 
1542 /*
1543  * Defragment a mbuf chain, returning the shortest possible
1544  * chain of mbufs and clusters.  If allocation fails and
1545  * this cannot be completed, NULL will be returned, but
1546  * the passed in chain will be unchanged.  Upon success,
1547  * the original chain will be freed, and the new chain
1548  * will be returned.
1549  *
1550  * If a non-packet header is passed in, the original
1551  * mbuf (chain?) will be returned unharmed.
1552  */
1553 struct mbuf *
1554 m_defrag(struct mbuf *m0, int how)
1555 {
1556 	struct mbuf *m_new = NULL, *m_final = NULL;
1557 	int progress = 0, length;
1558 
1559 	MBUF_CHECKSLEEP(how);
1560 	if (!(m0->m_flags & M_PKTHDR))
1561 		return (m0);
1562 
1563 	m_fixhdr(m0); /* Needed sanity check */
1564 
1565 #ifdef MBUF_STRESS_TEST
1566 	if (m_defragrandomfailures) {
1567 		int temp = arc4random() & 0xff;
1568 		if (temp == 0xba)
1569 			goto nospace;
1570 	}
1571 #endif
1572 
1573 	if (m0->m_pkthdr.len > MHLEN)
1574 		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1575 	else
1576 		m_final = m_gethdr(how, MT_DATA);
1577 
1578 	if (m_final == NULL)
1579 		goto nospace;
1580 
1581 	if (m_dup_pkthdr(m_final, m0, how) == 0)
1582 		goto nospace;
1583 
1584 	m_new = m_final;
1585 
1586 	while (progress < m0->m_pkthdr.len) {
1587 		length = m0->m_pkthdr.len - progress;
1588 		if (length > MCLBYTES)
1589 			length = MCLBYTES;
1590 
1591 		if (m_new == NULL) {
1592 			if (length > MLEN)
1593 				m_new = m_getcl(how, MT_DATA, 0);
1594 			else
1595 				m_new = m_get(how, MT_DATA);
1596 			if (m_new == NULL)
1597 				goto nospace;
1598 		}
1599 
1600 		m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1601 		progress += length;
1602 		m_new->m_len = length;
1603 		if (m_new != m_final)
1604 			m_cat(m_final, m_new);
1605 		m_new = NULL;
1606 	}
1607 #ifdef MBUF_STRESS_TEST
1608 	if (m0->m_next == NULL)
1609 		m_defraguseless++;
1610 #endif
1611 	m_freem(m0);
1612 	m0 = m_final;
1613 #ifdef MBUF_STRESS_TEST
1614 	m_defragpackets++;
1615 	m_defragbytes += m0->m_pkthdr.len;
1616 #endif
1617 	return (m0);
1618 nospace:
1619 #ifdef MBUF_STRESS_TEST
1620 	m_defragfailure++;
1621 #endif
1622 	if (m_final)
1623 		m_freem(m_final);
1624 	return (NULL);
1625 }
1626 
1627 /*
1628  * Defragment an mbuf chain, returning at most maxfrags separate
1629  * mbufs+clusters.  If this is not possible NULL is returned and
1630  * the original mbuf chain is left in it's present (potentially
1631  * modified) state.  We use two techniques: collapsing consecutive
1632  * mbufs and replacing consecutive mbufs by a cluster.
1633  *
1634  * NB: this should really be named m_defrag but that name is taken
1635  */
1636 struct mbuf *
1637 m_collapse(struct mbuf *m0, int how, int maxfrags)
1638 {
1639 	struct mbuf *m, *n, *n2, **prev;
1640 	u_int curfrags;
1641 
1642 	/*
1643 	 * Calculate the current number of frags.
1644 	 */
1645 	curfrags = 0;
1646 	for (m = m0; m != NULL; m = m->m_next)
1647 		curfrags++;
1648 	/*
1649 	 * First, try to collapse mbufs.  Note that we always collapse
1650 	 * towards the front so we don't need to deal with moving the
1651 	 * pkthdr.  This may be suboptimal if the first mbuf has much
1652 	 * less data than the following.
1653 	 */
1654 	m = m0;
1655 again:
1656 	for (;;) {
1657 		n = m->m_next;
1658 		if (n == NULL)
1659 			break;
1660 		if (M_WRITABLE(m) &&
1661 		    n->m_len < M_TRAILINGSPACE(m)) {
1662 			bcopy(mtod(n, void *), mtod(m, char *) + m->m_len,
1663 				n->m_len);
1664 			m->m_len += n->m_len;
1665 			m->m_next = n->m_next;
1666 			m_free(n);
1667 			if (--curfrags <= maxfrags)
1668 				return m0;
1669 		} else
1670 			m = n;
1671 	}
1672 	KASSERT(maxfrags > 1,
1673 		("maxfrags %u, but normal collapse failed", maxfrags));
1674 	/*
1675 	 * Collapse consecutive mbufs to a cluster.
1676 	 */
1677 	prev = &m0->m_next;		/* NB: not the first mbuf */
1678 	while ((n = *prev) != NULL) {
1679 		if ((n2 = n->m_next) != NULL &&
1680 		    n->m_len + n2->m_len < MCLBYTES) {
1681 			m = m_getcl(how, MT_DATA, 0);
1682 			if (m == NULL)
1683 				goto bad;
1684 			bcopy(mtod(n, void *), mtod(m, void *), n->m_len);
1685 			bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len,
1686 				n2->m_len);
1687 			m->m_len = n->m_len + n2->m_len;
1688 			m->m_next = n2->m_next;
1689 			*prev = m;
1690 			m_free(n);
1691 			m_free(n2);
1692 			if (--curfrags <= maxfrags)	/* +1 cl -2 mbufs */
1693 				return m0;
1694 			/*
1695 			 * Still not there, try the normal collapse
1696 			 * again before we allocate another cluster.
1697 			 */
1698 			goto again;
1699 		}
1700 		prev = &n->m_next;
1701 	}
1702 	/*
1703 	 * No place where we can collapse to a cluster; punt.
1704 	 * This can occur if, for example, you request 2 frags
1705 	 * but the packet requires that both be clusters (we
1706 	 * never reallocate the first mbuf to avoid moving the
1707 	 * packet header).
1708 	 */
1709 bad:
1710 	return NULL;
1711 }
1712 
1713 #ifdef MBUF_STRESS_TEST
1714 
1715 /*
1716  * Fragment an mbuf chain.  There's no reason you'd ever want to do
1717  * this in normal usage, but it's great for stress testing various
1718  * mbuf consumers.
1719  *
1720  * If fragmentation is not possible, the original chain will be
1721  * returned.
1722  *
1723  * Possible length values:
1724  * 0	 no fragmentation will occur
1725  * > 0	each fragment will be of the specified length
1726  * -1	each fragment will be the same random value in length
1727  * -2	each fragment's length will be entirely random
1728  * (Random values range from 1 to 256)
1729  */
1730 struct mbuf *
1731 m_fragment(struct mbuf *m0, int how, int length)
1732 {
1733 	struct mbuf *m_new = NULL, *m_final = NULL;
1734 	int progress = 0;
1735 
1736 	if (!(m0->m_flags & M_PKTHDR))
1737 		return (m0);
1738 
1739 	if ((length == 0) || (length < -2))
1740 		return (m0);
1741 
1742 	m_fixhdr(m0); /* Needed sanity check */
1743 
1744 	m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1745 
1746 	if (m_final == NULL)
1747 		goto nospace;
1748 
1749 	if (m_dup_pkthdr(m_final, m0, how) == 0)
1750 		goto nospace;
1751 
1752 	m_new = m_final;
1753 
1754 	if (length == -1)
1755 		length = 1 + (arc4random() & 255);
1756 
1757 	while (progress < m0->m_pkthdr.len) {
1758 		int fraglen;
1759 
1760 		if (length > 0)
1761 			fraglen = length;
1762 		else
1763 			fraglen = 1 + (arc4random() & 255);
1764 		if (fraglen > m0->m_pkthdr.len - progress)
1765 			fraglen = m0->m_pkthdr.len - progress;
1766 
1767 		if (fraglen > MCLBYTES)
1768 			fraglen = MCLBYTES;
1769 
1770 		if (m_new == NULL) {
1771 			m_new = m_getcl(how, MT_DATA, 0);
1772 			if (m_new == NULL)
1773 				goto nospace;
1774 		}
1775 
1776 		m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
1777 		progress += fraglen;
1778 		m_new->m_len = fraglen;
1779 		if (m_new != m_final)
1780 			m_cat(m_final, m_new);
1781 		m_new = NULL;
1782 	}
1783 	m_freem(m0);
1784 	m0 = m_final;
1785 	return (m0);
1786 nospace:
1787 	if (m_final)
1788 		m_freem(m_final);
1789 	/* Return the original chain on failure */
1790 	return (m0);
1791 }
1792 
1793 #endif
1794 
1795 /*
1796  * Copy the contents of uio into a properly sized mbuf chain.
1797  */
1798 struct mbuf *
1799 m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
1800 {
1801 	struct mbuf *m, *mb;
1802 	int error, length;
1803 	ssize_t total;
1804 	int progress = 0;
1805 
1806 	/*
1807 	 * len can be zero or an arbitrary large value bound by
1808 	 * the total data supplied by the uio.
1809 	 */
1810 	if (len > 0)
1811 		total = min(uio->uio_resid, len);
1812 	else
1813 		total = uio->uio_resid;
1814 
1815 	/*
1816 	 * The smallest unit returned by m_getm2() is a single mbuf
1817 	 * with pkthdr.  We can't align past it.
1818 	 */
1819 	if (align >= MHLEN)
1820 		return (NULL);
1821 
1822 	/*
1823 	 * Give us the full allocation or nothing.
1824 	 * If len is zero return the smallest empty mbuf.
1825 	 */
1826 	m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
1827 	if (m == NULL)
1828 		return (NULL);
1829 	m->m_data += align;
1830 
1831 	/* Fill all mbufs with uio data and update header information. */
1832 	for (mb = m; mb != NULL; mb = mb->m_next) {
1833 		length = min(M_TRAILINGSPACE(mb), total - progress);
1834 
1835 		error = uiomove(mtod(mb, void *), length, uio);
1836 		if (error) {
1837 			m_freem(m);
1838 			return (NULL);
1839 		}
1840 
1841 		mb->m_len = length;
1842 		progress += length;
1843 		if (flags & M_PKTHDR)
1844 			m->m_pkthdr.len += length;
1845 	}
1846 	KASSERT(progress == total, ("%s: progress != total", __func__));
1847 
1848 	return (m);
1849 }
1850 
1851 /*
1852  * Copy an mbuf chain into a uio limited by len if set.
1853  */
1854 int
1855 m_mbuftouio(struct uio *uio, struct mbuf *m, int len)
1856 {
1857 	int error, length, total;
1858 	int progress = 0;
1859 
1860 	if (len > 0)
1861 		total = min(uio->uio_resid, len);
1862 	else
1863 		total = uio->uio_resid;
1864 
1865 	/* Fill the uio with data from the mbufs. */
1866 	for (; m != NULL; m = m->m_next) {
1867 		length = min(m->m_len, total - progress);
1868 
1869 		error = uiomove(mtod(m, void *), length, uio);
1870 		if (error)
1871 			return (error);
1872 
1873 		progress += length;
1874 	}
1875 
1876 	return (0);
1877 }
1878 
1879 /*
1880  * Set the m_data pointer of a newly-allocated mbuf
1881  * to place an object of the specified size at the
1882  * end of the mbuf, longword aligned.
1883  */
1884 void
1885 m_align(struct mbuf *m, int len)
1886 {
1887 #ifdef INVARIANTS
1888 	const char *msg = "%s: not a virgin mbuf";
1889 #endif
1890 	int adjust;
1891 
1892 	if (m->m_flags & M_EXT) {
1893 		KASSERT(m->m_data == m->m_ext.ext_buf, (msg, __func__));
1894 		adjust = m->m_ext.ext_size - len;
1895 	} else if (m->m_flags & M_PKTHDR) {
1896 		KASSERT(m->m_data == m->m_pktdat, (msg, __func__));
1897 		adjust = MHLEN - len;
1898 	} else {
1899 		KASSERT(m->m_data == m->m_dat, (msg, __func__));
1900 		adjust = MLEN - len;
1901 	}
1902 
1903 	m->m_data += adjust &~ (sizeof(long)-1);
1904 }
1905 
1906 /*
1907  * Create a writable copy of the mbuf chain.  While doing this
1908  * we compact the chain with a goal of producing a chain with
1909  * at most two mbufs.  The second mbuf in this chain is likely
1910  * to be a cluster.  The primary purpose of this work is to create
1911  * a writable packet for encryption, compression, etc.  The
1912  * secondary goal is to linearize the data so the data can be
1913  * passed to crypto hardware in the most efficient manner possible.
1914  */
1915 struct mbuf *
1916 m_unshare(struct mbuf *m0, int how)
1917 {
1918 	struct mbuf *m, *mprev;
1919 	struct mbuf *n, *mfirst, *mlast;
1920 	int len, off;
1921 
1922 	mprev = NULL;
1923 	for (m = m0; m != NULL; m = mprev->m_next) {
1924 		/*
1925 		 * Regular mbufs are ignored unless there's a cluster
1926 		 * in front of it that we can use to coalesce.  We do
1927 		 * the latter mainly so later clusters can be coalesced
1928 		 * also w/o having to handle them specially (i.e. convert
1929 		 * mbuf+cluster -> cluster).  This optimization is heavily
1930 		 * influenced by the assumption that we're running over
1931 		 * Ethernet where MCLBYTES is large enough that the max
1932 		 * packet size will permit lots of coalescing into a
1933 		 * single cluster.  This in turn permits efficient
1934 		 * crypto operations, especially when using hardware.
1935 		 */
1936 		if ((m->m_flags & M_EXT) == 0) {
1937 			if (mprev && (mprev->m_flags & M_EXT) &&
1938 			    m->m_len <= M_TRAILINGSPACE(mprev)) {
1939 				/* XXX: this ignores mbuf types */
1940 				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1941 				       mtod(m, caddr_t), m->m_len);
1942 				mprev->m_len += m->m_len;
1943 				mprev->m_next = m->m_next;	/* unlink from chain */
1944 				m_free(m);			/* reclaim mbuf */
1945 #if 0
1946 				newipsecstat.ips_mbcoalesced++;
1947 #endif
1948 			} else {
1949 				mprev = m;
1950 			}
1951 			continue;
1952 		}
1953 		/*
1954 		 * Writable mbufs are left alone (for now).
1955 		 */
1956 		if (M_WRITABLE(m)) {
1957 			mprev = m;
1958 			continue;
1959 		}
1960 
1961 		/*
1962 		 * Not writable, replace with a copy or coalesce with
1963 		 * the previous mbuf if possible (since we have to copy
1964 		 * it anyway, we try to reduce the number of mbufs and
1965 		 * clusters so that future work is easier).
1966 		 */
1967 		KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
1968 		/* NB: we only coalesce into a cluster or larger */
1969 		if (mprev != NULL && (mprev->m_flags & M_EXT) &&
1970 		    m->m_len <= M_TRAILINGSPACE(mprev)) {
1971 			/* XXX: this ignores mbuf types */
1972 			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1973 			       mtod(m, caddr_t), m->m_len);
1974 			mprev->m_len += m->m_len;
1975 			mprev->m_next = m->m_next;	/* unlink from chain */
1976 			m_free(m);			/* reclaim mbuf */
1977 #if 0
1978 			newipsecstat.ips_clcoalesced++;
1979 #endif
1980 			continue;
1981 		}
1982 
1983 		/*
1984 		 * Allocate new space to hold the copy and copy the data.
1985 		 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
1986 		 * splitting them into clusters.  We could just malloc a
1987 		 * buffer and make it external but too many device drivers
1988 		 * don't know how to break up the non-contiguous memory when
1989 		 * doing DMA.
1990 		 */
1991 		n = m_getcl(how, m->m_type, m->m_flags);
1992 		if (n == NULL) {
1993 			m_freem(m0);
1994 			return (NULL);
1995 		}
1996 		len = m->m_len;
1997 		off = 0;
1998 		mfirst = n;
1999 		mlast = NULL;
2000 		for (;;) {
2001 			int cc = min(len, MCLBYTES);
2002 			memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
2003 			n->m_len = cc;
2004 			if (mlast != NULL)
2005 				mlast->m_next = n;
2006 			mlast = n;
2007 #if 0
2008 			newipsecstat.ips_clcopied++;
2009 #endif
2010 
2011 			len -= cc;
2012 			if (len <= 0)
2013 				break;
2014 			off += cc;
2015 
2016 			n = m_getcl(how, m->m_type, m->m_flags);
2017 			if (n == NULL) {
2018 				m_freem(mfirst);
2019 				m_freem(m0);
2020 				return (NULL);
2021 			}
2022 		}
2023 		n->m_next = m->m_next;
2024 		if (mprev == NULL)
2025 			m0 = mfirst;		/* new head of chain */
2026 		else
2027 			mprev->m_next = mfirst;	/* replace old mbuf */
2028 		m_free(m);			/* release old mbuf */
2029 		mprev = mfirst;
2030 	}
2031 	return (m0);
2032 }
2033 
2034 #ifdef MBUF_PROFILING
2035 
2036 #define MP_BUCKETS 32 /* don't just change this as things may overflow.*/
2037 struct mbufprofile {
2038 	uintmax_t wasted[MP_BUCKETS];
2039 	uintmax_t used[MP_BUCKETS];
2040 	uintmax_t segments[MP_BUCKETS];
2041 } mbprof;
2042 
2043 #define MP_MAXDIGITS 21	/* strlen("16,000,000,000,000,000,000") == 21 */
2044 #define MP_NUMLINES 6
2045 #define MP_NUMSPERLINE 16
2046 #define MP_EXTRABYTES 64	/* > strlen("used:\nwasted:\nsegments:\n") */
2047 /* work out max space needed and add a bit of spare space too */
2048 #define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE)
2049 #define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES)
2050 
2051 char mbprofbuf[MP_BUFSIZE];
2052 
2053 void
2054 m_profile(struct mbuf *m)
2055 {
2056 	int segments = 0;
2057 	int used = 0;
2058 	int wasted = 0;
2059 
2060 	while (m) {
2061 		segments++;
2062 		used += m->m_len;
2063 		if (m->m_flags & M_EXT) {
2064 			wasted += MHLEN - sizeof(m->m_ext) +
2065 			    m->m_ext.ext_size - m->m_len;
2066 		} else {
2067 			if (m->m_flags & M_PKTHDR)
2068 				wasted += MHLEN - m->m_len;
2069 			else
2070 				wasted += MLEN - m->m_len;
2071 		}
2072 		m = m->m_next;
2073 	}
2074 	/* be paranoid.. it helps */
2075 	if (segments > MP_BUCKETS - 1)
2076 		segments = MP_BUCKETS - 1;
2077 	if (used > 100000)
2078 		used = 100000;
2079 	if (wasted > 100000)
2080 		wasted = 100000;
2081 	/* store in the appropriate bucket */
2082 	/* don't bother locking. if it's slightly off, so what? */
2083 	mbprof.segments[segments]++;
2084 	mbprof.used[fls(used)]++;
2085 	mbprof.wasted[fls(wasted)]++;
2086 }
2087 
2088 static void
2089 mbprof_textify(void)
2090 {
2091 	int offset;
2092 	char *c;
2093 	uint64_t *p;
2094 
2095 
2096 	p = &mbprof.wasted[0];
2097 	c = mbprofbuf;
2098 	offset = snprintf(c, MP_MAXLINE + 10,
2099 	    "wasted:\n"
2100 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2101 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2102 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2103 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2104 #ifdef BIG_ARRAY
2105 	p = &mbprof.wasted[16];
2106 	c += offset;
2107 	offset = snprintf(c, MP_MAXLINE,
2108 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2109 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2110 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2111 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2112 #endif
2113 	p = &mbprof.used[0];
2114 	c += offset;
2115 	offset = snprintf(c, MP_MAXLINE + 10,
2116 	    "used:\n"
2117 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2118 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2119 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2120 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2121 #ifdef BIG_ARRAY
2122 	p = &mbprof.used[16];
2123 	c += offset;
2124 	offset = snprintf(c, MP_MAXLINE,
2125 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2126 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2127 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2128 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2129 #endif
2130 	p = &mbprof.segments[0];
2131 	c += offset;
2132 	offset = snprintf(c, MP_MAXLINE + 10,
2133 	    "segments:\n"
2134 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2135 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2136 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2137 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2138 #ifdef BIG_ARRAY
2139 	p = &mbprof.segments[16];
2140 	c += offset;
2141 	offset = snprintf(c, MP_MAXLINE,
2142 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2143 	    "%ju %ju %ju %ju %ju %ju %ju %jju",
2144 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2145 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2146 #endif
2147 }
2148 
2149 static int
2150 mbprof_handler(SYSCTL_HANDLER_ARGS)
2151 {
2152 	int error;
2153 
2154 	mbprof_textify();
2155 	error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1);
2156 	return (error);
2157 }
2158 
2159 static int
2160 mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
2161 {
2162 	int clear, error;
2163 
2164 	clear = 0;
2165 	error = sysctl_handle_int(oidp, &clear, 0, req);
2166 	if (error || !req->newptr)
2167 		return (error);
2168 
2169 	if (clear) {
2170 		bzero(&mbprof, sizeof(mbprof));
2171 	}
2172 
2173 	return (error);
2174 }
2175 
2176 
2177 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD,
2178 	    NULL, 0, mbprof_handler, "A", "mbuf profiling statistics");
2179 
2180 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW,
2181 	    NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics");
2182 #endif
2183 
2184