xref: /freebsd/sys/kern/uipc_mbuf.c (revision 0e97acdf58fe27b09c4824a474b0344daf997c5f)
1 /*-
2  * Copyright (c) 1982, 1986, 1988, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 4. Neither the name of the University nor the names of its contributors
14  *    may be used to endorse or promote products derived from this software
15  *    without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  *	@(#)uipc_mbuf.c	8.2 (Berkeley) 1/4/94
30  */
31 
32 #include <sys/cdefs.h>
33 __FBSDID("$FreeBSD$");
34 
35 #include "opt_param.h"
36 #include "opt_mbuf_stress_test.h"
37 #include "opt_mbuf_profiling.h"
38 
39 #include <sys/param.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/limits.h>
43 #include <sys/lock.h>
44 #include <sys/malloc.h>
45 #include <sys/mbuf.h>
46 #include <sys/sysctl.h>
47 #include <sys/domain.h>
48 #include <sys/protosw.h>
49 #include <sys/uio.h>
50 
51 int	max_linkhdr;
52 int	max_protohdr;
53 int	max_hdr;
54 int	max_datalen;
55 #ifdef MBUF_STRESS_TEST
56 int	m_defragpackets;
57 int	m_defragbytes;
58 int	m_defraguseless;
59 int	m_defragfailure;
60 int	m_defragrandomfailures;
61 #endif
62 
63 /*
64  * sysctl(8) exported objects
65  */
66 SYSCTL_INT(_kern_ipc, KIPC_MAX_LINKHDR, max_linkhdr, CTLFLAG_RD,
67 	   &max_linkhdr, 0, "Size of largest link layer header");
68 SYSCTL_INT(_kern_ipc, KIPC_MAX_PROTOHDR, max_protohdr, CTLFLAG_RD,
69 	   &max_protohdr, 0, "Size of largest protocol layer header");
70 SYSCTL_INT(_kern_ipc, KIPC_MAX_HDR, max_hdr, CTLFLAG_RD,
71 	   &max_hdr, 0, "Size of largest link plus protocol header");
72 SYSCTL_INT(_kern_ipc, KIPC_MAX_DATALEN, max_datalen, CTLFLAG_RD,
73 	   &max_datalen, 0, "Minimum space left in mbuf after max_hdr");
74 #ifdef MBUF_STRESS_TEST
75 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragpackets, CTLFLAG_RD,
76 	   &m_defragpackets, 0, "");
77 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragbytes, CTLFLAG_RD,
78 	   &m_defragbytes, 0, "");
79 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defraguseless, CTLFLAG_RD,
80 	   &m_defraguseless, 0, "");
81 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragfailure, CTLFLAG_RD,
82 	   &m_defragfailure, 0, "");
83 SYSCTL_INT(_kern_ipc, OID_AUTO, m_defragrandomfailures, CTLFLAG_RW,
84 	   &m_defragrandomfailures, 0, "");
85 #endif
86 
87 /*
88  * Ensure the correct size of various mbuf parameters.  It could be off due
89  * to compiler-induced padding and alignment artifacts.
90  */
91 CTASSERT(sizeof(struct mbuf) == MSIZE);
92 CTASSERT(MSIZE - offsetof(struct mbuf, m_dat) == MLEN);
93 CTASSERT(MSIZE - offsetof(struct mbuf, m_pktdat) == MHLEN);
94 
95 /*
96  * m_get2() allocates minimum mbuf that would fit "size" argument.
97  */
98 struct mbuf *
99 m_get2(int size, int how, short type, int flags)
100 {
101 	struct mb_args args;
102 	struct mbuf *m, *n;
103 
104 	args.flags = flags;
105 	args.type = type;
106 
107 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
108 		return (uma_zalloc_arg(zone_mbuf, &args, how));
109 	if (size <= MCLBYTES)
110 		return (uma_zalloc_arg(zone_pack, &args, how));
111 
112 	if (size > MJUMPAGESIZE)
113 		return (NULL);
114 
115 	m = uma_zalloc_arg(zone_mbuf, &args, how);
116 	if (m == NULL)
117 		return (NULL);
118 
119 	n = uma_zalloc_arg(zone_jumbop, m, how);
120 	if (n == NULL) {
121 		uma_zfree(zone_mbuf, m);
122 		return (NULL);
123 	}
124 
125 	return (m);
126 }
127 
128 /*
129  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
130  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
131  */
132 struct mbuf *
133 m_getjcl(int how, short type, int flags, int size)
134 {
135 	struct mb_args args;
136 	struct mbuf *m, *n;
137 	uma_zone_t zone;
138 
139 	if (size == MCLBYTES)
140 		return m_getcl(how, type, flags);
141 
142 	args.flags = flags;
143 	args.type = type;
144 
145 	m = uma_zalloc_arg(zone_mbuf, &args, how);
146 	if (m == NULL)
147 		return (NULL);
148 
149 	zone = m_getzone(size);
150 	n = uma_zalloc_arg(zone, m, how);
151 	if (n == NULL) {
152 		uma_zfree(zone_mbuf, m);
153 		return (NULL);
154 	}
155 	return (m);
156 }
157 
158 /*
159  * Allocate a given length worth of mbufs and/or clusters (whatever fits
160  * best) and return a pointer to the top of the allocated chain.  If an
161  * existing mbuf chain is provided, then we will append the new chain
162  * to the existing one but still return the top of the newly allocated
163  * chain.
164  */
165 struct mbuf *
166 m_getm2(struct mbuf *m, int len, int how, short type, int flags)
167 {
168 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
169 
170 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
171 
172 	/* Validate flags. */
173 	flags &= (M_PKTHDR | M_EOR);
174 
175 	/* Packet header mbuf must be first in chain. */
176 	if ((flags & M_PKTHDR) && m != NULL)
177 		flags &= ~M_PKTHDR;
178 
179 	/* Loop and append maximum sized mbufs to the chain tail. */
180 	while (len > 0) {
181 		if (len > MCLBYTES)
182 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
183 			    MJUMPAGESIZE);
184 		else if (len >= MINCLSIZE)
185 			mb = m_getcl(how, type, (flags & M_PKTHDR));
186 		else if (flags & M_PKTHDR)
187 			mb = m_gethdr(how, type);
188 		else
189 			mb = m_get(how, type);
190 
191 		/* Fail the whole operation if one mbuf can't be allocated. */
192 		if (mb == NULL) {
193 			if (nm != NULL)
194 				m_freem(nm);
195 			return (NULL);
196 		}
197 
198 		/* Book keeping. */
199 		len -= (mb->m_flags & M_EXT) ? mb->m_ext.ext_size :
200 			((mb->m_flags & M_PKTHDR) ? MHLEN : MLEN);
201 		if (mtail != NULL)
202 			mtail->m_next = mb;
203 		else
204 			nm = mb;
205 		mtail = mb;
206 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
207 	}
208 	if (flags & M_EOR)
209 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
210 
211 	/* If mbuf was supplied, append new chain to the end of it. */
212 	if (m != NULL) {
213 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
214 			;
215 		mtail->m_next = nm;
216 		mtail->m_flags &= ~M_EOR;
217 	} else
218 		m = nm;
219 
220 	return (m);
221 }
222 
223 /*
224  * Free an entire chain of mbufs and associated external buffers, if
225  * applicable.
226  */
227 void
228 m_freem(struct mbuf *mb)
229 {
230 
231 	while (mb != NULL)
232 		mb = m_free(mb);
233 }
234 
235 /*-
236  * Configure a provided mbuf to refer to the provided external storage
237  * buffer and setup a reference count for said buffer.  If the setting
238  * up of the reference count fails, the M_EXT bit will not be set.  If
239  * successfull, the M_EXT bit is set in the mbuf's flags.
240  *
241  * Arguments:
242  *    mb     The existing mbuf to which to attach the provided buffer.
243  *    buf    The address of the provided external storage buffer.
244  *    size   The size of the provided buffer.
245  *    freef  A pointer to a routine that is responsible for freeing the
246  *           provided external storage buffer.
247  *    args   A pointer to an argument structure (of any type) to be passed
248  *           to the provided freef routine (may be NULL).
249  *    flags  Any other flags to be passed to the provided mbuf.
250  *    type   The type that the external storage buffer should be
251  *           labeled with.
252  *
253  * Returns:
254  *    Nothing.
255  */
256 int
257 m_extadd(struct mbuf *mb, caddr_t buf, u_int size,
258     void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2,
259     int flags, int type, int wait)
260 {
261 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
262 
263 	if (type != EXT_EXTREF)
264 		mb->m_ext.ext_cnt = uma_zalloc(zone_ext_refcnt, wait);
265 
266 	if (mb->m_ext.ext_cnt == NULL)
267 		return (ENOMEM);
268 
269 	*(mb->m_ext.ext_cnt) = 1;
270 	mb->m_flags |= (M_EXT | flags);
271 	mb->m_ext.ext_buf = buf;
272 	mb->m_data = mb->m_ext.ext_buf;
273 	mb->m_ext.ext_size = size;
274 	mb->m_ext.ext_free = freef;
275 	mb->m_ext.ext_arg1 = arg1;
276 	mb->m_ext.ext_arg2 = arg2;
277 	mb->m_ext.ext_type = type;
278 	mb->m_ext.ext_flags = 0;
279 
280 	return (0);
281 }
282 
283 /*
284  * Non-directly-exported function to clean up after mbufs with M_EXT
285  * storage attached to them if the reference count hits 1.
286  */
287 void
288 mb_free_ext(struct mbuf *m)
289 {
290 	int freembuf;
291 
292 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
293 
294 	/*
295 	 * Check if the header is embedded in the cluster.
296 	 */
297 	freembuf = (m->m_flags & M_NOFREE) ? 0 : 1;
298 
299 	switch (m->m_ext.ext_type) {
300 	case EXT_SFBUF:
301 		sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
302 		break;
303 	default:
304 		KASSERT(m->m_ext.ext_cnt != NULL,
305 		    ("%s: no refcounting pointer on %p", __func__, m));
306 		/*
307 		 * Free attached storage if this mbuf is the only
308 		 * reference to it.
309 		 */
310 		if (*(m->m_ext.ext_cnt) != 1) {
311 			if (atomic_fetchadd_int(m->m_ext.ext_cnt, -1) != 1)
312 				break;
313 		}
314 
315 		switch (m->m_ext.ext_type) {
316 		case EXT_PACKET:	/* The packet zone is special. */
317 			if (*(m->m_ext.ext_cnt) == 0)
318 				*(m->m_ext.ext_cnt) = 1;
319 			uma_zfree(zone_pack, m);
320 			return;		/* Job done. */
321 		case EXT_CLUSTER:
322 			uma_zfree(zone_clust, m->m_ext.ext_buf);
323 			break;
324 		case EXT_JUMBOP:
325 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
326 			break;
327 		case EXT_JUMBO9:
328 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
329 			break;
330 		case EXT_JUMBO16:
331 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
332 			break;
333 		case EXT_NET_DRV:
334 		case EXT_MOD_TYPE:
335 		case EXT_DISPOSABLE:
336 			*(m->m_ext.ext_cnt) = 0;
337 			uma_zfree(zone_ext_refcnt, __DEVOLATILE(u_int *,
338 				m->m_ext.ext_cnt));
339 			/* FALLTHROUGH */
340 		case EXT_EXTREF:
341 			KASSERT(m->m_ext.ext_free != NULL,
342 				("%s: ext_free not set", __func__));
343 			(*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1,
344 			    m->m_ext.ext_arg2);
345 			break;
346 		default:
347 			KASSERT(m->m_ext.ext_type == 0,
348 				("%s: unknown ext_type", __func__));
349 		}
350 	}
351 
352 	if (freembuf)
353 		uma_zfree(zone_mbuf, m);
354 }
355 
356 /*
357  * Attach the cluster from *m to *n, set up m_ext in *n
358  * and bump the refcount of the cluster.
359  */
360 static void
361 mb_dupcl(struct mbuf *n, struct mbuf *m)
362 {
363 
364 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
365 	KASSERT(!(n->m_flags & M_EXT), ("%s: M_EXT set on %p", __func__, n));
366 
367 	switch (m->m_ext.ext_type) {
368 	case EXT_SFBUF:
369 		sf_ext_ref(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
370 		break;
371 	default:
372 		KASSERT(m->m_ext.ext_cnt != NULL,
373 		    ("%s: no refcounting pointer on %p", __func__, m));
374 		if (*(m->m_ext.ext_cnt) == 1)
375 			*(m->m_ext.ext_cnt) += 1;
376 		else
377 			atomic_add_int(m->m_ext.ext_cnt, 1);
378 	}
379 
380 	n->m_ext = m->m_ext;
381 	n->m_flags |= M_EXT;
382 	n->m_flags |= m->m_flags & M_RDONLY;
383 }
384 
385 /*
386  * Clean up mbuf (chain) from any tags and packet headers.
387  * If "all" is set then the first mbuf in the chain will be
388  * cleaned too.
389  */
390 void
391 m_demote(struct mbuf *m0, int all)
392 {
393 	struct mbuf *m;
394 
395 	for (m = all ? m0 : m0->m_next; m != NULL; m = m->m_next) {
396 		KASSERT(m->m_nextpkt == NULL, ("%s: m_nextpkt in m %p, m0 %p",
397 		    __func__, m, m0));
398 		if (m->m_flags & M_PKTHDR) {
399 			m_tag_delete_chain(m, NULL);
400 			m->m_flags &= ~M_PKTHDR;
401 			bzero(&m->m_pkthdr, sizeof(struct pkthdr));
402 		}
403 		m->m_flags = m->m_flags & (M_EXT|M_RDONLY|M_NOFREE);
404 	}
405 }
406 
407 /*
408  * Sanity checks on mbuf (chain) for use in KASSERT() and general
409  * debugging.
410  * Returns 0 or panics when bad and 1 on all tests passed.
411  * Sanitize, 0 to run M_SANITY_ACTION, 1 to garble things so they
412  * blow up later.
413  */
414 int
415 m_sanity(struct mbuf *m0, int sanitize)
416 {
417 	struct mbuf *m;
418 	caddr_t a, b;
419 	int pktlen = 0;
420 
421 #ifdef INVARIANTS
422 #define	M_SANITY_ACTION(s)	panic("mbuf %p: " s, m)
423 #else
424 #define	M_SANITY_ACTION(s)	printf("mbuf %p: " s, m)
425 #endif
426 
427 	for (m = m0; m != NULL; m = m->m_next) {
428 		/*
429 		 * Basic pointer checks.  If any of these fails then some
430 		 * unrelated kernel memory before or after us is trashed.
431 		 * No way to recover from that.
432 		 */
433 		a = ((m->m_flags & M_EXT) ? m->m_ext.ext_buf :
434 			((m->m_flags & M_PKTHDR) ? (caddr_t)(&m->m_pktdat) :
435 			 (caddr_t)(&m->m_dat)) );
436 		b = (caddr_t)(a + (m->m_flags & M_EXT ? m->m_ext.ext_size :
437 			((m->m_flags & M_PKTHDR) ? MHLEN : MLEN)));
438 		if ((caddr_t)m->m_data < a)
439 			M_SANITY_ACTION("m_data outside mbuf data range left");
440 		if ((caddr_t)m->m_data > b)
441 			M_SANITY_ACTION("m_data outside mbuf data range right");
442 		if ((caddr_t)m->m_data + m->m_len > b)
443 			M_SANITY_ACTION("m_data + m_len exeeds mbuf space");
444 
445 		/* m->m_nextpkt may only be set on first mbuf in chain. */
446 		if (m != m0 && m->m_nextpkt != NULL) {
447 			if (sanitize) {
448 				m_freem(m->m_nextpkt);
449 				m->m_nextpkt = (struct mbuf *)0xDEADC0DE;
450 			} else
451 				M_SANITY_ACTION("m->m_nextpkt on in-chain mbuf");
452 		}
453 
454 		/* packet length (not mbuf length!) calculation */
455 		if (m0->m_flags & M_PKTHDR)
456 			pktlen += m->m_len;
457 
458 		/* m_tags may only be attached to first mbuf in chain. */
459 		if (m != m0 && m->m_flags & M_PKTHDR &&
460 		    !SLIST_EMPTY(&m->m_pkthdr.tags)) {
461 			if (sanitize) {
462 				m_tag_delete_chain(m, NULL);
463 				/* put in 0xDEADC0DE perhaps? */
464 			} else
465 				M_SANITY_ACTION("m_tags on in-chain mbuf");
466 		}
467 
468 		/* M_PKTHDR may only be set on first mbuf in chain */
469 		if (m != m0 && m->m_flags & M_PKTHDR) {
470 			if (sanitize) {
471 				bzero(&m->m_pkthdr, sizeof(m->m_pkthdr));
472 				m->m_flags &= ~M_PKTHDR;
473 				/* put in 0xDEADCODE and leave hdr flag in */
474 			} else
475 				M_SANITY_ACTION("M_PKTHDR on in-chain mbuf");
476 		}
477 	}
478 	m = m0;
479 	if (pktlen && pktlen != m->m_pkthdr.len) {
480 		if (sanitize)
481 			m->m_pkthdr.len = 0;
482 		else
483 			M_SANITY_ACTION("m_pkthdr.len != mbuf chain length");
484 	}
485 	return 1;
486 
487 #undef	M_SANITY_ACTION
488 }
489 
490 
491 /*
492  * "Move" mbuf pkthdr from "from" to "to".
493  * "from" must have M_PKTHDR set, and "to" must be empty.
494  */
495 void
496 m_move_pkthdr(struct mbuf *to, struct mbuf *from)
497 {
498 
499 #if 0
500 	/* see below for why these are not enabled */
501 	M_ASSERTPKTHDR(to);
502 	/* Note: with MAC, this may not be a good assertion. */
503 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags),
504 	    ("m_move_pkthdr: to has tags"));
505 #endif
506 #ifdef MAC
507 	/*
508 	 * XXXMAC: It could be this should also occur for non-MAC?
509 	 */
510 	if (to->m_flags & M_PKTHDR)
511 		m_tag_delete_chain(to, NULL);
512 #endif
513 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
514 	if ((to->m_flags & M_EXT) == 0)
515 		to->m_data = to->m_pktdat;
516 	to->m_pkthdr = from->m_pkthdr;		/* especially tags */
517 	SLIST_INIT(&from->m_pkthdr.tags);	/* purge tags from src */
518 	from->m_flags &= ~M_PKTHDR;
519 }
520 
521 /*
522  * Duplicate "from"'s mbuf pkthdr in "to".
523  * "from" must have M_PKTHDR set, and "to" must be empty.
524  * In particular, this does a deep copy of the packet tags.
525  */
526 int
527 m_dup_pkthdr(struct mbuf *to, struct mbuf *from, int how)
528 {
529 
530 #if 0
531 	/*
532 	 * The mbuf allocator only initializes the pkthdr
533 	 * when the mbuf is allocated with m_gethdr(). Many users
534 	 * (e.g. m_copy*, m_prepend) use m_get() and then
535 	 * smash the pkthdr as needed causing these
536 	 * assertions to trip.  For now just disable them.
537 	 */
538 	M_ASSERTPKTHDR(to);
539 	/* Note: with MAC, this may not be a good assertion. */
540 	KASSERT(SLIST_EMPTY(&to->m_pkthdr.tags), ("m_dup_pkthdr: to has tags"));
541 #endif
542 	MBUF_CHECKSLEEP(how);
543 #ifdef MAC
544 	if (to->m_flags & M_PKTHDR)
545 		m_tag_delete_chain(to, NULL);
546 #endif
547 	to->m_flags = (from->m_flags & M_COPYFLAGS) | (to->m_flags & M_EXT);
548 	if ((to->m_flags & M_EXT) == 0)
549 		to->m_data = to->m_pktdat;
550 	to->m_pkthdr = from->m_pkthdr;
551 	SLIST_INIT(&to->m_pkthdr.tags);
552 	return (m_tag_copy_chain(to, from, how));
553 }
554 
555 /*
556  * Lesser-used path for M_PREPEND:
557  * allocate new mbuf to prepend to chain,
558  * copy junk along.
559  */
560 struct mbuf *
561 m_prepend(struct mbuf *m, int len, int how)
562 {
563 	struct mbuf *mn;
564 
565 	if (m->m_flags & M_PKTHDR)
566 		mn = m_gethdr(how, m->m_type);
567 	else
568 		mn = m_get(how, m->m_type);
569 	if (mn == NULL) {
570 		m_freem(m);
571 		return (NULL);
572 	}
573 	if (m->m_flags & M_PKTHDR)
574 		m_move_pkthdr(mn, m);
575 	mn->m_next = m;
576 	m = mn;
577 	if(m->m_flags & M_PKTHDR) {
578 		if (len < MHLEN)
579 			MH_ALIGN(m, len);
580 	} else {
581 		if (len < MLEN)
582 			M_ALIGN(m, len);
583 	}
584 	m->m_len = len;
585 	return (m);
586 }
587 
588 /*
589  * Make a copy of an mbuf chain starting "off0" bytes from the beginning,
590  * continuing for "len" bytes.  If len is M_COPYALL, copy to end of mbuf.
591  * The wait parameter is a choice of M_WAITOK/M_NOWAIT from caller.
592  * Note that the copy is read-only, because clusters are not copied,
593  * only their reference counts are incremented.
594  */
595 struct mbuf *
596 m_copym(struct mbuf *m, int off0, int len, int wait)
597 {
598 	struct mbuf *n, **np;
599 	int off = off0;
600 	struct mbuf *top;
601 	int copyhdr = 0;
602 
603 	KASSERT(off >= 0, ("m_copym, negative off %d", off));
604 	KASSERT(len >= 0, ("m_copym, negative len %d", len));
605 	MBUF_CHECKSLEEP(wait);
606 	if (off == 0 && m->m_flags & M_PKTHDR)
607 		copyhdr = 1;
608 	while (off > 0) {
609 		KASSERT(m != NULL, ("m_copym, offset > size of mbuf chain"));
610 		if (off < m->m_len)
611 			break;
612 		off -= m->m_len;
613 		m = m->m_next;
614 	}
615 	np = &top;
616 	top = 0;
617 	while (len > 0) {
618 		if (m == NULL) {
619 			KASSERT(len == M_COPYALL,
620 			    ("m_copym, length > size of mbuf chain"));
621 			break;
622 		}
623 		if (copyhdr)
624 			n = m_gethdr(wait, m->m_type);
625 		else
626 			n = m_get(wait, m->m_type);
627 		*np = n;
628 		if (n == NULL)
629 			goto nospace;
630 		if (copyhdr) {
631 			if (!m_dup_pkthdr(n, m, wait))
632 				goto nospace;
633 			if (len == M_COPYALL)
634 				n->m_pkthdr.len -= off0;
635 			else
636 				n->m_pkthdr.len = len;
637 			copyhdr = 0;
638 		}
639 		n->m_len = min(len, m->m_len - off);
640 		if (m->m_flags & M_EXT) {
641 			n->m_data = m->m_data + off;
642 			mb_dupcl(n, m);
643 		} else
644 			bcopy(mtod(m, caddr_t)+off, mtod(n, caddr_t),
645 			    (u_int)n->m_len);
646 		if (len != M_COPYALL)
647 			len -= n->m_len;
648 		off = 0;
649 		m = m->m_next;
650 		np = &n->m_next;
651 	}
652 
653 	return (top);
654 nospace:
655 	m_freem(top);
656 	return (NULL);
657 }
658 
659 /*
660  * Returns mbuf chain with new head for the prepending case.
661  * Copies from mbuf (chain) n from off for len to mbuf (chain) m
662  * either prepending or appending the data.
663  * The resulting mbuf (chain) m is fully writeable.
664  * m is destination (is made writeable)
665  * n is source, off is offset in source, len is len from offset
666  * dir, 0 append, 1 prepend
667  * how, wait or nowait
668  */
669 
670 static int
671 m_bcopyxxx(void *s, void *t, u_int len)
672 {
673 	bcopy(s, t, (size_t)len);
674 	return 0;
675 }
676 
677 struct mbuf *
678 m_copymdata(struct mbuf *m, struct mbuf *n, int off, int len,
679     int prep, int how)
680 {
681 	struct mbuf *mm, *x, *z, *prev = NULL;
682 	caddr_t p;
683 	int i, nlen = 0;
684 	caddr_t buf[MLEN];
685 
686 	KASSERT(m != NULL && n != NULL, ("m_copymdata, no target or source"));
687 	KASSERT(off >= 0, ("m_copymdata, negative off %d", off));
688 	KASSERT(len >= 0, ("m_copymdata, negative len %d", len));
689 	KASSERT(prep == 0 || prep == 1, ("m_copymdata, unknown direction %d", prep));
690 
691 	mm = m;
692 	if (!prep) {
693 		while(mm->m_next) {
694 			prev = mm;
695 			mm = mm->m_next;
696 		}
697 	}
698 	for (z = n; z != NULL; z = z->m_next)
699 		nlen += z->m_len;
700 	if (len == M_COPYALL)
701 		len = nlen - off;
702 	if (off + len > nlen || len < 1)
703 		return NULL;
704 
705 	if (!M_WRITABLE(mm)) {
706 		/* XXX: Use proper m_xxx function instead. */
707 		x = m_getcl(how, MT_DATA, mm->m_flags);
708 		if (x == NULL)
709 			return NULL;
710 		bcopy(mm->m_ext.ext_buf, x->m_ext.ext_buf, x->m_ext.ext_size);
711 		p = x->m_ext.ext_buf + (mm->m_data - mm->m_ext.ext_buf);
712 		x->m_data = p;
713 		mm->m_next = NULL;
714 		if (mm != m)
715 			prev->m_next = x;
716 		m_free(mm);
717 		mm = x;
718 	}
719 
720 	/*
721 	 * Append/prepend the data.  Allocating mbufs as necessary.
722 	 */
723 	/* Shortcut if enough free space in first/last mbuf. */
724 	if (!prep && M_TRAILINGSPACE(mm) >= len) {
725 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t) +
726 			 mm->m_len);
727 		mm->m_len += len;
728 		mm->m_pkthdr.len += len;
729 		return m;
730 	}
731 	if (prep && M_LEADINGSPACE(mm) >= len) {
732 		mm->m_data = mtod(mm, caddr_t) - len;
733 		m_apply(n, off, len, m_bcopyxxx, mtod(mm, caddr_t));
734 		mm->m_len += len;
735 		mm->m_pkthdr.len += len;
736 		return mm;
737 	}
738 
739 	/* Expand first/last mbuf to cluster if possible. */
740 	if (!prep && !(mm->m_flags & M_EXT) && len > M_TRAILINGSPACE(mm)) {
741 		bcopy(mm->m_data, &buf, mm->m_len);
742 		m_clget(mm, how);
743 		if (!(mm->m_flags & M_EXT))
744 			return NULL;
745 		bcopy(&buf, mm->m_ext.ext_buf, mm->m_len);
746 		mm->m_data = mm->m_ext.ext_buf;
747 	}
748 	if (prep && !(mm->m_flags & M_EXT) && len > M_LEADINGSPACE(mm)) {
749 		bcopy(mm->m_data, &buf, mm->m_len);
750 		m_clget(mm, how);
751 		if (!(mm->m_flags & M_EXT))
752 			return NULL;
753 		bcopy(&buf, (caddr_t *)mm->m_ext.ext_buf +
754 		    mm->m_ext.ext_size - mm->m_len, mm->m_len);
755 		mm->m_data = (caddr_t)mm->m_ext.ext_buf +
756 		    mm->m_ext.ext_size - mm->m_len;
757 	}
758 
759 	/* Append/prepend as many mbuf (clusters) as necessary to fit len. */
760 	if (!prep && len > M_TRAILINGSPACE(mm)) {
761 		if (!m_getm(mm, len - M_TRAILINGSPACE(mm), how, MT_DATA))
762 			return NULL;
763 	}
764 	if (prep && len > M_LEADINGSPACE(mm)) {
765 		if (!(z = m_getm(NULL, len - M_LEADINGSPACE(mm), how, MT_DATA)))
766 			return NULL;
767 		i = 0;
768 		for (x = z; x != NULL; x = x->m_next) {
769 			i += x->m_flags & M_EXT ? x->m_ext.ext_size :
770 			    (x->m_flags & M_PKTHDR ? MHLEN : MLEN);
771 			if (!x->m_next)
772 				break;
773 		}
774 		z->m_data += i - len;
775 		m_move_pkthdr(mm, z);
776 		x->m_next = mm;
777 		mm = z;
778 	}
779 
780 	/* Seek to start position in source mbuf. Optimization for long chains. */
781 	while (off > 0) {
782 		if (off < n->m_len)
783 			break;
784 		off -= n->m_len;
785 		n = n->m_next;
786 	}
787 
788 	/* Copy data into target mbuf. */
789 	z = mm;
790 	while (len > 0) {
791 		KASSERT(z != NULL, ("m_copymdata, falling off target edge"));
792 		i = M_TRAILINGSPACE(z);
793 		m_apply(n, off, i, m_bcopyxxx, mtod(z, caddr_t) + z->m_len);
794 		z->m_len += i;
795 		/* fixup pkthdr.len if necessary */
796 		if ((prep ? mm : m)->m_flags & M_PKTHDR)
797 			(prep ? mm : m)->m_pkthdr.len += i;
798 		off += i;
799 		len -= i;
800 		z = z->m_next;
801 	}
802 	return (prep ? mm : m);
803 }
804 
805 /*
806  * Copy an entire packet, including header (which must be present).
807  * An optimization of the common case `m_copym(m, 0, M_COPYALL, how)'.
808  * Note that the copy is read-only, because clusters are not copied,
809  * only their reference counts are incremented.
810  * Preserve alignment of the first mbuf so if the creator has left
811  * some room at the beginning (e.g. for inserting protocol headers)
812  * the copies still have the room available.
813  */
814 struct mbuf *
815 m_copypacket(struct mbuf *m, int how)
816 {
817 	struct mbuf *top, *n, *o;
818 
819 	MBUF_CHECKSLEEP(how);
820 	n = m_get(how, m->m_type);
821 	top = n;
822 	if (n == NULL)
823 		goto nospace;
824 
825 	if (!m_dup_pkthdr(n, m, how))
826 		goto nospace;
827 	n->m_len = m->m_len;
828 	if (m->m_flags & M_EXT) {
829 		n->m_data = m->m_data;
830 		mb_dupcl(n, m);
831 	} else {
832 		n->m_data = n->m_pktdat + (m->m_data - m->m_pktdat );
833 		bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
834 	}
835 
836 	m = m->m_next;
837 	while (m) {
838 		o = m_get(how, m->m_type);
839 		if (o == NULL)
840 			goto nospace;
841 
842 		n->m_next = o;
843 		n = n->m_next;
844 
845 		n->m_len = m->m_len;
846 		if (m->m_flags & M_EXT) {
847 			n->m_data = m->m_data;
848 			mb_dupcl(n, m);
849 		} else {
850 			bcopy(mtod(m, char *), mtod(n, char *), n->m_len);
851 		}
852 
853 		m = m->m_next;
854 	}
855 	return top;
856 nospace:
857 	m_freem(top);
858 	return (NULL);
859 }
860 
861 /*
862  * Copy data from an mbuf chain starting "off" bytes from the beginning,
863  * continuing for "len" bytes, into the indicated buffer.
864  */
865 void
866 m_copydata(const struct mbuf *m, int off, int len, caddr_t cp)
867 {
868 	u_int count;
869 
870 	KASSERT(off >= 0, ("m_copydata, negative off %d", off));
871 	KASSERT(len >= 0, ("m_copydata, negative len %d", len));
872 	while (off > 0) {
873 		KASSERT(m != NULL, ("m_copydata, offset > size of mbuf chain"));
874 		if (off < m->m_len)
875 			break;
876 		off -= m->m_len;
877 		m = m->m_next;
878 	}
879 	while (len > 0) {
880 		KASSERT(m != NULL, ("m_copydata, length > size of mbuf chain"));
881 		count = min(m->m_len - off, len);
882 		bcopy(mtod(m, caddr_t) + off, cp, count);
883 		len -= count;
884 		cp += count;
885 		off = 0;
886 		m = m->m_next;
887 	}
888 }
889 
890 /*
891  * Copy a packet header mbuf chain into a completely new chain, including
892  * copying any mbuf clusters.  Use this instead of m_copypacket() when
893  * you need a writable copy of an mbuf chain.
894  */
895 struct mbuf *
896 m_dup(struct mbuf *m, int how)
897 {
898 	struct mbuf **p, *top = NULL;
899 	int remain, moff, nsize;
900 
901 	MBUF_CHECKSLEEP(how);
902 	/* Sanity check */
903 	if (m == NULL)
904 		return (NULL);
905 	M_ASSERTPKTHDR(m);
906 
907 	/* While there's more data, get a new mbuf, tack it on, and fill it */
908 	remain = m->m_pkthdr.len;
909 	moff = 0;
910 	p = &top;
911 	while (remain > 0 || top == NULL) {	/* allow m->m_pkthdr.len == 0 */
912 		struct mbuf *n;
913 
914 		/* Get the next new mbuf */
915 		if (remain >= MINCLSIZE) {
916 			n = m_getcl(how, m->m_type, 0);
917 			nsize = MCLBYTES;
918 		} else {
919 			n = m_get(how, m->m_type);
920 			nsize = MLEN;
921 		}
922 		if (n == NULL)
923 			goto nospace;
924 
925 		if (top == NULL) {		/* First one, must be PKTHDR */
926 			if (!m_dup_pkthdr(n, m, how)) {
927 				m_free(n);
928 				goto nospace;
929 			}
930 			if ((n->m_flags & M_EXT) == 0)
931 				nsize = MHLEN;
932 		}
933 		n->m_len = 0;
934 
935 		/* Link it into the new chain */
936 		*p = n;
937 		p = &n->m_next;
938 
939 		/* Copy data from original mbuf(s) into new mbuf */
940 		while (n->m_len < nsize && m != NULL) {
941 			int chunk = min(nsize - n->m_len, m->m_len - moff);
942 
943 			bcopy(m->m_data + moff, n->m_data + n->m_len, chunk);
944 			moff += chunk;
945 			n->m_len += chunk;
946 			remain -= chunk;
947 			if (moff == m->m_len) {
948 				m = m->m_next;
949 				moff = 0;
950 			}
951 		}
952 
953 		/* Check correct total mbuf length */
954 		KASSERT((remain > 0 && m != NULL) || (remain == 0 && m == NULL),
955 		    	("%s: bogus m_pkthdr.len", __func__));
956 	}
957 	return (top);
958 
959 nospace:
960 	m_freem(top);
961 	return (NULL);
962 }
963 
964 /*
965  * Concatenate mbuf chain n to m.
966  * Both chains must be of the same type (e.g. MT_DATA).
967  * Any m_pkthdr is not updated.
968  */
969 void
970 m_cat(struct mbuf *m, struct mbuf *n)
971 {
972 	while (m->m_next)
973 		m = m->m_next;
974 	while (n) {
975 		if (!M_WRITABLE(m) ||
976 		    M_TRAILINGSPACE(m) < n->m_len) {
977 			/* just join the two chains */
978 			m->m_next = n;
979 			return;
980 		}
981 		/* splat the data from one into the other */
982 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
983 		    (u_int)n->m_len);
984 		m->m_len += n->m_len;
985 		n = m_free(n);
986 	}
987 }
988 
989 /*
990  * Concatenate two pkthdr mbuf chains.
991  */
992 void
993 m_catpkt(struct mbuf *m, struct mbuf *n)
994 {
995 
996 	M_ASSERTPKTHDR(m);
997 	M_ASSERTPKTHDR(n);
998 
999 	m->m_pkthdr.len += n->m_pkthdr.len;
1000 	m_demote(n, 1);
1001 
1002 	m_cat(m, n);
1003 }
1004 
1005 void
1006 m_adj(struct mbuf *mp, int req_len)
1007 {
1008 	int len = req_len;
1009 	struct mbuf *m;
1010 	int count;
1011 
1012 	if ((m = mp) == NULL)
1013 		return;
1014 	if (len >= 0) {
1015 		/*
1016 		 * Trim from head.
1017 		 */
1018 		while (m != NULL && len > 0) {
1019 			if (m->m_len <= len) {
1020 				len -= m->m_len;
1021 				m->m_len = 0;
1022 				m = m->m_next;
1023 			} else {
1024 				m->m_len -= len;
1025 				m->m_data += len;
1026 				len = 0;
1027 			}
1028 		}
1029 		if (mp->m_flags & M_PKTHDR)
1030 			mp->m_pkthdr.len -= (req_len - len);
1031 	} else {
1032 		/*
1033 		 * Trim from tail.  Scan the mbuf chain,
1034 		 * calculating its length and finding the last mbuf.
1035 		 * If the adjustment only affects this mbuf, then just
1036 		 * adjust and return.  Otherwise, rescan and truncate
1037 		 * after the remaining size.
1038 		 */
1039 		len = -len;
1040 		count = 0;
1041 		for (;;) {
1042 			count += m->m_len;
1043 			if (m->m_next == (struct mbuf *)0)
1044 				break;
1045 			m = m->m_next;
1046 		}
1047 		if (m->m_len >= len) {
1048 			m->m_len -= len;
1049 			if (mp->m_flags & M_PKTHDR)
1050 				mp->m_pkthdr.len -= len;
1051 			return;
1052 		}
1053 		count -= len;
1054 		if (count < 0)
1055 			count = 0;
1056 		/*
1057 		 * Correct length for chain is "count".
1058 		 * Find the mbuf with last data, adjust its length,
1059 		 * and toss data from remaining mbufs on chain.
1060 		 */
1061 		m = mp;
1062 		if (m->m_flags & M_PKTHDR)
1063 			m->m_pkthdr.len = count;
1064 		for (; m; m = m->m_next) {
1065 			if (m->m_len >= count) {
1066 				m->m_len = count;
1067 				if (m->m_next != NULL) {
1068 					m_freem(m->m_next);
1069 					m->m_next = NULL;
1070 				}
1071 				break;
1072 			}
1073 			count -= m->m_len;
1074 		}
1075 	}
1076 }
1077 
1078 /*
1079  * Rearange an mbuf chain so that len bytes are contiguous
1080  * and in the data area of an mbuf (so that mtod will work
1081  * for a structure of size len).  Returns the resulting
1082  * mbuf chain on success, frees it and returns null on failure.
1083  * If there is room, it will add up to max_protohdr-len extra bytes to the
1084  * contiguous region in an attempt to avoid being called next time.
1085  */
1086 struct mbuf *
1087 m_pullup(struct mbuf *n, int len)
1088 {
1089 	struct mbuf *m;
1090 	int count;
1091 	int space;
1092 
1093 	/*
1094 	 * If first mbuf has no cluster, and has room for len bytes
1095 	 * without shifting current data, pullup into it,
1096 	 * otherwise allocate a new mbuf to prepend to the chain.
1097 	 */
1098 	if ((n->m_flags & M_EXT) == 0 &&
1099 	    n->m_data + len < &n->m_dat[MLEN] && n->m_next) {
1100 		if (n->m_len >= len)
1101 			return (n);
1102 		m = n;
1103 		n = n->m_next;
1104 		len -= m->m_len;
1105 	} else {
1106 		if (len > MHLEN)
1107 			goto bad;
1108 		m = m_get(M_NOWAIT, n->m_type);
1109 		if (m == NULL)
1110 			goto bad;
1111 		if (n->m_flags & M_PKTHDR)
1112 			m_move_pkthdr(m, n);
1113 	}
1114 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1115 	do {
1116 		count = min(min(max(len, max_protohdr), space), n->m_len);
1117 		bcopy(mtod(n, caddr_t), mtod(m, caddr_t) + m->m_len,
1118 		  (u_int)count);
1119 		len -= count;
1120 		m->m_len += count;
1121 		n->m_len -= count;
1122 		space -= count;
1123 		if (n->m_len)
1124 			n->m_data += count;
1125 		else
1126 			n = m_free(n);
1127 	} while (len > 0 && n);
1128 	if (len > 0) {
1129 		(void) m_free(m);
1130 		goto bad;
1131 	}
1132 	m->m_next = n;
1133 	return (m);
1134 bad:
1135 	m_freem(n);
1136 	return (NULL);
1137 }
1138 
1139 /*
1140  * Like m_pullup(), except a new mbuf is always allocated, and we allow
1141  * the amount of empty space before the data in the new mbuf to be specified
1142  * (in the event that the caller expects to prepend later).
1143  */
1144 int MSFail;
1145 
1146 struct mbuf *
1147 m_copyup(struct mbuf *n, int len, int dstoff)
1148 {
1149 	struct mbuf *m;
1150 	int count, space;
1151 
1152 	if (len > (MHLEN - dstoff))
1153 		goto bad;
1154 	m = m_get(M_NOWAIT, n->m_type);
1155 	if (m == NULL)
1156 		goto bad;
1157 	if (n->m_flags & M_PKTHDR)
1158 		m_move_pkthdr(m, n);
1159 	m->m_data += dstoff;
1160 	space = &m->m_dat[MLEN] - (m->m_data + m->m_len);
1161 	do {
1162 		count = min(min(max(len, max_protohdr), space), n->m_len);
1163 		memcpy(mtod(m, caddr_t) + m->m_len, mtod(n, caddr_t),
1164 		    (unsigned)count);
1165 		len -= count;
1166 		m->m_len += count;
1167 		n->m_len -= count;
1168 		space -= count;
1169 		if (n->m_len)
1170 			n->m_data += count;
1171 		else
1172 			n = m_free(n);
1173 	} while (len > 0 && n);
1174 	if (len > 0) {
1175 		(void) m_free(m);
1176 		goto bad;
1177 	}
1178 	m->m_next = n;
1179 	return (m);
1180  bad:
1181 	m_freem(n);
1182 	MSFail++;
1183 	return (NULL);
1184 }
1185 
1186 /*
1187  * Partition an mbuf chain in two pieces, returning the tail --
1188  * all but the first len0 bytes.  In case of failure, it returns NULL and
1189  * attempts to restore the chain to its original state.
1190  *
1191  * Note that the resulting mbufs might be read-only, because the new
1192  * mbuf can end up sharing an mbuf cluster with the original mbuf if
1193  * the "breaking point" happens to lie within a cluster mbuf. Use the
1194  * M_WRITABLE() macro to check for this case.
1195  */
1196 struct mbuf *
1197 m_split(struct mbuf *m0, int len0, int wait)
1198 {
1199 	struct mbuf *m, *n;
1200 	u_int len = len0, remain;
1201 
1202 	MBUF_CHECKSLEEP(wait);
1203 	for (m = m0; m && len > m->m_len; m = m->m_next)
1204 		len -= m->m_len;
1205 	if (m == NULL)
1206 		return (NULL);
1207 	remain = m->m_len - len;
1208 	if (m0->m_flags & M_PKTHDR && remain == 0) {
1209 		n = m_gethdr(wait, m0->m_type);
1210 		if (n == NULL)
1211 			return (NULL);
1212 		n->m_next = m->m_next;
1213 		m->m_next = NULL;
1214 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1215 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1216 		m0->m_pkthdr.len = len0;
1217 		return (n);
1218 	} else if (m0->m_flags & M_PKTHDR) {
1219 		n = m_gethdr(wait, m0->m_type);
1220 		if (n == NULL)
1221 			return (NULL);
1222 		n->m_pkthdr.rcvif = m0->m_pkthdr.rcvif;
1223 		n->m_pkthdr.len = m0->m_pkthdr.len - len0;
1224 		m0->m_pkthdr.len = len0;
1225 		if (m->m_flags & M_EXT)
1226 			goto extpacket;
1227 		if (remain > MHLEN) {
1228 			/* m can't be the lead packet */
1229 			MH_ALIGN(n, 0);
1230 			n->m_next = m_split(m, len, wait);
1231 			if (n->m_next == NULL) {
1232 				(void) m_free(n);
1233 				return (NULL);
1234 			} else {
1235 				n->m_len = 0;
1236 				return (n);
1237 			}
1238 		} else
1239 			MH_ALIGN(n, remain);
1240 	} else if (remain == 0) {
1241 		n = m->m_next;
1242 		m->m_next = NULL;
1243 		return (n);
1244 	} else {
1245 		n = m_get(wait, m->m_type);
1246 		if (n == NULL)
1247 			return (NULL);
1248 		M_ALIGN(n, remain);
1249 	}
1250 extpacket:
1251 	if (m->m_flags & M_EXT) {
1252 		n->m_data = m->m_data + len;
1253 		mb_dupcl(n, m);
1254 	} else {
1255 		bcopy(mtod(m, caddr_t) + len, mtod(n, caddr_t), remain);
1256 	}
1257 	n->m_len = remain;
1258 	m->m_len = len;
1259 	n->m_next = m->m_next;
1260 	m->m_next = NULL;
1261 	return (n);
1262 }
1263 /*
1264  * Routine to copy from device local memory into mbufs.
1265  * Note that `off' argument is offset into first mbuf of target chain from
1266  * which to begin copying the data to.
1267  */
1268 struct mbuf *
1269 m_devget(char *buf, int totlen, int off, struct ifnet *ifp,
1270     void (*copy)(char *from, caddr_t to, u_int len))
1271 {
1272 	struct mbuf *m;
1273 	struct mbuf *top = NULL, **mp = &top;
1274 	int len;
1275 
1276 	if (off < 0 || off > MHLEN)
1277 		return (NULL);
1278 
1279 	while (totlen > 0) {
1280 		if (top == NULL) {	/* First one, must be PKTHDR */
1281 			if (totlen + off >= MINCLSIZE) {
1282 				m = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
1283 				len = MCLBYTES;
1284 			} else {
1285 				m = m_gethdr(M_NOWAIT, MT_DATA);
1286 				len = MHLEN;
1287 
1288 				/* Place initial small packet/header at end of mbuf */
1289 				if (m && totlen + off + max_linkhdr <= MLEN) {
1290 					m->m_data += max_linkhdr;
1291 					len -= max_linkhdr;
1292 				}
1293 			}
1294 			if (m == NULL)
1295 				return NULL;
1296 			m->m_pkthdr.rcvif = ifp;
1297 			m->m_pkthdr.len = totlen;
1298 		} else {
1299 			if (totlen + off >= MINCLSIZE) {
1300 				m = m_getcl(M_NOWAIT, MT_DATA, 0);
1301 				len = MCLBYTES;
1302 			} else {
1303 				m = m_get(M_NOWAIT, MT_DATA);
1304 				len = MLEN;
1305 			}
1306 			if (m == NULL) {
1307 				m_freem(top);
1308 				return NULL;
1309 			}
1310 		}
1311 		if (off) {
1312 			m->m_data += off;
1313 			len -= off;
1314 			off = 0;
1315 		}
1316 		m->m_len = len = min(totlen, len);
1317 		if (copy)
1318 			copy(buf, mtod(m, caddr_t), (u_int)len);
1319 		else
1320 			bcopy(buf, mtod(m, caddr_t), (u_int)len);
1321 		buf += len;
1322 		*mp = m;
1323 		mp = &m->m_next;
1324 		totlen -= len;
1325 	}
1326 	return (top);
1327 }
1328 
1329 /*
1330  * Copy data from a buffer back into the indicated mbuf chain,
1331  * starting "off" bytes from the beginning, extending the mbuf
1332  * chain if necessary.
1333  */
1334 void
1335 m_copyback(struct mbuf *m0, int off, int len, c_caddr_t cp)
1336 {
1337 	int mlen;
1338 	struct mbuf *m = m0, *n;
1339 	int totlen = 0;
1340 
1341 	if (m0 == NULL)
1342 		return;
1343 	while (off > (mlen = m->m_len)) {
1344 		off -= mlen;
1345 		totlen += mlen;
1346 		if (m->m_next == NULL) {
1347 			n = m_get(M_NOWAIT, m->m_type);
1348 			if (n == NULL)
1349 				goto out;
1350 			bzero(mtod(n, caddr_t), MLEN);
1351 			n->m_len = min(MLEN, len + off);
1352 			m->m_next = n;
1353 		}
1354 		m = m->m_next;
1355 	}
1356 	while (len > 0) {
1357 		if (m->m_next == NULL && (len > m->m_len - off)) {
1358 			m->m_len += min(len - (m->m_len - off),
1359 			    M_TRAILINGSPACE(m));
1360 		}
1361 		mlen = min (m->m_len - off, len);
1362 		bcopy(cp, off + mtod(m, caddr_t), (u_int)mlen);
1363 		cp += mlen;
1364 		len -= mlen;
1365 		mlen += off;
1366 		off = 0;
1367 		totlen += mlen;
1368 		if (len == 0)
1369 			break;
1370 		if (m->m_next == NULL) {
1371 			n = m_get(M_NOWAIT, m->m_type);
1372 			if (n == NULL)
1373 				break;
1374 			n->m_len = min(MLEN, len);
1375 			m->m_next = n;
1376 		}
1377 		m = m->m_next;
1378 	}
1379 out:	if (((m = m0)->m_flags & M_PKTHDR) && (m->m_pkthdr.len < totlen))
1380 		m->m_pkthdr.len = totlen;
1381 }
1382 
1383 /*
1384  * Append the specified data to the indicated mbuf chain,
1385  * Extend the mbuf chain if the new data does not fit in
1386  * existing space.
1387  *
1388  * Return 1 if able to complete the job; otherwise 0.
1389  */
1390 int
1391 m_append(struct mbuf *m0, int len, c_caddr_t cp)
1392 {
1393 	struct mbuf *m, *n;
1394 	int remainder, space;
1395 
1396 	for (m = m0; m->m_next != NULL; m = m->m_next)
1397 		;
1398 	remainder = len;
1399 	space = M_TRAILINGSPACE(m);
1400 	if (space > 0) {
1401 		/*
1402 		 * Copy into available space.
1403 		 */
1404 		if (space > remainder)
1405 			space = remainder;
1406 		bcopy(cp, mtod(m, caddr_t) + m->m_len, space);
1407 		m->m_len += space;
1408 		cp += space, remainder -= space;
1409 	}
1410 	while (remainder > 0) {
1411 		/*
1412 		 * Allocate a new mbuf; could check space
1413 		 * and allocate a cluster instead.
1414 		 */
1415 		n = m_get(M_NOWAIT, m->m_type);
1416 		if (n == NULL)
1417 			break;
1418 		n->m_len = min(MLEN, remainder);
1419 		bcopy(cp, mtod(n, caddr_t), n->m_len);
1420 		cp += n->m_len, remainder -= n->m_len;
1421 		m->m_next = n;
1422 		m = n;
1423 	}
1424 	if (m0->m_flags & M_PKTHDR)
1425 		m0->m_pkthdr.len += len - remainder;
1426 	return (remainder == 0);
1427 }
1428 
1429 /*
1430  * Apply function f to the data in an mbuf chain starting "off" bytes from
1431  * the beginning, continuing for "len" bytes.
1432  */
1433 int
1434 m_apply(struct mbuf *m, int off, int len,
1435     int (*f)(void *, void *, u_int), void *arg)
1436 {
1437 	u_int count;
1438 	int rval;
1439 
1440 	KASSERT(off >= 0, ("m_apply, negative off %d", off));
1441 	KASSERT(len >= 0, ("m_apply, negative len %d", len));
1442 	while (off > 0) {
1443 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1444 		if (off < m->m_len)
1445 			break;
1446 		off -= m->m_len;
1447 		m = m->m_next;
1448 	}
1449 	while (len > 0) {
1450 		KASSERT(m != NULL, ("m_apply, offset > size of mbuf chain"));
1451 		count = min(m->m_len - off, len);
1452 		rval = (*f)(arg, mtod(m, caddr_t) + off, count);
1453 		if (rval)
1454 			return (rval);
1455 		len -= count;
1456 		off = 0;
1457 		m = m->m_next;
1458 	}
1459 	return (0);
1460 }
1461 
1462 /*
1463  * Return a pointer to mbuf/offset of location in mbuf chain.
1464  */
1465 struct mbuf *
1466 m_getptr(struct mbuf *m, int loc, int *off)
1467 {
1468 
1469 	while (loc >= 0) {
1470 		/* Normal end of search. */
1471 		if (m->m_len > loc) {
1472 			*off = loc;
1473 			return (m);
1474 		} else {
1475 			loc -= m->m_len;
1476 			if (m->m_next == NULL) {
1477 				if (loc == 0) {
1478 					/* Point at the end of valid data. */
1479 					*off = m->m_len;
1480 					return (m);
1481 				}
1482 				return (NULL);
1483 			}
1484 			m = m->m_next;
1485 		}
1486 	}
1487 	return (NULL);
1488 }
1489 
1490 void
1491 m_print(const struct mbuf *m, int maxlen)
1492 {
1493 	int len;
1494 	int pdata;
1495 	const struct mbuf *m2;
1496 
1497 	if (m == NULL) {
1498 		printf("mbuf: %p\n", m);
1499 		return;
1500 	}
1501 
1502 	if (m->m_flags & M_PKTHDR)
1503 		len = m->m_pkthdr.len;
1504 	else
1505 		len = -1;
1506 	m2 = m;
1507 	while (m2 != NULL && (len == -1 || len)) {
1508 		pdata = m2->m_len;
1509 		if (maxlen != -1 && pdata > maxlen)
1510 			pdata = maxlen;
1511 		printf("mbuf: %p len: %d, next: %p, %b%s", m2, m2->m_len,
1512 		    m2->m_next, m2->m_flags, "\20\20freelist\17skipfw"
1513 		    "\11proto5\10proto4\7proto3\6proto2\5proto1\4rdonly"
1514 		    "\3eor\2pkthdr\1ext", pdata ? "" : "\n");
1515 		if (pdata)
1516 			printf(", %*D\n", pdata, (u_char *)m2->m_data, "-");
1517 		if (len != -1)
1518 			len -= m2->m_len;
1519 		m2 = m2->m_next;
1520 	}
1521 	if (len > 0)
1522 		printf("%d bytes unaccounted for.\n", len);
1523 	return;
1524 }
1525 
1526 u_int
1527 m_fixhdr(struct mbuf *m0)
1528 {
1529 	u_int len;
1530 
1531 	len = m_length(m0, NULL);
1532 	m0->m_pkthdr.len = len;
1533 	return (len);
1534 }
1535 
1536 u_int
1537 m_length(struct mbuf *m0, struct mbuf **last)
1538 {
1539 	struct mbuf *m;
1540 	u_int len;
1541 
1542 	len = 0;
1543 	for (m = m0; m != NULL; m = m->m_next) {
1544 		len += m->m_len;
1545 		if (m->m_next == NULL)
1546 			break;
1547 	}
1548 	if (last != NULL)
1549 		*last = m;
1550 	return (len);
1551 }
1552 
1553 /*
1554  * Defragment a mbuf chain, returning the shortest possible
1555  * chain of mbufs and clusters.  If allocation fails and
1556  * this cannot be completed, NULL will be returned, but
1557  * the passed in chain will be unchanged.  Upon success,
1558  * the original chain will be freed, and the new chain
1559  * will be returned.
1560  *
1561  * If a non-packet header is passed in, the original
1562  * mbuf (chain?) will be returned unharmed.
1563  */
1564 struct mbuf *
1565 m_defrag(struct mbuf *m0, int how)
1566 {
1567 	struct mbuf *m_new = NULL, *m_final = NULL;
1568 	int progress = 0, length;
1569 
1570 	MBUF_CHECKSLEEP(how);
1571 	if (!(m0->m_flags & M_PKTHDR))
1572 		return (m0);
1573 
1574 	m_fixhdr(m0); /* Needed sanity check */
1575 
1576 #ifdef MBUF_STRESS_TEST
1577 	if (m_defragrandomfailures) {
1578 		int temp = arc4random() & 0xff;
1579 		if (temp == 0xba)
1580 			goto nospace;
1581 	}
1582 #endif
1583 
1584 	if (m0->m_pkthdr.len > MHLEN)
1585 		m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1586 	else
1587 		m_final = m_gethdr(how, MT_DATA);
1588 
1589 	if (m_final == NULL)
1590 		goto nospace;
1591 
1592 	if (m_dup_pkthdr(m_final, m0, how) == 0)
1593 		goto nospace;
1594 
1595 	m_new = m_final;
1596 
1597 	while (progress < m0->m_pkthdr.len) {
1598 		length = m0->m_pkthdr.len - progress;
1599 		if (length > MCLBYTES)
1600 			length = MCLBYTES;
1601 
1602 		if (m_new == NULL) {
1603 			if (length > MLEN)
1604 				m_new = m_getcl(how, MT_DATA, 0);
1605 			else
1606 				m_new = m_get(how, MT_DATA);
1607 			if (m_new == NULL)
1608 				goto nospace;
1609 		}
1610 
1611 		m_copydata(m0, progress, length, mtod(m_new, caddr_t));
1612 		progress += length;
1613 		m_new->m_len = length;
1614 		if (m_new != m_final)
1615 			m_cat(m_final, m_new);
1616 		m_new = NULL;
1617 	}
1618 #ifdef MBUF_STRESS_TEST
1619 	if (m0->m_next == NULL)
1620 		m_defraguseless++;
1621 #endif
1622 	m_freem(m0);
1623 	m0 = m_final;
1624 #ifdef MBUF_STRESS_TEST
1625 	m_defragpackets++;
1626 	m_defragbytes += m0->m_pkthdr.len;
1627 #endif
1628 	return (m0);
1629 nospace:
1630 #ifdef MBUF_STRESS_TEST
1631 	m_defragfailure++;
1632 #endif
1633 	if (m_final)
1634 		m_freem(m_final);
1635 	return (NULL);
1636 }
1637 
1638 /*
1639  * Defragment an mbuf chain, returning at most maxfrags separate
1640  * mbufs+clusters.  If this is not possible NULL is returned and
1641  * the original mbuf chain is left in it's present (potentially
1642  * modified) state.  We use two techniques: collapsing consecutive
1643  * mbufs and replacing consecutive mbufs by a cluster.
1644  *
1645  * NB: this should really be named m_defrag but that name is taken
1646  */
1647 struct mbuf *
1648 m_collapse(struct mbuf *m0, int how, int maxfrags)
1649 {
1650 	struct mbuf *m, *n, *n2, **prev;
1651 	u_int curfrags;
1652 
1653 	/*
1654 	 * Calculate the current number of frags.
1655 	 */
1656 	curfrags = 0;
1657 	for (m = m0; m != NULL; m = m->m_next)
1658 		curfrags++;
1659 	/*
1660 	 * First, try to collapse mbufs.  Note that we always collapse
1661 	 * towards the front so we don't need to deal with moving the
1662 	 * pkthdr.  This may be suboptimal if the first mbuf has much
1663 	 * less data than the following.
1664 	 */
1665 	m = m0;
1666 again:
1667 	for (;;) {
1668 		n = m->m_next;
1669 		if (n == NULL)
1670 			break;
1671 		if (M_WRITABLE(m) &&
1672 		    n->m_len < M_TRAILINGSPACE(m)) {
1673 			bcopy(mtod(n, void *), mtod(m, char *) + m->m_len,
1674 				n->m_len);
1675 			m->m_len += n->m_len;
1676 			m->m_next = n->m_next;
1677 			m_free(n);
1678 			if (--curfrags <= maxfrags)
1679 				return m0;
1680 		} else
1681 			m = n;
1682 	}
1683 	KASSERT(maxfrags > 1,
1684 		("maxfrags %u, but normal collapse failed", maxfrags));
1685 	/*
1686 	 * Collapse consecutive mbufs to a cluster.
1687 	 */
1688 	prev = &m0->m_next;		/* NB: not the first mbuf */
1689 	while ((n = *prev) != NULL) {
1690 		if ((n2 = n->m_next) != NULL &&
1691 		    n->m_len + n2->m_len < MCLBYTES) {
1692 			m = m_getcl(how, MT_DATA, 0);
1693 			if (m == NULL)
1694 				goto bad;
1695 			bcopy(mtod(n, void *), mtod(m, void *), n->m_len);
1696 			bcopy(mtod(n2, void *), mtod(m, char *) + n->m_len,
1697 				n2->m_len);
1698 			m->m_len = n->m_len + n2->m_len;
1699 			m->m_next = n2->m_next;
1700 			*prev = m;
1701 			m_free(n);
1702 			m_free(n2);
1703 			if (--curfrags <= maxfrags)	/* +1 cl -2 mbufs */
1704 				return m0;
1705 			/*
1706 			 * Still not there, try the normal collapse
1707 			 * again before we allocate another cluster.
1708 			 */
1709 			goto again;
1710 		}
1711 		prev = &n->m_next;
1712 	}
1713 	/*
1714 	 * No place where we can collapse to a cluster; punt.
1715 	 * This can occur if, for example, you request 2 frags
1716 	 * but the packet requires that both be clusters (we
1717 	 * never reallocate the first mbuf to avoid moving the
1718 	 * packet header).
1719 	 */
1720 bad:
1721 	return NULL;
1722 }
1723 
1724 #ifdef MBUF_STRESS_TEST
1725 
1726 /*
1727  * Fragment an mbuf chain.  There's no reason you'd ever want to do
1728  * this in normal usage, but it's great for stress testing various
1729  * mbuf consumers.
1730  *
1731  * If fragmentation is not possible, the original chain will be
1732  * returned.
1733  *
1734  * Possible length values:
1735  * 0	 no fragmentation will occur
1736  * > 0	each fragment will be of the specified length
1737  * -1	each fragment will be the same random value in length
1738  * -2	each fragment's length will be entirely random
1739  * (Random values range from 1 to 256)
1740  */
1741 struct mbuf *
1742 m_fragment(struct mbuf *m0, int how, int length)
1743 {
1744 	struct mbuf *m_new = NULL, *m_final = NULL;
1745 	int progress = 0;
1746 
1747 	if (!(m0->m_flags & M_PKTHDR))
1748 		return (m0);
1749 
1750 	if ((length == 0) || (length < -2))
1751 		return (m0);
1752 
1753 	m_fixhdr(m0); /* Needed sanity check */
1754 
1755 	m_final = m_getcl(how, MT_DATA, M_PKTHDR);
1756 
1757 	if (m_final == NULL)
1758 		goto nospace;
1759 
1760 	if (m_dup_pkthdr(m_final, m0, how) == 0)
1761 		goto nospace;
1762 
1763 	m_new = m_final;
1764 
1765 	if (length == -1)
1766 		length = 1 + (arc4random() & 255);
1767 
1768 	while (progress < m0->m_pkthdr.len) {
1769 		int fraglen;
1770 
1771 		if (length > 0)
1772 			fraglen = length;
1773 		else
1774 			fraglen = 1 + (arc4random() & 255);
1775 		if (fraglen > m0->m_pkthdr.len - progress)
1776 			fraglen = m0->m_pkthdr.len - progress;
1777 
1778 		if (fraglen > MCLBYTES)
1779 			fraglen = MCLBYTES;
1780 
1781 		if (m_new == NULL) {
1782 			m_new = m_getcl(how, MT_DATA, 0);
1783 			if (m_new == NULL)
1784 				goto nospace;
1785 		}
1786 
1787 		m_copydata(m0, progress, fraglen, mtod(m_new, caddr_t));
1788 		progress += fraglen;
1789 		m_new->m_len = fraglen;
1790 		if (m_new != m_final)
1791 			m_cat(m_final, m_new);
1792 		m_new = NULL;
1793 	}
1794 	m_freem(m0);
1795 	m0 = m_final;
1796 	return (m0);
1797 nospace:
1798 	if (m_final)
1799 		m_freem(m_final);
1800 	/* Return the original chain on failure */
1801 	return (m0);
1802 }
1803 
1804 #endif
1805 
1806 /*
1807  * Copy the contents of uio into a properly sized mbuf chain.
1808  */
1809 struct mbuf *
1810 m_uiotombuf(struct uio *uio, int how, int len, int align, int flags)
1811 {
1812 	struct mbuf *m, *mb;
1813 	int error, length;
1814 	ssize_t total;
1815 	int progress = 0;
1816 
1817 	/*
1818 	 * len can be zero or an arbitrary large value bound by
1819 	 * the total data supplied by the uio.
1820 	 */
1821 	if (len > 0)
1822 		total = min(uio->uio_resid, len);
1823 	else
1824 		total = uio->uio_resid;
1825 
1826 	/*
1827 	 * The smallest unit returned by m_getm2() is a single mbuf
1828 	 * with pkthdr.  We can't align past it.
1829 	 */
1830 	if (align >= MHLEN)
1831 		return (NULL);
1832 
1833 	/*
1834 	 * Give us the full allocation or nothing.
1835 	 * If len is zero return the smallest empty mbuf.
1836 	 */
1837 	m = m_getm2(NULL, max(total + align, 1), how, MT_DATA, flags);
1838 	if (m == NULL)
1839 		return (NULL);
1840 	m->m_data += align;
1841 
1842 	/* Fill all mbufs with uio data and update header information. */
1843 	for (mb = m; mb != NULL; mb = mb->m_next) {
1844 		length = min(M_TRAILINGSPACE(mb), total - progress);
1845 
1846 		error = uiomove(mtod(mb, void *), length, uio);
1847 		if (error) {
1848 			m_freem(m);
1849 			return (NULL);
1850 		}
1851 
1852 		mb->m_len = length;
1853 		progress += length;
1854 		if (flags & M_PKTHDR)
1855 			m->m_pkthdr.len += length;
1856 	}
1857 	KASSERT(progress == total, ("%s: progress != total", __func__));
1858 
1859 	return (m);
1860 }
1861 
1862 /*
1863  * Copy an mbuf chain into a uio limited by len if set.
1864  */
1865 int
1866 m_mbuftouio(struct uio *uio, struct mbuf *m, int len)
1867 {
1868 	int error, length, total;
1869 	int progress = 0;
1870 
1871 	if (len > 0)
1872 		total = min(uio->uio_resid, len);
1873 	else
1874 		total = uio->uio_resid;
1875 
1876 	/* Fill the uio with data from the mbufs. */
1877 	for (; m != NULL; m = m->m_next) {
1878 		length = min(m->m_len, total - progress);
1879 
1880 		error = uiomove(mtod(m, void *), length, uio);
1881 		if (error)
1882 			return (error);
1883 
1884 		progress += length;
1885 	}
1886 
1887 	return (0);
1888 }
1889 
1890 /*
1891  * Set the m_data pointer of a newly-allocated mbuf
1892  * to place an object of the specified size at the
1893  * end of the mbuf, longword aligned.
1894  */
1895 void
1896 m_align(struct mbuf *m, int len)
1897 {
1898 #ifdef INVARIANTS
1899 	const char *msg = "%s: not a virgin mbuf";
1900 #endif
1901 	int adjust;
1902 
1903 	if (m->m_flags & M_EXT) {
1904 		KASSERT(m->m_data == m->m_ext.ext_buf, (msg, __func__));
1905 		adjust = m->m_ext.ext_size - len;
1906 	} else if (m->m_flags & M_PKTHDR) {
1907 		KASSERT(m->m_data == m->m_pktdat, (msg, __func__));
1908 		adjust = MHLEN - len;
1909 	} else {
1910 		KASSERT(m->m_data == m->m_dat, (msg, __func__));
1911 		adjust = MLEN - len;
1912 	}
1913 
1914 	m->m_data += adjust &~ (sizeof(long)-1);
1915 }
1916 
1917 /*
1918  * Create a writable copy of the mbuf chain.  While doing this
1919  * we compact the chain with a goal of producing a chain with
1920  * at most two mbufs.  The second mbuf in this chain is likely
1921  * to be a cluster.  The primary purpose of this work is to create
1922  * a writable packet for encryption, compression, etc.  The
1923  * secondary goal is to linearize the data so the data can be
1924  * passed to crypto hardware in the most efficient manner possible.
1925  */
1926 struct mbuf *
1927 m_unshare(struct mbuf *m0, int how)
1928 {
1929 	struct mbuf *m, *mprev;
1930 	struct mbuf *n, *mfirst, *mlast;
1931 	int len, off;
1932 
1933 	mprev = NULL;
1934 	for (m = m0; m != NULL; m = mprev->m_next) {
1935 		/*
1936 		 * Regular mbufs are ignored unless there's a cluster
1937 		 * in front of it that we can use to coalesce.  We do
1938 		 * the latter mainly so later clusters can be coalesced
1939 		 * also w/o having to handle them specially (i.e. convert
1940 		 * mbuf+cluster -> cluster).  This optimization is heavily
1941 		 * influenced by the assumption that we're running over
1942 		 * Ethernet where MCLBYTES is large enough that the max
1943 		 * packet size will permit lots of coalescing into a
1944 		 * single cluster.  This in turn permits efficient
1945 		 * crypto operations, especially when using hardware.
1946 		 */
1947 		if ((m->m_flags & M_EXT) == 0) {
1948 			if (mprev && (mprev->m_flags & M_EXT) &&
1949 			    m->m_len <= M_TRAILINGSPACE(mprev)) {
1950 				/* XXX: this ignores mbuf types */
1951 				memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1952 				    mtod(m, caddr_t), m->m_len);
1953 				mprev->m_len += m->m_len;
1954 				mprev->m_next = m->m_next;	/* unlink from chain */
1955 				m_free(m);			/* reclaim mbuf */
1956 #if 0
1957 				newipsecstat.ips_mbcoalesced++;
1958 #endif
1959 			} else {
1960 				mprev = m;
1961 			}
1962 			continue;
1963 		}
1964 		/*
1965 		 * Writable mbufs are left alone (for now).
1966 		 */
1967 		if (M_WRITABLE(m)) {
1968 			mprev = m;
1969 			continue;
1970 		}
1971 
1972 		/*
1973 		 * Not writable, replace with a copy or coalesce with
1974 		 * the previous mbuf if possible (since we have to copy
1975 		 * it anyway, we try to reduce the number of mbufs and
1976 		 * clusters so that future work is easier).
1977 		 */
1978 		KASSERT(m->m_flags & M_EXT, ("m_flags 0x%x", m->m_flags));
1979 		/* NB: we only coalesce into a cluster or larger */
1980 		if (mprev != NULL && (mprev->m_flags & M_EXT) &&
1981 		    m->m_len <= M_TRAILINGSPACE(mprev)) {
1982 			/* XXX: this ignores mbuf types */
1983 			memcpy(mtod(mprev, caddr_t) + mprev->m_len,
1984 			    mtod(m, caddr_t), m->m_len);
1985 			mprev->m_len += m->m_len;
1986 			mprev->m_next = m->m_next;	/* unlink from chain */
1987 			m_free(m);			/* reclaim mbuf */
1988 #if 0
1989 			newipsecstat.ips_clcoalesced++;
1990 #endif
1991 			continue;
1992 		}
1993 
1994 		/*
1995 		 * Allocate new space to hold the copy and copy the data.
1996 		 * We deal with jumbo mbufs (i.e. m_len > MCLBYTES) by
1997 		 * splitting them into clusters.  We could just malloc a
1998 		 * buffer and make it external but too many device drivers
1999 		 * don't know how to break up the non-contiguous memory when
2000 		 * doing DMA.
2001 		 */
2002 		n = m_getcl(how, m->m_type, m->m_flags);
2003 		if (n == NULL) {
2004 			m_freem(m0);
2005 			return (NULL);
2006 		}
2007 		len = m->m_len;
2008 		off = 0;
2009 		mfirst = n;
2010 		mlast = NULL;
2011 		for (;;) {
2012 			int cc = min(len, MCLBYTES);
2013 			memcpy(mtod(n, caddr_t), mtod(m, caddr_t) + off, cc);
2014 			n->m_len = cc;
2015 			if (mlast != NULL)
2016 				mlast->m_next = n;
2017 			mlast = n;
2018 #if 0
2019 			newipsecstat.ips_clcopied++;
2020 #endif
2021 
2022 			len -= cc;
2023 			if (len <= 0)
2024 				break;
2025 			off += cc;
2026 
2027 			n = m_getcl(how, m->m_type, m->m_flags);
2028 			if (n == NULL) {
2029 				m_freem(mfirst);
2030 				m_freem(m0);
2031 				return (NULL);
2032 			}
2033 		}
2034 		n->m_next = m->m_next;
2035 		if (mprev == NULL)
2036 			m0 = mfirst;		/* new head of chain */
2037 		else
2038 			mprev->m_next = mfirst;	/* replace old mbuf */
2039 		m_free(m);			/* release old mbuf */
2040 		mprev = mfirst;
2041 	}
2042 	return (m0);
2043 }
2044 
2045 #ifdef MBUF_PROFILING
2046 
2047 #define MP_BUCKETS 32 /* don't just change this as things may overflow.*/
2048 struct mbufprofile {
2049 	uintmax_t wasted[MP_BUCKETS];
2050 	uintmax_t used[MP_BUCKETS];
2051 	uintmax_t segments[MP_BUCKETS];
2052 } mbprof;
2053 
2054 #define MP_MAXDIGITS 21	/* strlen("16,000,000,000,000,000,000") == 21 */
2055 #define MP_NUMLINES 6
2056 #define MP_NUMSPERLINE 16
2057 #define MP_EXTRABYTES 64	/* > strlen("used:\nwasted:\nsegments:\n") */
2058 /* work out max space needed and add a bit of spare space too */
2059 #define MP_MAXLINE ((MP_MAXDIGITS+1) * MP_NUMSPERLINE)
2060 #define MP_BUFSIZE ((MP_MAXLINE * MP_NUMLINES) + 1 + MP_EXTRABYTES)
2061 
2062 char mbprofbuf[MP_BUFSIZE];
2063 
2064 void
2065 m_profile(struct mbuf *m)
2066 {
2067 	int segments = 0;
2068 	int used = 0;
2069 	int wasted = 0;
2070 
2071 	while (m) {
2072 		segments++;
2073 		used += m->m_len;
2074 		if (m->m_flags & M_EXT) {
2075 			wasted += MHLEN - sizeof(m->m_ext) +
2076 			    m->m_ext.ext_size - m->m_len;
2077 		} else {
2078 			if (m->m_flags & M_PKTHDR)
2079 				wasted += MHLEN - m->m_len;
2080 			else
2081 				wasted += MLEN - m->m_len;
2082 		}
2083 		m = m->m_next;
2084 	}
2085 	/* be paranoid.. it helps */
2086 	if (segments > MP_BUCKETS - 1)
2087 		segments = MP_BUCKETS - 1;
2088 	if (used > 100000)
2089 		used = 100000;
2090 	if (wasted > 100000)
2091 		wasted = 100000;
2092 	/* store in the appropriate bucket */
2093 	/* don't bother locking. if it's slightly off, so what? */
2094 	mbprof.segments[segments]++;
2095 	mbprof.used[fls(used)]++;
2096 	mbprof.wasted[fls(wasted)]++;
2097 }
2098 
2099 static void
2100 mbprof_textify(void)
2101 {
2102 	int offset;
2103 	char *c;
2104 	uint64_t *p;
2105 
2106 	p = &mbprof.wasted[0];
2107 	c = mbprofbuf;
2108 	offset = snprintf(c, MP_MAXLINE + 10,
2109 	    "wasted:\n"
2110 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2111 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2112 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2113 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2114 #ifdef BIG_ARRAY
2115 	p = &mbprof.wasted[16];
2116 	c += offset;
2117 	offset = snprintf(c, MP_MAXLINE,
2118 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2119 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2120 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2121 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2122 #endif
2123 	p = &mbprof.used[0];
2124 	c += offset;
2125 	offset = snprintf(c, MP_MAXLINE + 10,
2126 	    "used:\n"
2127 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2128 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2129 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2130 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2131 #ifdef BIG_ARRAY
2132 	p = &mbprof.used[16];
2133 	c += offset;
2134 	offset = snprintf(c, MP_MAXLINE,
2135 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2136 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2137 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2138 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2139 #endif
2140 	p = &mbprof.segments[0];
2141 	c += offset;
2142 	offset = snprintf(c, MP_MAXLINE + 10,
2143 	    "segments:\n"
2144 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2145 	    "%ju %ju %ju %ju %ju %ju %ju %ju\n",
2146 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2147 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2148 #ifdef BIG_ARRAY
2149 	p = &mbprof.segments[16];
2150 	c += offset;
2151 	offset = snprintf(c, MP_MAXLINE,
2152 	    "%ju %ju %ju %ju %ju %ju %ju %ju "
2153 	    "%ju %ju %ju %ju %ju %ju %ju %jju",
2154 	    p[0], p[1], p[2], p[3], p[4], p[5], p[6], p[7],
2155 	    p[8], p[9], p[10], p[11], p[12], p[13], p[14], p[15]);
2156 #endif
2157 }
2158 
2159 static int
2160 mbprof_handler(SYSCTL_HANDLER_ARGS)
2161 {
2162 	int error;
2163 
2164 	mbprof_textify();
2165 	error = SYSCTL_OUT(req, mbprofbuf, strlen(mbprofbuf) + 1);
2166 	return (error);
2167 }
2168 
2169 static int
2170 mbprof_clr_handler(SYSCTL_HANDLER_ARGS)
2171 {
2172 	int clear, error;
2173 
2174 	clear = 0;
2175 	error = sysctl_handle_int(oidp, &clear, 0, req);
2176 	if (error || !req->newptr)
2177 		return (error);
2178 
2179 	if (clear) {
2180 		bzero(&mbprof, sizeof(mbprof));
2181 	}
2182 
2183 	return (error);
2184 }
2185 
2186 
2187 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofile, CTLTYPE_STRING|CTLFLAG_RD,
2188 	    NULL, 0, mbprof_handler, "A", "mbuf profiling statistics");
2189 
2190 SYSCTL_PROC(_kern_ipc, OID_AUTO, mbufprofileclr, CTLTYPE_INT|CTLFLAG_RW,
2191 	    NULL, 0, mbprof_clr_handler, "I", "clear mbuf profiling statistics");
2192 #endif
2193 
2194