xref: /freebsd/sys/kern/kern_mbuf.c (revision f6a3b357e9be4c6423c85eff9a847163a0d307c8)
1 /*-
2  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
3  *
4  * Copyright (c) 2004, 2005,
5  *	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice unmodified, this list of conditions and the following
12  *    disclaimer.
13  * 2. Redistributions in binary form must reproduce the above copyright
14  *    notice, this list of conditions and the following disclaimer in the
15  *    documentation and/or other materials provided with the distribution.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  */
29 
30 #include <sys/cdefs.h>
31 __FBSDID("$FreeBSD$");
32 
33 #include "opt_param.h"
34 #include "opt_kern_tls.h"
35 
36 #include <sys/param.h>
37 #include <sys/conf.h>
38 #include <sys/domainset.h>
39 #include <sys/malloc.h>
40 #include <sys/systm.h>
41 #include <sys/mbuf.h>
42 #include <sys/domain.h>
43 #include <sys/eventhandler.h>
44 #include <sys/kernel.h>
45 #include <sys/ktls.h>
46 #include <sys/limits.h>
47 #include <sys/lock.h>
48 #include <sys/mutex.h>
49 #include <sys/protosw.h>
50 #include <sys/refcount.h>
51 #include <sys/sf_buf.h>
52 #include <sys/smp.h>
53 #include <sys/socket.h>
54 #include <sys/sysctl.h>
55 
56 #include <net/if.h>
57 #include <net/if_var.h>
58 
59 #include <vm/vm.h>
60 #include <vm/vm_extern.h>
61 #include <vm/vm_kern.h>
62 #include <vm/vm_page.h>
63 #include <vm/vm_map.h>
64 #include <vm/uma.h>
65 #include <vm/uma_dbg.h>
66 
67 /*
68  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
69  * Zones.
70  *
71  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
72  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
73  * administrator so desires.
74  *
75  * Mbufs are allocated from a UMA Master Zone called the Mbuf
76  * Zone.
77  *
78  * Additionally, FreeBSD provides a Packet Zone, which it
79  * configures as a Secondary Zone to the Mbuf Master Zone,
80  * thus sharing backend Slab kegs with the Mbuf Master Zone.
81  *
82  * Thus common-case allocations and locking are simplified:
83  *
84  *  m_clget()                m_getcl()
85  *    |                         |
86  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
87  *    |   |             [     Packet   ]            |
88  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
89  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
90  *        |                       \________         |
91  *  [ Cluster Keg   ]                      \       /
92  *        |	                         [ Mbuf Keg   ]
93  *  [ Cluster Slabs ]                         |
94  *        |                              [ Mbuf Slabs ]
95  *         \____________(VM)_________________/
96  *
97  *
98  * Whenever an object is allocated with uma_zalloc() out of
99  * one of the Zones its _ctor_ function is executed.  The same
100  * for any deallocation through uma_zfree() the _dtor_ function
101  * is executed.
102  *
103  * Caches are per-CPU and are filled from the Master Zone.
104  *
105  * Whenever an object is allocated from the underlying global
106  * memory pool it gets pre-initialized with the _zinit_ functions.
107  * When the Keg's are overfull objects get decommissioned with
108  * _zfini_ functions and free'd back to the global memory pool.
109  *
110  */
111 
112 int nmbufs;			/* limits number of mbufs */
113 int nmbclusters;		/* limits number of mbuf clusters */
114 int nmbjumbop;			/* limits number of page size jumbo clusters */
115 int nmbjumbo9;			/* limits number of 9k jumbo clusters */
116 int nmbjumbo16;			/* limits number of 16k jumbo clusters */
117 
118 bool mb_use_ext_pgs;		/* use EXT_PGS mbufs for sendfile & TLS */
119 SYSCTL_BOOL(_kern_ipc, OID_AUTO, mb_use_ext_pgs, CTLFLAG_RWTUN,
120     &mb_use_ext_pgs, 0,
121     "Use unmapped mbufs for sendfile(2) and TLS offload");
122 
123 static quad_t maxmbufmem;	/* overall real memory limit for all mbufs */
124 
125 SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0,
126     "Maximum real memory allocatable to various mbuf types");
127 
128 static counter_u64_t snd_tag_count;
129 SYSCTL_COUNTER_U64(_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW,
130     &snd_tag_count, "# of active mbuf send tags");
131 
132 /*
133  * tunable_mbinit() has to be run before any mbuf allocations are done.
134  */
135 static void
136 tunable_mbinit(void *dummy)
137 {
138 	quad_t realmem;
139 
140 	/*
141 	 * The default limit for all mbuf related memory is 1/2 of all
142 	 * available kernel memory (physical or kmem).
143 	 * At most it can be 3/4 of available kernel memory.
144 	 */
145 	realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size);
146 	maxmbufmem = realmem / 2;
147 	TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
148 	if (maxmbufmem > realmem / 4 * 3)
149 		maxmbufmem = realmem / 4 * 3;
150 
151 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
152 	if (nmbclusters == 0)
153 		nmbclusters = maxmbufmem / MCLBYTES / 4;
154 
155 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
156 	if (nmbjumbop == 0)
157 		nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
158 
159 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
160 	if (nmbjumbo9 == 0)
161 		nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
162 
163 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
164 	if (nmbjumbo16 == 0)
165 		nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
166 
167 	/*
168 	 * We need at least as many mbufs as we have clusters of
169 	 * the various types added together.
170 	 */
171 	TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
172 	if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
173 		nmbufs = lmax(maxmbufmem / MSIZE / 5,
174 		    nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
175 }
176 SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
177 
178 static int
179 sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
180 {
181 	int error, newnmbclusters;
182 
183 	newnmbclusters = nmbclusters;
184 	error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
185 	if (error == 0 && req->newptr && newnmbclusters != nmbclusters) {
186 		if (newnmbclusters > nmbclusters &&
187 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
188 			nmbclusters = newnmbclusters;
189 			nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
190 			EVENTHANDLER_INVOKE(nmbclusters_change);
191 		} else
192 			error = EINVAL;
193 	}
194 	return (error);
195 }
196 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
197 &nmbclusters, 0, sysctl_nmbclusters, "IU",
198     "Maximum number of mbuf clusters allowed");
199 
200 static int
201 sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
202 {
203 	int error, newnmbjumbop;
204 
205 	newnmbjumbop = nmbjumbop;
206 	error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
207 	if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) {
208 		if (newnmbjumbop > nmbjumbop &&
209 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
210 			nmbjumbop = newnmbjumbop;
211 			nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
212 		} else
213 			error = EINVAL;
214 	}
215 	return (error);
216 }
217 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
218 &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
219     "Maximum number of mbuf page size jumbo clusters allowed");
220 
221 static int
222 sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
223 {
224 	int error, newnmbjumbo9;
225 
226 	newnmbjumbo9 = nmbjumbo9;
227 	error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
228 	if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) {
229 		if (newnmbjumbo9 > nmbjumbo9 &&
230 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
231 			nmbjumbo9 = newnmbjumbo9;
232 			nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
233 		} else
234 			error = EINVAL;
235 	}
236 	return (error);
237 }
238 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
239 &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
240     "Maximum number of mbuf 9k jumbo clusters allowed");
241 
242 static int
243 sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
244 {
245 	int error, newnmbjumbo16;
246 
247 	newnmbjumbo16 = nmbjumbo16;
248 	error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
249 	if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) {
250 		if (newnmbjumbo16 > nmbjumbo16 &&
251 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
252 			nmbjumbo16 = newnmbjumbo16;
253 			nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
254 		} else
255 			error = EINVAL;
256 	}
257 	return (error);
258 }
259 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
260 &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
261     "Maximum number of mbuf 16k jumbo clusters allowed");
262 
263 static int
264 sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
265 {
266 	int error, newnmbufs;
267 
268 	newnmbufs = nmbufs;
269 	error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
270 	if (error == 0 && req->newptr && newnmbufs != nmbufs) {
271 		if (newnmbufs > nmbufs) {
272 			nmbufs = newnmbufs;
273 			nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
274 			EVENTHANDLER_INVOKE(nmbufs_change);
275 		} else
276 			error = EINVAL;
277 	}
278 	return (error);
279 }
280 SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW,
281 &nmbufs, 0, sysctl_nmbufs, "IU",
282     "Maximum number of mbufs allowed");
283 
284 /*
285  * Zones from which we allocate.
286  */
287 uma_zone_t	zone_mbuf;
288 uma_zone_t	zone_clust;
289 uma_zone_t	zone_pack;
290 uma_zone_t	zone_jumbop;
291 uma_zone_t	zone_jumbo9;
292 uma_zone_t	zone_jumbo16;
293 uma_zone_t	zone_extpgs;
294 
295 /*
296  * Local prototypes.
297  */
298 static int	mb_ctor_mbuf(void *, int, void *, int);
299 static int	mb_ctor_clust(void *, int, void *, int);
300 static int	mb_ctor_pack(void *, int, void *, int);
301 static void	mb_dtor_mbuf(void *, int, void *);
302 static void	mb_dtor_pack(void *, int, void *);
303 static int	mb_zinit_pack(void *, int, int);
304 static void	mb_zfini_pack(void *, int);
305 static void	mb_reclaim(uma_zone_t, int);
306 static void    *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
307 
308 /* Ensure that MSIZE is a power of 2. */
309 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
310 
311 _Static_assert(sizeof(struct mbuf_ext_pgs) == 256,
312     "mbuf_ext_pgs size mismatch");
313 
314 /*
315  * Initialize FreeBSD Network buffer allocation.
316  */
317 static void
318 mbuf_init(void *dummy)
319 {
320 
321 	/*
322 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
323 	 */
324 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
325 	    mb_ctor_mbuf, mb_dtor_mbuf,
326 #ifdef INVARIANTS
327 	    trash_init, trash_fini,
328 #else
329 	    NULL, NULL,
330 #endif
331 	    MSIZE - 1, UMA_ZONE_MAXBUCKET);
332 	if (nmbufs > 0)
333 		nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
334 	uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
335 	uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
336 
337 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
338 	    mb_ctor_clust,
339 #ifdef INVARIANTS
340 	    trash_dtor, trash_init, trash_fini,
341 #else
342 	    NULL, NULL, NULL,
343 #endif
344 	    UMA_ALIGN_PTR, 0);
345 	if (nmbclusters > 0)
346 		nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
347 	uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
348 	uma_zone_set_maxaction(zone_clust, mb_reclaim);
349 
350 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
351 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
352 
353 	/* Make jumbo frame zone too. Page size, 9k and 16k. */
354 	zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
355 	    mb_ctor_clust,
356 #ifdef INVARIANTS
357 	    trash_dtor, trash_init, trash_fini,
358 #else
359 	    NULL, NULL, NULL,
360 #endif
361 	    UMA_ALIGN_PTR, 0);
362 	if (nmbjumbop > 0)
363 		nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
364 	uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
365 	uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
366 
367 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
368 	    mb_ctor_clust,
369 #ifdef INVARIANTS
370 	    trash_dtor, trash_init, trash_fini,
371 #else
372 	    NULL, NULL, NULL,
373 #endif
374 	    UMA_ALIGN_PTR, 0);
375 	uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
376 	if (nmbjumbo9 > 0)
377 		nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
378 	uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
379 	uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
380 
381 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
382 	    mb_ctor_clust,
383 #ifdef INVARIANTS
384 	    trash_dtor, trash_init, trash_fini,
385 #else
386 	    NULL, NULL, NULL,
387 #endif
388 	    UMA_ALIGN_PTR, 0);
389 	uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
390 	if (nmbjumbo16 > 0)
391 		nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
392 	uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
393 	uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
394 
395 	zone_extpgs = uma_zcreate(MBUF_EXTPGS_MEM_NAME,
396 	    sizeof(struct mbuf_ext_pgs),
397 #ifdef INVARIANTS
398 	    trash_ctor, trash_dtor, trash_init, trash_fini,
399 #else
400 	    NULL, NULL, NULL, NULL,
401 #endif
402 	    UMA_ALIGN_CACHE, 0);
403 
404 	/*
405 	 * Hook event handler for low-memory situation, used to
406 	 * drain protocols and push data back to the caches (UMA
407 	 * later pushes it back to VM).
408 	 */
409 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
410 	    EVENTHANDLER_PRI_FIRST);
411 
412 	snd_tag_count = counter_u64_alloc(M_WAITOK);
413 }
414 SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
415 
416 #ifdef NETDUMP
417 /*
418  * netdump makes use of a pre-allocated pool of mbufs and clusters.  When
419  * netdump is configured, we initialize a set of UMA cache zones which return
420  * items from this pool.  At panic-time, the regular UMA zone pointers are
421  * overwritten with those of the cache zones so that drivers may allocate and
422  * free mbufs and clusters without attempting to allocate physical memory.
423  *
424  * We keep mbufs and clusters in a pair of mbuf queues.  In particular, for
425  * the purpose of caching clusters, we treat them as mbufs.
426  */
427 static struct mbufq nd_mbufq =
428     { STAILQ_HEAD_INITIALIZER(nd_mbufq.mq_head), 0, INT_MAX };
429 static struct mbufq nd_clustq =
430     { STAILQ_HEAD_INITIALIZER(nd_clustq.mq_head), 0, INT_MAX };
431 
432 static int nd_clsize;
433 static uma_zone_t nd_zone_mbuf;
434 static uma_zone_t nd_zone_clust;
435 static uma_zone_t nd_zone_pack;
436 
437 static int
438 nd_buf_import(void *arg, void **store, int count, int domain __unused,
439     int flags)
440 {
441 	struct mbufq *q;
442 	struct mbuf *m;
443 	int i;
444 
445 	q = arg;
446 
447 	for (i = 0; i < count; i++) {
448 		m = mbufq_dequeue(q);
449 		if (m == NULL)
450 			break;
451 		trash_init(m, q == &nd_mbufq ? MSIZE : nd_clsize, flags);
452 		store[i] = m;
453 	}
454 	KASSERT((flags & M_WAITOK) == 0 || i == count,
455 	    ("%s: ran out of pre-allocated mbufs", __func__));
456 	return (i);
457 }
458 
459 static void
460 nd_buf_release(void *arg, void **store, int count)
461 {
462 	struct mbufq *q;
463 	struct mbuf *m;
464 	int i;
465 
466 	q = arg;
467 
468 	for (i = 0; i < count; i++) {
469 		m = store[i];
470 		(void)mbufq_enqueue(q, m);
471 	}
472 }
473 
474 static int
475 nd_pack_import(void *arg __unused, void **store, int count, int domain __unused,
476     int flags __unused)
477 {
478 	struct mbuf *m;
479 	void *clust;
480 	int i;
481 
482 	for (i = 0; i < count; i++) {
483 		m = m_get(MT_DATA, M_NOWAIT);
484 		if (m == NULL)
485 			break;
486 		clust = uma_zalloc(nd_zone_clust, M_NOWAIT);
487 		if (clust == NULL) {
488 			m_free(m);
489 			break;
490 		}
491 		mb_ctor_clust(clust, nd_clsize, m, 0);
492 		store[i] = m;
493 	}
494 	KASSERT((flags & M_WAITOK) == 0 || i == count,
495 	    ("%s: ran out of pre-allocated mbufs", __func__));
496 	return (i);
497 }
498 
499 static void
500 nd_pack_release(void *arg __unused, void **store, int count)
501 {
502 	struct mbuf *m;
503 	void *clust;
504 	int i;
505 
506 	for (i = 0; i < count; i++) {
507 		m = store[i];
508 		clust = m->m_ext.ext_buf;
509 		uma_zfree(nd_zone_clust, clust);
510 		uma_zfree(nd_zone_mbuf, m);
511 	}
512 }
513 
514 /*
515  * Free the pre-allocated mbufs and clusters reserved for netdump, and destroy
516  * the corresponding UMA cache zones.
517  */
518 void
519 netdump_mbuf_drain(void)
520 {
521 	struct mbuf *m;
522 	void *item;
523 
524 	if (nd_zone_mbuf != NULL) {
525 		uma_zdestroy(nd_zone_mbuf);
526 		nd_zone_mbuf = NULL;
527 	}
528 	if (nd_zone_clust != NULL) {
529 		uma_zdestroy(nd_zone_clust);
530 		nd_zone_clust = NULL;
531 	}
532 	if (nd_zone_pack != NULL) {
533 		uma_zdestroy(nd_zone_pack);
534 		nd_zone_pack = NULL;
535 	}
536 
537 	while ((m = mbufq_dequeue(&nd_mbufq)) != NULL)
538 		m_free(m);
539 	while ((item = mbufq_dequeue(&nd_clustq)) != NULL)
540 		uma_zfree(m_getzone(nd_clsize), item);
541 }
542 
543 /*
544  * Callback invoked immediately prior to starting a netdump.
545  */
546 void
547 netdump_mbuf_dump(void)
548 {
549 
550 	/*
551 	 * All cluster zones return buffers of the size requested by the
552 	 * drivers.  It's up to the driver to reinitialize the zones if the
553 	 * MTU of a netdump-enabled interface changes.
554 	 */
555 	printf("netdump: overwriting mbuf zone pointers\n");
556 	zone_mbuf = nd_zone_mbuf;
557 	zone_clust = nd_zone_clust;
558 	zone_pack = nd_zone_pack;
559 	zone_jumbop = nd_zone_clust;
560 	zone_jumbo9 = nd_zone_clust;
561 	zone_jumbo16 = nd_zone_clust;
562 }
563 
564 /*
565  * Reinitialize the netdump mbuf+cluster pool and cache zones.
566  */
567 void
568 netdump_mbuf_reinit(int nmbuf, int nclust, int clsize)
569 {
570 	struct mbuf *m;
571 	void *item;
572 
573 	netdump_mbuf_drain();
574 
575 	nd_clsize = clsize;
576 
577 	nd_zone_mbuf = uma_zcache_create("netdump_" MBUF_MEM_NAME,
578 	    MSIZE, mb_ctor_mbuf, mb_dtor_mbuf,
579 #ifdef INVARIANTS
580 	    trash_init, trash_fini,
581 #else
582 	    NULL, NULL,
583 #endif
584 	    nd_buf_import, nd_buf_release,
585 	    &nd_mbufq, UMA_ZONE_NOBUCKET);
586 
587 	nd_zone_clust = uma_zcache_create("netdump_" MBUF_CLUSTER_MEM_NAME,
588 	    clsize, mb_ctor_clust,
589 #ifdef INVARIANTS
590 	    trash_dtor, trash_init, trash_fini,
591 #else
592 	    NULL, NULL, NULL,
593 #endif
594 	    nd_buf_import, nd_buf_release,
595 	    &nd_clustq, UMA_ZONE_NOBUCKET);
596 
597 	nd_zone_pack = uma_zcache_create("netdump_" MBUF_PACKET_MEM_NAME,
598 	    MCLBYTES, mb_ctor_pack, mb_dtor_pack, NULL, NULL,
599 	    nd_pack_import, nd_pack_release,
600 	    NULL, UMA_ZONE_NOBUCKET);
601 
602 	while (nmbuf-- > 0) {
603 		m = m_get(MT_DATA, M_WAITOK);
604 		uma_zfree(nd_zone_mbuf, m);
605 	}
606 	while (nclust-- > 0) {
607 		item = uma_zalloc(m_getzone(nd_clsize), M_WAITOK);
608 		uma_zfree(nd_zone_clust, item);
609 	}
610 }
611 #endif /* NETDUMP */
612 
613 /*
614  * UMA backend page allocator for the jumbo frame zones.
615  *
616  * Allocates kernel virtual memory that is backed by contiguous physical
617  * pages.
618  */
619 static void *
620 mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
621     int wait)
622 {
623 
624 	/* Inform UMA that this allocator uses kernel_map/object. */
625 	*flags = UMA_SLAB_KERNEL;
626 	return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
627 	    bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0,
628 	    VM_MEMATTR_DEFAULT));
629 }
630 
631 /*
632  * Constructor for Mbuf master zone.
633  *
634  * The 'arg' pointer points to a mb_args structure which
635  * contains call-specific information required to support the
636  * mbuf allocation API.  See mbuf.h.
637  */
638 static int
639 mb_ctor_mbuf(void *mem, int size, void *arg, int how)
640 {
641 	struct mbuf *m;
642 	struct mb_args *args;
643 	int error;
644 	int flags;
645 	short type;
646 
647 #ifdef INVARIANTS
648 	trash_ctor(mem, size, arg, how);
649 #endif
650 	args = (struct mb_args *)arg;
651 	type = args->type;
652 
653 	/*
654 	 * The mbuf is initialized later.  The caller has the
655 	 * responsibility to set up any MAC labels too.
656 	 */
657 	if (type == MT_NOINIT)
658 		return (0);
659 
660 	m = (struct mbuf *)mem;
661 	flags = args->flags;
662 	MPASS((flags & M_NOFREE) == 0);
663 
664 	error = m_init(m, how, type, flags);
665 
666 	return (error);
667 }
668 
669 /*
670  * The Mbuf master zone destructor.
671  */
672 static void
673 mb_dtor_mbuf(void *mem, int size, void *arg)
674 {
675 	struct mbuf *m;
676 	unsigned long flags;
677 
678 	m = (struct mbuf *)mem;
679 	flags = (unsigned long)arg;
680 
681 	KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
682 	if (!(flags & MB_DTOR_SKIP) && (m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags))
683 		m_tag_delete_chain(m, NULL);
684 #ifdef INVARIANTS
685 	trash_dtor(mem, size, arg);
686 #endif
687 }
688 
689 /*
690  * The Mbuf Packet zone destructor.
691  */
692 static void
693 mb_dtor_pack(void *mem, int size, void *arg)
694 {
695 	struct mbuf *m;
696 
697 	m = (struct mbuf *)mem;
698 	if ((m->m_flags & M_PKTHDR) != 0)
699 		m_tag_delete_chain(m, NULL);
700 
701 	/* Make sure we've got a clean cluster back. */
702 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
703 	KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
704 	KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
705 	KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
706 	KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
707 	KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
708 	KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
709 #ifdef INVARIANTS
710 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
711 #endif
712 	/*
713 	 * If there are processes blocked on zone_clust, waiting for pages
714 	 * to be freed up, cause them to be woken up by draining the
715 	 * packet zone.  We are exposed to a race here (in the check for
716 	 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
717 	 * is deliberate. We don't want to acquire the zone lock for every
718 	 * mbuf free.
719 	 */
720 	if (uma_zone_exhausted_nolock(zone_clust))
721 		uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN);
722 }
723 
724 /*
725  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
726  *
727  * Here the 'arg' pointer points to the Mbuf which we
728  * are configuring cluster storage for.  If 'arg' is
729  * empty we allocate just the cluster without setting
730  * the mbuf to it.  See mbuf.h.
731  */
732 static int
733 mb_ctor_clust(void *mem, int size, void *arg, int how)
734 {
735 	struct mbuf *m;
736 
737 #ifdef INVARIANTS
738 	trash_ctor(mem, size, arg, how);
739 #endif
740 	m = (struct mbuf *)arg;
741 	if (m != NULL) {
742 		m->m_ext.ext_buf = (char *)mem;
743 		m->m_data = m->m_ext.ext_buf;
744 		m->m_flags |= M_EXT;
745 		m->m_ext.ext_free = NULL;
746 		m->m_ext.ext_arg1 = NULL;
747 		m->m_ext.ext_arg2 = NULL;
748 		m->m_ext.ext_size = size;
749 		m->m_ext.ext_type = m_gettype(size);
750 		m->m_ext.ext_flags = EXT_FLAG_EMBREF;
751 		m->m_ext.ext_count = 1;
752 	}
753 
754 	return (0);
755 }
756 
757 /*
758  * The Packet secondary zone's init routine, executed on the
759  * object's transition from mbuf keg slab to zone cache.
760  */
761 static int
762 mb_zinit_pack(void *mem, int size, int how)
763 {
764 	struct mbuf *m;
765 
766 	m = (struct mbuf *)mem;		/* m is virgin. */
767 	if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
768 	    m->m_ext.ext_buf == NULL)
769 		return (ENOMEM);
770 	m->m_ext.ext_type = EXT_PACKET;	/* Override. */
771 #ifdef INVARIANTS
772 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
773 #endif
774 	return (0);
775 }
776 
777 /*
778  * The Packet secondary zone's fini routine, executed on the
779  * object's transition from zone cache to keg slab.
780  */
781 static void
782 mb_zfini_pack(void *mem, int size)
783 {
784 	struct mbuf *m;
785 
786 	m = (struct mbuf *)mem;
787 #ifdef INVARIANTS
788 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
789 #endif
790 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
791 #ifdef INVARIANTS
792 	trash_dtor(mem, size, NULL);
793 #endif
794 }
795 
796 /*
797  * The "packet" keg constructor.
798  */
799 static int
800 mb_ctor_pack(void *mem, int size, void *arg, int how)
801 {
802 	struct mbuf *m;
803 	struct mb_args *args;
804 	int error, flags;
805 	short type;
806 
807 	m = (struct mbuf *)mem;
808 	args = (struct mb_args *)arg;
809 	flags = args->flags;
810 	type = args->type;
811 	MPASS((flags & M_NOFREE) == 0);
812 
813 #ifdef INVARIANTS
814 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
815 #endif
816 
817 	error = m_init(m, how, type, flags);
818 
819 	/* m_ext is already initialized. */
820 	m->m_data = m->m_ext.ext_buf;
821  	m->m_flags = (flags | M_EXT);
822 
823 	return (error);
824 }
825 
826 /*
827  * This is the protocol drain routine.  Called by UMA whenever any of the
828  * mbuf zones is closed to its limit.
829  *
830  * No locks should be held when this is called.  The drain routines have to
831  * presently acquire some locks which raises the possibility of lock order
832  * reversal.
833  */
834 static void
835 mb_reclaim(uma_zone_t zone __unused, int pending __unused)
836 {
837 	struct domain *dp;
838 	struct protosw *pr;
839 
840 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
841 
842 	for (dp = domains; dp != NULL; dp = dp->dom_next)
843 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
844 			if (pr->pr_drain != NULL)
845 				(*pr->pr_drain)();
846 }
847 
848 /*
849  * Free "count" units of I/O from an mbuf chain.  They could be held
850  * in EXT_PGS or just as a normal mbuf.  This code is intended to be
851  * called in an error path (I/O error, closed connection, etc).
852  */
853 void
854 mb_free_notready(struct mbuf *m, int count)
855 {
856 	int i;
857 
858 	for (i = 0; i < count && m != NULL; i++) {
859 		if ((m->m_flags & M_EXT) != 0 &&
860 		    m->m_ext.ext_type == EXT_PGS) {
861 			m->m_ext.ext_pgs->nrdy--;
862 			if (m->m_ext.ext_pgs->nrdy != 0)
863 				continue;
864 		}
865 		m = m_free(m);
866 	}
867 	KASSERT(i == count, ("Removed only %d items from %p", i, m));
868 }
869 
870 /*
871  * Compress an unmapped mbuf into a simple mbuf when it holds a small
872  * amount of data.  This is used as a DOS defense to avoid having
873  * small packets tie up wired pages, an ext_pgs structure, and an
874  * mbuf.  Since this converts the existing mbuf in place, it can only
875  * be used if there are no other references to 'm'.
876  */
877 int
878 mb_unmapped_compress(struct mbuf *m)
879 {
880 	volatile u_int *refcnt;
881 	struct mbuf m_temp;
882 
883 	/*
884 	 * Assert that 'm' does not have a packet header.  If 'm' had
885 	 * a packet header, it would only be able to hold MHLEN bytes
886 	 * and m_data would have to be initialized differently.
887 	 */
888 	KASSERT((m->m_flags & M_PKTHDR) == 0 && (m->m_flags & M_EXT) &&
889 	    m->m_ext.ext_type == EXT_PGS,
890             ("%s: m %p !M_EXT or !EXT_PGS or M_PKTHDR", __func__, m));
891 	KASSERT(m->m_len <= MLEN, ("m_len too large %p", m));
892 
893 	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
894 		refcnt = &m->m_ext.ext_count;
895 	} else {
896 		KASSERT(m->m_ext.ext_cnt != NULL,
897 		    ("%s: no refcounting pointer on %p", __func__, m));
898 		refcnt = m->m_ext.ext_cnt;
899 	}
900 
901 	if (*refcnt != 1)
902 		return (EBUSY);
903 
904 	/*
905 	 * Copy mbuf header and m_ext portion of 'm' to 'm_temp' to
906 	 * create a "fake" EXT_PGS mbuf that can be used with
907 	 * m_copydata() as well as the ext_free callback.
908 	 */
909 	memcpy(&m_temp, m, offsetof(struct mbuf, m_ext) + sizeof (m->m_ext));
910 	m_temp.m_next = NULL;
911 	m_temp.m_nextpkt = NULL;
912 
913 	/* Turn 'm' into a "normal" mbuf. */
914 	m->m_flags &= ~(M_EXT | M_RDONLY | M_NOMAP);
915 	m->m_data = m->m_dat;
916 
917 	/* Copy data from template's ext_pgs. */
918 	m_copydata(&m_temp, 0, m_temp.m_len, mtod(m, caddr_t));
919 
920 	/* Free the backing pages. */
921 	m_temp.m_ext.ext_free(&m_temp);
922 
923 	/* Finally, free the ext_pgs struct. */
924 	uma_zfree(zone_extpgs, m_temp.m_ext.ext_pgs);
925 	return (0);
926 }
927 
928 /*
929  * These next few routines are used to permit downgrading an unmapped
930  * mbuf to a chain of mapped mbufs.  This is used when an interface
931  * doesn't supported unmapped mbufs or if checksums need to be
932  * computed in software.
933  *
934  * Each unmapped mbuf is converted to a chain of mbufs.  First, any
935  * TLS header data is stored in a regular mbuf.  Second, each page of
936  * unmapped data is stored in an mbuf with an EXT_SFBUF external
937  * cluster.  These mbufs use an sf_buf to provide a valid KVA for the
938  * associated physical page.  They also hold a reference on the
939  * original EXT_PGS mbuf to ensure the physical page doesn't go away.
940  * Finally, any TLS trailer data is stored in a regular mbuf.
941  *
942  * mb_unmapped_free_mext() is the ext_free handler for the EXT_SFBUF
943  * mbufs.  It frees the associated sf_buf and releases its reference
944  * on the original EXT_PGS mbuf.
945  *
946  * _mb_unmapped_to_ext() is a helper function that converts a single
947  * unmapped mbuf into a chain of mbufs.
948  *
949  * mb_unmapped_to_ext() is the public function that walks an mbuf
950  * chain converting any unmapped mbufs to mapped mbufs.  It returns
951  * the new chain of unmapped mbufs on success.  On failure it frees
952  * the original mbuf chain and returns NULL.
953  */
954 static void
955 mb_unmapped_free_mext(struct mbuf *m)
956 {
957 	struct sf_buf *sf;
958 	struct mbuf *old_m;
959 
960 	sf = m->m_ext.ext_arg1;
961 	sf_buf_free(sf);
962 
963 	/* Drop the reference on the backing EXT_PGS mbuf. */
964 	old_m = m->m_ext.ext_arg2;
965 	mb_free_ext(old_m);
966 }
967 
968 static struct mbuf *
969 _mb_unmapped_to_ext(struct mbuf *m)
970 {
971 	struct mbuf_ext_pgs *ext_pgs;
972 	struct mbuf *m_new, *top, *prev, *mref;
973 	struct sf_buf *sf;
974 	vm_page_t pg;
975 	int i, len, off, pglen, pgoff, seglen, segoff;
976 	volatile u_int *refcnt;
977 	u_int ref_inc = 0;
978 
979 	MBUF_EXT_PGS_ASSERT(m);
980 	ext_pgs = m->m_ext.ext_pgs;
981 	len = m->m_len;
982 	KASSERT(ext_pgs->tls == NULL, ("%s: can't convert TLS mbuf %p",
983 	    __func__, m));
984 
985 	/* See if this is the mbuf that holds the embedded refcount. */
986 	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
987 		refcnt = &m->m_ext.ext_count;
988 		mref = m;
989 	} else {
990 		KASSERT(m->m_ext.ext_cnt != NULL,
991 		    ("%s: no refcounting pointer on %p", __func__, m));
992 		refcnt = m->m_ext.ext_cnt;
993 		mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
994 	}
995 
996 	/* Skip over any data removed from the front. */
997 	off = mtod(m, vm_offset_t);
998 
999 	top = NULL;
1000 	if (ext_pgs->hdr_len != 0) {
1001 		if (off >= ext_pgs->hdr_len) {
1002 			off -= ext_pgs->hdr_len;
1003 		} else {
1004 			seglen = ext_pgs->hdr_len - off;
1005 			segoff = off;
1006 			seglen = min(seglen, len);
1007 			off = 0;
1008 			len -= seglen;
1009 			m_new = m_get(M_NOWAIT, MT_DATA);
1010 			if (m_new == NULL)
1011 				goto fail;
1012 			m_new->m_len = seglen;
1013 			prev = top = m_new;
1014 			memcpy(mtod(m_new, void *), &ext_pgs->hdr[segoff],
1015 			    seglen);
1016 		}
1017 	}
1018 	pgoff = ext_pgs->first_pg_off;
1019 	for (i = 0; i < ext_pgs->npgs && len > 0; i++) {
1020 		pglen = mbuf_ext_pg_len(ext_pgs, i, pgoff);
1021 		if (off >= pglen) {
1022 			off -= pglen;
1023 			pgoff = 0;
1024 			continue;
1025 		}
1026 		seglen = pglen - off;
1027 		segoff = pgoff + off;
1028 		off = 0;
1029 		seglen = min(seglen, len);
1030 		len -= seglen;
1031 
1032 		pg = PHYS_TO_VM_PAGE(ext_pgs->pa[i]);
1033 		m_new = m_get(M_NOWAIT, MT_DATA);
1034 		if (m_new == NULL)
1035 			goto fail;
1036 		if (top == NULL) {
1037 			top = prev = m_new;
1038 		} else {
1039 			prev->m_next = m_new;
1040 			prev = m_new;
1041 		}
1042 		sf = sf_buf_alloc(pg, SFB_NOWAIT);
1043 		if (sf == NULL)
1044 			goto fail;
1045 
1046 		ref_inc++;
1047 		m_extadd(m_new, (char *)sf_buf_kva(sf), PAGE_SIZE,
1048 		    mb_unmapped_free_mext, sf, mref, M_RDONLY, EXT_SFBUF);
1049 		m_new->m_data += segoff;
1050 		m_new->m_len = seglen;
1051 
1052 		pgoff = 0;
1053 	};
1054 	if (len != 0) {
1055 		KASSERT((off + len) <= ext_pgs->trail_len,
1056 		    ("off + len > trail (%d + %d > %d)", off, len,
1057 		    ext_pgs->trail_len));
1058 		m_new = m_get(M_NOWAIT, MT_DATA);
1059 		if (m_new == NULL)
1060 			goto fail;
1061 		if (top == NULL)
1062 			top = m_new;
1063 		else
1064 			prev->m_next = m_new;
1065 		m_new->m_len = len;
1066 		memcpy(mtod(m_new, void *), &ext_pgs->trail[off], len);
1067 	}
1068 
1069 	if (ref_inc != 0) {
1070 		/*
1071 		 * Obtain an additional reference on the old mbuf for
1072 		 * each created EXT_SFBUF mbuf.  They will be dropped
1073 		 * in mb_unmapped_free_mext().
1074 		 */
1075 		if (*refcnt == 1)
1076 			*refcnt += ref_inc;
1077 		else
1078 			atomic_add_int(refcnt, ref_inc);
1079 	}
1080 	m_free(m);
1081 	return (top);
1082 
1083 fail:
1084 	if (ref_inc != 0) {
1085 		/*
1086 		 * Obtain an additional reference on the old mbuf for
1087 		 * each created EXT_SFBUF mbuf.  They will be
1088 		 * immediately dropped when these mbufs are freed
1089 		 * below.
1090 		 */
1091 		if (*refcnt == 1)
1092 			*refcnt += ref_inc;
1093 		else
1094 			atomic_add_int(refcnt, ref_inc);
1095 	}
1096 	m_free(m);
1097 	m_freem(top);
1098 	return (NULL);
1099 }
1100 
1101 struct mbuf *
1102 mb_unmapped_to_ext(struct mbuf *top)
1103 {
1104 	struct mbuf *m, *next, *prev = NULL;
1105 
1106 	prev = NULL;
1107 	for (m = top; m != NULL; m = next) {
1108 		/* m might be freed, so cache the next pointer. */
1109 		next = m->m_next;
1110 		if (m->m_flags & M_NOMAP) {
1111 			if (prev != NULL) {
1112 				/*
1113 				 * Remove 'm' from the new chain so
1114 				 * that the 'top' chain terminates
1115 				 * before 'm' in case 'top' is freed
1116 				 * due to an error.
1117 				 */
1118 				prev->m_next = NULL;
1119 			}
1120 			m = _mb_unmapped_to_ext(m);
1121 			if (m == NULL) {
1122 				m_freem(top);
1123 				m_freem(next);
1124 				return (NULL);
1125 			}
1126 			if (prev == NULL) {
1127 				top = m;
1128 			} else {
1129 				prev->m_next = m;
1130 			}
1131 
1132 			/*
1133 			 * Replaced one mbuf with a chain, so we must
1134 			 * find the end of chain.
1135 			 */
1136 			prev = m_last(m);
1137 		} else {
1138 			if (prev != NULL) {
1139 				prev->m_next = m;
1140 			}
1141 			prev = m;
1142 		}
1143 	}
1144 	return (top);
1145 }
1146 
1147 /*
1148  * Allocate an empty EXT_PGS mbuf.  The ext_free routine is
1149  * responsible for freeing any pages backing this mbuf when it is
1150  * freed.
1151  */
1152 struct mbuf *
1153 mb_alloc_ext_pgs(int how, bool pkthdr, m_ext_free_t ext_free)
1154 {
1155 	struct mbuf *m;
1156 	struct mbuf_ext_pgs *ext_pgs;
1157 
1158 	if (pkthdr)
1159 		m = m_gethdr(how, MT_DATA);
1160 	else
1161 		m = m_get(how, MT_DATA);
1162 	if (m == NULL)
1163 		return (NULL);
1164 
1165 	ext_pgs = uma_zalloc(zone_extpgs, how);
1166 	if (ext_pgs == NULL) {
1167 		m_free(m);
1168 		return (NULL);
1169 	}
1170 	ext_pgs->npgs = 0;
1171 	ext_pgs->nrdy = 0;
1172 	ext_pgs->first_pg_off = 0;
1173 	ext_pgs->last_pg_len = 0;
1174 	ext_pgs->hdr_len = 0;
1175 	ext_pgs->trail_len = 0;
1176 	ext_pgs->tls = NULL;
1177 	ext_pgs->so = NULL;
1178 	m->m_data = NULL;
1179 	m->m_flags |= (M_EXT | M_RDONLY | M_NOMAP);
1180 	m->m_ext.ext_type = EXT_PGS;
1181 	m->m_ext.ext_flags = EXT_FLAG_EMBREF;
1182 	m->m_ext.ext_count = 1;
1183 	m->m_ext.ext_pgs = ext_pgs;
1184 	m->m_ext.ext_size = 0;
1185 	m->m_ext.ext_free = ext_free;
1186 	return (m);
1187 }
1188 
1189 #ifdef INVARIANT_SUPPORT
1190 void
1191 mb_ext_pgs_check(struct mbuf_ext_pgs *ext_pgs)
1192 {
1193 
1194 	/*
1195 	 * NB: This expects a non-empty buffer (npgs > 0 and
1196 	 * last_pg_len > 0).
1197 	 */
1198 	KASSERT(ext_pgs->npgs > 0,
1199 	    ("ext_pgs with no valid pages: %p", ext_pgs));
1200 	KASSERT(ext_pgs->npgs <= nitems(ext_pgs->pa),
1201 	    ("ext_pgs with too many pages: %p", ext_pgs));
1202 	KASSERT(ext_pgs->nrdy <= ext_pgs->npgs,
1203 	    ("ext_pgs with too many ready pages: %p", ext_pgs));
1204 	KASSERT(ext_pgs->first_pg_off < PAGE_SIZE,
1205 	    ("ext_pgs with too large page offset: %p", ext_pgs));
1206 	KASSERT(ext_pgs->last_pg_len > 0,
1207 	    ("ext_pgs with zero last page length: %p", ext_pgs));
1208 	KASSERT(ext_pgs->last_pg_len <= PAGE_SIZE,
1209 	    ("ext_pgs with too large last page length: %p", ext_pgs));
1210 	if (ext_pgs->npgs == 1) {
1211 		KASSERT(ext_pgs->first_pg_off + ext_pgs->last_pg_len <=
1212 		    PAGE_SIZE, ("ext_pgs with single page too large: %p",
1213 		    ext_pgs));
1214 	}
1215 	KASSERT(ext_pgs->hdr_len <= sizeof(ext_pgs->hdr),
1216 	    ("ext_pgs with too large header length: %p", ext_pgs));
1217 	KASSERT(ext_pgs->trail_len <= sizeof(ext_pgs->trail),
1218 	    ("ext_pgs with too large header length: %p", ext_pgs));
1219 }
1220 #endif
1221 
1222 /*
1223  * Clean up after mbufs with M_EXT storage attached to them if the
1224  * reference count hits 1.
1225  */
1226 void
1227 mb_free_ext(struct mbuf *m)
1228 {
1229 	volatile u_int *refcnt;
1230 	struct mbuf *mref;
1231 	int freembuf;
1232 
1233 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
1234 
1235 	/* See if this is the mbuf that holds the embedded refcount. */
1236 	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
1237 		refcnt = &m->m_ext.ext_count;
1238 		mref = m;
1239 	} else {
1240 		KASSERT(m->m_ext.ext_cnt != NULL,
1241 		    ("%s: no refcounting pointer on %p", __func__, m));
1242 		refcnt = m->m_ext.ext_cnt;
1243 		mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
1244 	}
1245 
1246 	/*
1247 	 * Check if the header is embedded in the cluster.  It is
1248 	 * important that we can't touch any of the mbuf fields
1249 	 * after we have freed the external storage, since mbuf
1250 	 * could have been embedded in it.  For now, the mbufs
1251 	 * embedded into the cluster are always of type EXT_EXTREF,
1252 	 * and for this type we won't free the mref.
1253 	 */
1254 	if (m->m_flags & M_NOFREE) {
1255 		freembuf = 0;
1256 		KASSERT(m->m_ext.ext_type == EXT_EXTREF ||
1257 		    m->m_ext.ext_type == EXT_RXRING,
1258 		    ("%s: no-free mbuf %p has wrong type", __func__, m));
1259 	} else
1260 		freembuf = 1;
1261 
1262 	/* Free attached storage if this mbuf is the only reference to it. */
1263 	if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) {
1264 		switch (m->m_ext.ext_type) {
1265 		case EXT_PACKET:
1266 			/* The packet zone is special. */
1267 			if (*refcnt == 0)
1268 				*refcnt = 1;
1269 			uma_zfree(zone_pack, mref);
1270 			break;
1271 		case EXT_CLUSTER:
1272 			uma_zfree(zone_clust, m->m_ext.ext_buf);
1273 			uma_zfree(zone_mbuf, mref);
1274 			break;
1275 		case EXT_JUMBOP:
1276 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
1277 			uma_zfree(zone_mbuf, mref);
1278 			break;
1279 		case EXT_JUMBO9:
1280 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
1281 			uma_zfree(zone_mbuf, mref);
1282 			break;
1283 		case EXT_JUMBO16:
1284 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
1285 			uma_zfree(zone_mbuf, mref);
1286 			break;
1287 		case EXT_PGS: {
1288 #ifdef KERN_TLS
1289 			struct mbuf_ext_pgs *pgs;
1290 			struct ktls_session *tls;
1291 #endif
1292 
1293 			KASSERT(mref->m_ext.ext_free != NULL,
1294 			    ("%s: ext_free not set", __func__));
1295 			mref->m_ext.ext_free(mref);
1296 #ifdef KERN_TLS
1297 			pgs = mref->m_ext.ext_pgs;
1298 			tls = pgs->tls;
1299 			if (tls != NULL &&
1300 			    !refcount_release_if_not_last(&tls->refcount))
1301 				ktls_enqueue_to_free(pgs);
1302 			else
1303 #endif
1304 				uma_zfree(zone_extpgs, mref->m_ext.ext_pgs);
1305 			uma_zfree(zone_mbuf, mref);
1306 			break;
1307 		}
1308 		case EXT_SFBUF:
1309 		case EXT_NET_DRV:
1310 		case EXT_MOD_TYPE:
1311 		case EXT_DISPOSABLE:
1312 			KASSERT(mref->m_ext.ext_free != NULL,
1313 			    ("%s: ext_free not set", __func__));
1314 			mref->m_ext.ext_free(mref);
1315 			uma_zfree(zone_mbuf, mref);
1316 			break;
1317 		case EXT_EXTREF:
1318 			KASSERT(m->m_ext.ext_free != NULL,
1319 			    ("%s: ext_free not set", __func__));
1320 			m->m_ext.ext_free(m);
1321 			break;
1322 		case EXT_RXRING:
1323 			KASSERT(m->m_ext.ext_free == NULL,
1324 			    ("%s: ext_free is set", __func__));
1325 			break;
1326 		default:
1327 			KASSERT(m->m_ext.ext_type == 0,
1328 			    ("%s: unknown ext_type", __func__));
1329 		}
1330 	}
1331 
1332 	if (freembuf && m != mref)
1333 		uma_zfree(zone_mbuf, m);
1334 }
1335 
1336 /*
1337  * Official mbuf(9) allocation KPI for stack and drivers:
1338  *
1339  * m_get()	- a single mbuf without any attachments, sys/mbuf.h.
1340  * m_gethdr()	- a single mbuf initialized as M_PKTHDR, sys/mbuf.h.
1341  * m_getcl()	- an mbuf + 2k cluster, sys/mbuf.h.
1342  * m_clget()	- attach cluster to already allocated mbuf.
1343  * m_cljget()	- attach jumbo cluster to already allocated mbuf.
1344  * m_get2()	- allocate minimum mbuf that would fit size argument.
1345  * m_getm2()	- allocate a chain of mbufs/clusters.
1346  * m_extadd()	- attach external cluster to mbuf.
1347  *
1348  * m_free()	- free single mbuf with its tags and ext, sys/mbuf.h.
1349  * m_freem()	- free chain of mbufs.
1350  */
1351 
1352 int
1353 m_clget(struct mbuf *m, int how)
1354 {
1355 
1356 	KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
1357 	    __func__, m));
1358 	m->m_ext.ext_buf = (char *)NULL;
1359 	uma_zalloc_arg(zone_clust, m, how);
1360 	/*
1361 	 * On a cluster allocation failure, drain the packet zone and retry,
1362 	 * we might be able to loosen a few clusters up on the drain.
1363 	 */
1364 	if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
1365 		uma_zone_reclaim(zone_pack, UMA_RECLAIM_DRAIN);
1366 		uma_zalloc_arg(zone_clust, m, how);
1367 	}
1368 	MBUF_PROBE2(m__clget, m, how);
1369 	return (m->m_flags & M_EXT);
1370 }
1371 
1372 /*
1373  * m_cljget() is different from m_clget() as it can allocate clusters without
1374  * attaching them to an mbuf.  In that case the return value is the pointer
1375  * to the cluster of the requested size.  If an mbuf was specified, it gets
1376  * the cluster attached to it and the return value can be safely ignored.
1377  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
1378  */
1379 void *
1380 m_cljget(struct mbuf *m, int how, int size)
1381 {
1382 	uma_zone_t zone;
1383 	void *retval;
1384 
1385 	if (m != NULL) {
1386 		KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
1387 		    __func__, m));
1388 		m->m_ext.ext_buf = NULL;
1389 	}
1390 
1391 	zone = m_getzone(size);
1392 	retval = uma_zalloc_arg(zone, m, how);
1393 
1394 	MBUF_PROBE4(m__cljget, m, how, size, retval);
1395 
1396 	return (retval);
1397 }
1398 
1399 /*
1400  * m_get2() allocates minimum mbuf that would fit "size" argument.
1401  */
1402 struct mbuf *
1403 m_get2(int size, int how, short type, int flags)
1404 {
1405 	struct mb_args args;
1406 	struct mbuf *m, *n;
1407 
1408 	args.flags = flags;
1409 	args.type = type;
1410 
1411 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
1412 		return (uma_zalloc_arg(zone_mbuf, &args, how));
1413 	if (size <= MCLBYTES)
1414 		return (uma_zalloc_arg(zone_pack, &args, how));
1415 
1416 	if (size > MJUMPAGESIZE)
1417 		return (NULL);
1418 
1419 	m = uma_zalloc_arg(zone_mbuf, &args, how);
1420 	if (m == NULL)
1421 		return (NULL);
1422 
1423 	n = uma_zalloc_arg(zone_jumbop, m, how);
1424 	if (n == NULL) {
1425 		uma_zfree(zone_mbuf, m);
1426 		return (NULL);
1427 	}
1428 
1429 	return (m);
1430 }
1431 
1432 /*
1433  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
1434  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
1435  */
1436 struct mbuf *
1437 m_getjcl(int how, short type, int flags, int size)
1438 {
1439 	struct mb_args args;
1440 	struct mbuf *m, *n;
1441 	uma_zone_t zone;
1442 
1443 	if (size == MCLBYTES)
1444 		return m_getcl(how, type, flags);
1445 
1446 	args.flags = flags;
1447 	args.type = type;
1448 
1449 	m = uma_zalloc_arg(zone_mbuf, &args, how);
1450 	if (m == NULL)
1451 		return (NULL);
1452 
1453 	zone = m_getzone(size);
1454 	n = uma_zalloc_arg(zone, m, how);
1455 	if (n == NULL) {
1456 		uma_zfree(zone_mbuf, m);
1457 		return (NULL);
1458 	}
1459 	return (m);
1460 }
1461 
1462 /*
1463  * Allocate a given length worth of mbufs and/or clusters (whatever fits
1464  * best) and return a pointer to the top of the allocated chain.  If an
1465  * existing mbuf chain is provided, then we will append the new chain
1466  * to the existing one and return a pointer to the provided mbuf.
1467  */
1468 struct mbuf *
1469 m_getm2(struct mbuf *m, int len, int how, short type, int flags)
1470 {
1471 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
1472 
1473 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
1474 
1475 	/* Validate flags. */
1476 	flags &= (M_PKTHDR | M_EOR);
1477 
1478 	/* Packet header mbuf must be first in chain. */
1479 	if ((flags & M_PKTHDR) && m != NULL)
1480 		flags &= ~M_PKTHDR;
1481 
1482 	/* Loop and append maximum sized mbufs to the chain tail. */
1483 	while (len > 0) {
1484 		if (len > MCLBYTES)
1485 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
1486 			    MJUMPAGESIZE);
1487 		else if (len >= MINCLSIZE)
1488 			mb = m_getcl(how, type, (flags & M_PKTHDR));
1489 		else if (flags & M_PKTHDR)
1490 			mb = m_gethdr(how, type);
1491 		else
1492 			mb = m_get(how, type);
1493 
1494 		/* Fail the whole operation if one mbuf can't be allocated. */
1495 		if (mb == NULL) {
1496 			if (nm != NULL)
1497 				m_freem(nm);
1498 			return (NULL);
1499 		}
1500 
1501 		/* Book keeping. */
1502 		len -= M_SIZE(mb);
1503 		if (mtail != NULL)
1504 			mtail->m_next = mb;
1505 		else
1506 			nm = mb;
1507 		mtail = mb;
1508 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
1509 	}
1510 	if (flags & M_EOR)
1511 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
1512 
1513 	/* If mbuf was supplied, append new chain to the end of it. */
1514 	if (m != NULL) {
1515 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
1516 			;
1517 		mtail->m_next = nm;
1518 		mtail->m_flags &= ~M_EOR;
1519 	} else
1520 		m = nm;
1521 
1522 	return (m);
1523 }
1524 
1525 /*-
1526  * Configure a provided mbuf to refer to the provided external storage
1527  * buffer and setup a reference count for said buffer.
1528  *
1529  * Arguments:
1530  *    mb     The existing mbuf to which to attach the provided buffer.
1531  *    buf    The address of the provided external storage buffer.
1532  *    size   The size of the provided buffer.
1533  *    freef  A pointer to a routine that is responsible for freeing the
1534  *           provided external storage buffer.
1535  *    args   A pointer to an argument structure (of any type) to be passed
1536  *           to the provided freef routine (may be NULL).
1537  *    flags  Any other flags to be passed to the provided mbuf.
1538  *    type   The type that the external storage buffer should be
1539  *           labeled with.
1540  *
1541  * Returns:
1542  *    Nothing.
1543  */
1544 void
1545 m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef,
1546     void *arg1, void *arg2, int flags, int type)
1547 {
1548 
1549 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
1550 
1551 	mb->m_flags |= (M_EXT | flags);
1552 	mb->m_ext.ext_buf = buf;
1553 	mb->m_data = mb->m_ext.ext_buf;
1554 	mb->m_ext.ext_size = size;
1555 	mb->m_ext.ext_free = freef;
1556 	mb->m_ext.ext_arg1 = arg1;
1557 	mb->m_ext.ext_arg2 = arg2;
1558 	mb->m_ext.ext_type = type;
1559 
1560 	if (type != EXT_EXTREF) {
1561 		mb->m_ext.ext_count = 1;
1562 		mb->m_ext.ext_flags = EXT_FLAG_EMBREF;
1563 	} else
1564 		mb->m_ext.ext_flags = 0;
1565 }
1566 
1567 /*
1568  * Free an entire chain of mbufs and associated external buffers, if
1569  * applicable.
1570  */
1571 void
1572 m_freem(struct mbuf *mb)
1573 {
1574 
1575 	MBUF_PROBE1(m__freem, mb);
1576 	while (mb != NULL)
1577 		mb = m_free(mb);
1578 }
1579 
1580 void
1581 m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp)
1582 {
1583 
1584 	if_ref(ifp);
1585 	mst->ifp = ifp;
1586 	refcount_init(&mst->refcount, 1);
1587 	counter_u64_add(snd_tag_count, 1);
1588 }
1589 
1590 void
1591 m_snd_tag_destroy(struct m_snd_tag *mst)
1592 {
1593 	struct ifnet *ifp;
1594 
1595 	ifp = mst->ifp;
1596 	ifp->if_snd_tag_free(mst);
1597 	if_rele(ifp);
1598 	counter_u64_add(snd_tag_count, -1);
1599 }
1600