xref: /freebsd/sys/kern/kern_mbuf.c (revision fb3bc59600e25268ed5750fdc351b1e9423a8fb4)
1099a0e58SBosko Milekic /*-
28a36da99SPedro F. Giffuni  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
38a36da99SPedro F. Giffuni  *
48076cb52SBosko Milekic  * Copyright (c) 2004, 2005,
58076cb52SBosko Milekic  *	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
6099a0e58SBosko Milekic  *
7099a0e58SBosko Milekic  * Redistribution and use in source and binary forms, with or without
8099a0e58SBosko Milekic  * modification, are permitted provided that the following conditions
9099a0e58SBosko Milekic  * are met:
10099a0e58SBosko Milekic  * 1. Redistributions of source code must retain the above copyright
11099a0e58SBosko Milekic  *    notice unmodified, this list of conditions and the following
12099a0e58SBosko Milekic  *    disclaimer.
13099a0e58SBosko Milekic  * 2. Redistributions in binary form must reproduce the above copyright
14099a0e58SBosko Milekic  *    notice, this list of conditions and the following disclaimer in the
15099a0e58SBosko Milekic  *    documentation and/or other materials provided with the distribution.
16099a0e58SBosko Milekic  *
17099a0e58SBosko Milekic  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18099a0e58SBosko Milekic  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19099a0e58SBosko Milekic  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20099a0e58SBosko Milekic  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21099a0e58SBosko Milekic  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22099a0e58SBosko Milekic  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23099a0e58SBosko Milekic  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24099a0e58SBosko Milekic  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25099a0e58SBosko Milekic  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26099a0e58SBosko Milekic  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27099a0e58SBosko Milekic  * SUCH DAMAGE.
28099a0e58SBosko Milekic  */
29099a0e58SBosko Milekic 
30099a0e58SBosko Milekic #include <sys/cdefs.h>
31099a0e58SBosko Milekic __FBSDID("$FreeBSD$");
32099a0e58SBosko Milekic 
33099a0e58SBosko Milekic #include "opt_param.h"
34099a0e58SBosko Milekic 
35099a0e58SBosko Milekic #include <sys/param.h>
363937ee75SConrad Meyer #include <sys/conf.h>
379978bd99SMark Johnston #include <sys/domainset.h>
38099a0e58SBosko Milekic #include <sys/malloc.h>
39099a0e58SBosko Milekic #include <sys/systm.h>
40099a0e58SBosko Milekic #include <sys/mbuf.h>
41099a0e58SBosko Milekic #include <sys/domain.h>
42099a0e58SBosko Milekic #include <sys/eventhandler.h>
43099a0e58SBosko Milekic #include <sys/kernel.h>
445475ca5aSMark Johnston #include <sys/limits.h>
4554503a13SJonathan T. Looney #include <sys/lock.h>
4654503a13SJonathan T. Looney #include <sys/mutex.h>
47099a0e58SBosko Milekic #include <sys/protosw.h>
48099a0e58SBosko Milekic #include <sys/smp.h>
49*fb3bc596SJohn Baldwin #include <sys/socket.h>
50099a0e58SBosko Milekic #include <sys/sysctl.h>
51099a0e58SBosko Milekic 
52*fb3bc596SJohn Baldwin #include <net/if.h>
53*fb3bc596SJohn Baldwin #include <net/if_var.h>
54*fb3bc596SJohn Baldwin 
55099a0e58SBosko Milekic #include <vm/vm.h>
56c45c0034SAlan Cox #include <vm/vm_extern.h>
57c45c0034SAlan Cox #include <vm/vm_kern.h>
58099a0e58SBosko Milekic #include <vm/vm_page.h>
5937140716SAndre Oppermann #include <vm/vm_map.h>
60099a0e58SBosko Milekic #include <vm/uma.h>
61121f0509SMike Silbersack #include <vm/uma_dbg.h>
62099a0e58SBosko Milekic 
63099a0e58SBosko Milekic /*
64099a0e58SBosko Milekic  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
65099a0e58SBosko Milekic  * Zones.
66099a0e58SBosko Milekic  *
67099a0e58SBosko Milekic  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
68099a0e58SBosko Milekic  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
69099a0e58SBosko Milekic  * administrator so desires.
70099a0e58SBosko Milekic  *
71099a0e58SBosko Milekic  * Mbufs are allocated from a UMA Master Zone called the Mbuf
72099a0e58SBosko Milekic  * Zone.
73099a0e58SBosko Milekic  *
74099a0e58SBosko Milekic  * Additionally, FreeBSD provides a Packet Zone, which it
75099a0e58SBosko Milekic  * configures as a Secondary Zone to the Mbuf Master Zone,
76099a0e58SBosko Milekic  * thus sharing backend Slab kegs with the Mbuf Master Zone.
77099a0e58SBosko Milekic  *
78099a0e58SBosko Milekic  * Thus common-case allocations and locking are simplified:
79099a0e58SBosko Milekic  *
80099a0e58SBosko Milekic  *  m_clget()                m_getcl()
81099a0e58SBosko Milekic  *    |                         |
82099a0e58SBosko Milekic  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
83099a0e58SBosko Milekic  *    |   |             [     Packet   ]            |
84099a0e58SBosko Milekic  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
85099a0e58SBosko Milekic  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
86099a0e58SBosko Milekic  *        |                       \________         |
87099a0e58SBosko Milekic  *  [ Cluster Keg   ]                      \       /
88099a0e58SBosko Milekic  *        |	                         [ Mbuf Keg   ]
89099a0e58SBosko Milekic  *  [ Cluster Slabs ]                         |
90099a0e58SBosko Milekic  *        |                              [ Mbuf Slabs ]
91099a0e58SBosko Milekic  *         \____________(VM)_________________/
9256a4e45aSAndre Oppermann  *
9356a4e45aSAndre Oppermann  *
94fcf90618SGleb Smirnoff  * Whenever an object is allocated with uma_zalloc() out of
9556a4e45aSAndre Oppermann  * one of the Zones its _ctor_ function is executed.  The same
96fcf90618SGleb Smirnoff  * for any deallocation through uma_zfree() the _dtor_ function
9756a4e45aSAndre Oppermann  * is executed.
9856a4e45aSAndre Oppermann  *
9956a4e45aSAndre Oppermann  * Caches are per-CPU and are filled from the Master Zone.
10056a4e45aSAndre Oppermann  *
101fcf90618SGleb Smirnoff  * Whenever an object is allocated from the underlying global
10256a4e45aSAndre Oppermann  * memory pool it gets pre-initialized with the _zinit_ functions.
103e3043798SPedro F. Giffuni  * When the Keg's are overfull objects get decommissioned with
10456a4e45aSAndre Oppermann  * _zfini_ functions and free'd back to the global memory pool.
10556a4e45aSAndre Oppermann  *
106099a0e58SBosko Milekic  */
107099a0e58SBosko Milekic 
108ead46972SAndre Oppermann int nmbufs;			/* limits number of mbufs */
10956a4e45aSAndre Oppermann int nmbclusters;		/* limits number of mbuf clusters */
110ec63cb90SAndre Oppermann int nmbjumbop;			/* limits number of page size jumbo clusters */
11156a4e45aSAndre Oppermann int nmbjumbo9;			/* limits number of 9k jumbo clusters */
11256a4e45aSAndre Oppermann int nmbjumbo16;			/* limits number of 16k jumbo clusters */
113099a0e58SBosko Milekic 
114e0c00addSAndre Oppermann static quad_t maxmbufmem;	/* overall real memory limit for all mbufs */
115e0c00addSAndre Oppermann 
116af3b2549SHans Petter Selasky SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0,
117b6f49c23SHiren Panchasara     "Maximum real memory allocatable to various mbuf types");
118e0c00addSAndre Oppermann 
119*fb3bc596SJohn Baldwin static counter_u64_t snd_tag_count;
120*fb3bc596SJohn Baldwin SYSCTL_COUNTER_U64(_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW,
121*fb3bc596SJohn Baldwin     &snd_tag_count, "# of active mbuf send tags");
122*fb3bc596SJohn Baldwin 
12362938659SBjoern A. Zeeb /*
12437140716SAndre Oppermann  * tunable_mbinit() has to be run before any mbuf allocations are done.
12562938659SBjoern A. Zeeb  */
126099a0e58SBosko Milekic static void
127099a0e58SBosko Milekic tunable_mbinit(void *dummy)
128099a0e58SBosko Milekic {
129e0c00addSAndre Oppermann 	quad_t realmem;
13037140716SAndre Oppermann 
13137140716SAndre Oppermann 	/*
13237140716SAndre Oppermann 	 * The default limit for all mbuf related memory is 1/2 of all
13337140716SAndre Oppermann 	 * available kernel memory (physical or kmem).
13437140716SAndre Oppermann 	 * At most it can be 3/4 of available kernel memory.
13537140716SAndre Oppermann 	 */
1365df87b21SJeff Roberson 	realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size);
13737140716SAndre Oppermann 	maxmbufmem = realmem / 2;
138e0c00addSAndre Oppermann 	TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
13937140716SAndre Oppermann 	if (maxmbufmem > realmem / 4 * 3)
14037140716SAndre Oppermann 		maxmbufmem = realmem / 4 * 3;
141099a0e58SBosko Milekic 
142812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
143416a434cSAndre Oppermann 	if (nmbclusters == 0)
144416a434cSAndre Oppermann 		nmbclusters = maxmbufmem / MCLBYTES / 4;
145812302c3SNavdeep Parhar 
146812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
147812302c3SNavdeep Parhar 	if (nmbjumbop == 0)
148416a434cSAndre Oppermann 		nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
149812302c3SNavdeep Parhar 
150812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
151812302c3SNavdeep Parhar 	if (nmbjumbo9 == 0)
152416a434cSAndre Oppermann 		nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
153812302c3SNavdeep Parhar 
154812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
155812302c3SNavdeep Parhar 	if (nmbjumbo16 == 0)
156416a434cSAndre Oppermann 		nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
157416a434cSAndre Oppermann 
158416a434cSAndre Oppermann 	/*
159416a434cSAndre Oppermann 	 * We need at least as many mbufs as we have clusters of
160416a434cSAndre Oppermann 	 * the various types added together.
161416a434cSAndre Oppermann 	 */
162416a434cSAndre Oppermann 	TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
163416a434cSAndre Oppermann 	if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
164416a434cSAndre Oppermann 		nmbufs = lmax(maxmbufmem / MSIZE / 5,
165416a434cSAndre Oppermann 		    nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
166099a0e58SBosko Milekic }
16737140716SAndre Oppermann SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
168099a0e58SBosko Milekic 
1694f590175SPaul Saab static int
1704f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
1714f590175SPaul Saab {
1724f590175SPaul Saab 	int error, newnmbclusters;
1734f590175SPaul Saab 
1744f590175SPaul Saab 	newnmbclusters = nmbclusters;
175041b706bSDavid Malone 	error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
176d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbclusters != nmbclusters) {
177ead46972SAndre Oppermann 		if (newnmbclusters > nmbclusters &&
178ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
1794f590175SPaul Saab 			nmbclusters = newnmbclusters;
180bc4a1b8cSAndre Oppermann 			nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
1814f590175SPaul Saab 			EVENTHANDLER_INVOKE(nmbclusters_change);
1824f590175SPaul Saab 		} else
1834f590175SPaul Saab 			error = EINVAL;
1844f590175SPaul Saab 	}
1854f590175SPaul Saab 	return (error);
1864f590175SPaul Saab }
1874f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
1884f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU",
189099a0e58SBosko Milekic     "Maximum number of mbuf clusters allowed");
190cf70a46bSRandall Stewart 
191cf70a46bSRandall Stewart static int
192cf70a46bSRandall Stewart sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
193cf70a46bSRandall Stewart {
194cf70a46bSRandall Stewart 	int error, newnmbjumbop;
195cf70a46bSRandall Stewart 
196cf70a46bSRandall Stewart 	newnmbjumbop = nmbjumbop;
197cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
198d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) {
199ead46972SAndre Oppermann 		if (newnmbjumbop > nmbjumbop &&
200ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
201cf70a46bSRandall Stewart 			nmbjumbop = newnmbjumbop;
202bc4a1b8cSAndre Oppermann 			nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
203cf70a46bSRandall Stewart 		} else
204cf70a46bSRandall Stewart 			error = EINVAL;
205cf70a46bSRandall Stewart 	}
206cf70a46bSRandall Stewart 	return (error);
207cf70a46bSRandall Stewart }
208cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
209cf70a46bSRandall Stewart &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
210ec63cb90SAndre Oppermann     "Maximum number of mbuf page size jumbo clusters allowed");
211cf70a46bSRandall Stewart 
212cf70a46bSRandall Stewart static int
213cf70a46bSRandall Stewart sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
214cf70a46bSRandall Stewart {
215cf70a46bSRandall Stewart 	int error, newnmbjumbo9;
216cf70a46bSRandall Stewart 
217cf70a46bSRandall Stewart 	newnmbjumbo9 = nmbjumbo9;
218cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
219d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) {
220ead46972SAndre Oppermann 		if (newnmbjumbo9 > nmbjumbo9 &&
221ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
222cf70a46bSRandall Stewart 			nmbjumbo9 = newnmbjumbo9;
223bc4a1b8cSAndre Oppermann 			nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
224cf70a46bSRandall Stewart 		} else
225cf70a46bSRandall Stewart 			error = EINVAL;
226cf70a46bSRandall Stewart 	}
227cf70a46bSRandall Stewart 	return (error);
228cf70a46bSRandall Stewart }
229cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
230cf70a46bSRandall Stewart &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
23156a4e45aSAndre Oppermann     "Maximum number of mbuf 9k jumbo clusters allowed");
232cf70a46bSRandall Stewart 
233cf70a46bSRandall Stewart static int
234cf70a46bSRandall Stewart sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
235cf70a46bSRandall Stewart {
236cf70a46bSRandall Stewart 	int error, newnmbjumbo16;
237cf70a46bSRandall Stewart 
238cf70a46bSRandall Stewart 	newnmbjumbo16 = nmbjumbo16;
239cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
240d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) {
241ead46972SAndre Oppermann 		if (newnmbjumbo16 > nmbjumbo16 &&
242ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
243cf70a46bSRandall Stewart 			nmbjumbo16 = newnmbjumbo16;
244bc4a1b8cSAndre Oppermann 			nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
245cf70a46bSRandall Stewart 		} else
246cf70a46bSRandall Stewart 			error = EINVAL;
247cf70a46bSRandall Stewart 	}
248cf70a46bSRandall Stewart 	return (error);
249cf70a46bSRandall Stewart }
250cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
251cf70a46bSRandall Stewart &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
25256a4e45aSAndre Oppermann     "Maximum number of mbuf 16k jumbo clusters allowed");
253cf70a46bSRandall Stewart 
254ead46972SAndre Oppermann static int
255ead46972SAndre Oppermann sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
256ead46972SAndre Oppermann {
257ead46972SAndre Oppermann 	int error, newnmbufs;
258ead46972SAndre Oppermann 
259ead46972SAndre Oppermann 	newnmbufs = nmbufs;
260ead46972SAndre Oppermann 	error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
261d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbufs != nmbufs) {
262ead46972SAndre Oppermann 		if (newnmbufs > nmbufs) {
263ead46972SAndre Oppermann 			nmbufs = newnmbufs;
264bc4a1b8cSAndre Oppermann 			nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
265ead46972SAndre Oppermann 			EVENTHANDLER_INVOKE(nmbufs_change);
266ead46972SAndre Oppermann 		} else
267ead46972SAndre Oppermann 			error = EINVAL;
268ead46972SAndre Oppermann 	}
269ead46972SAndre Oppermann 	return (error);
270ead46972SAndre Oppermann }
271e0c00addSAndre Oppermann SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW,
272ead46972SAndre Oppermann &nmbufs, 0, sysctl_nmbufs, "IU",
273ead46972SAndre Oppermann     "Maximum number of mbufs allowed");
274cf70a46bSRandall Stewart 
275099a0e58SBosko Milekic /*
276099a0e58SBosko Milekic  * Zones from which we allocate.
277099a0e58SBosko Milekic  */
278099a0e58SBosko Milekic uma_zone_t	zone_mbuf;
279099a0e58SBosko Milekic uma_zone_t	zone_clust;
280099a0e58SBosko Milekic uma_zone_t	zone_pack;
281ec63cb90SAndre Oppermann uma_zone_t	zone_jumbop;
28256a4e45aSAndre Oppermann uma_zone_t	zone_jumbo9;
28356a4e45aSAndre Oppermann uma_zone_t	zone_jumbo16;
284099a0e58SBosko Milekic 
285099a0e58SBosko Milekic /*
286099a0e58SBosko Milekic  * Local prototypes.
287099a0e58SBosko Milekic  */
288b23f72e9SBrian Feldman static int	mb_ctor_mbuf(void *, int, void *, int);
289b23f72e9SBrian Feldman static int	mb_ctor_clust(void *, int, void *, int);
290b23f72e9SBrian Feldman static int	mb_ctor_pack(void *, int, void *, int);
291099a0e58SBosko Milekic static void	mb_dtor_mbuf(void *, int, void *);
29256a4e45aSAndre Oppermann static void	mb_dtor_pack(void *, int, void *);
29356a4e45aSAndre Oppermann static int	mb_zinit_pack(void *, int, int);
29456a4e45aSAndre Oppermann static void	mb_zfini_pack(void *, int);
295e60b2fcbSGleb Smirnoff static void	mb_reclaim(uma_zone_t, int);
296ab3185d1SJeff Roberson static void    *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int);
297099a0e58SBosko Milekic 
29837140716SAndre Oppermann /* Ensure that MSIZE is a power of 2. */
299a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
300a04946cfSBrian Somers 
301099a0e58SBosko Milekic /*
302099a0e58SBosko Milekic  * Initialize FreeBSD Network buffer allocation.
303099a0e58SBosko Milekic  */
304099a0e58SBosko Milekic static void
305099a0e58SBosko Milekic mbuf_init(void *dummy)
306099a0e58SBosko Milekic {
307099a0e58SBosko Milekic 
308099a0e58SBosko Milekic 	/*
309099a0e58SBosko Milekic 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
310099a0e58SBosko Milekic 	 */
31156a4e45aSAndre Oppermann 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
31256a4e45aSAndre Oppermann 	    mb_ctor_mbuf, mb_dtor_mbuf,
313121f0509SMike Silbersack #ifdef INVARIANTS
31456a4e45aSAndre Oppermann 	    trash_init, trash_fini,
315121f0509SMike Silbersack #else
31656a4e45aSAndre Oppermann 	    NULL, NULL,
317121f0509SMike Silbersack #endif
31856a4e45aSAndre Oppermann 	    MSIZE - 1, UMA_ZONE_MAXBUCKET);
31945fe0bf7SPawel Jakub Dawidek 	if (nmbufs > 0)
32045fe0bf7SPawel Jakub Dawidek 		nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
3216e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
322e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
32356a4e45aSAndre Oppermann 
32468352adfSRobert Watson 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
32556a5f52eSGleb Smirnoff 	    mb_ctor_clust,
326121f0509SMike Silbersack #ifdef INVARIANTS
32756a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
328121f0509SMike Silbersack #else
32956a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
330121f0509SMike Silbersack #endif
33156a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
33245fe0bf7SPawel Jakub Dawidek 	if (nmbclusters > 0)
33345fe0bf7SPawel Jakub Dawidek 		nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
3346e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
335e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_clust, mb_reclaim);
336099a0e58SBosko Milekic 
33756a4e45aSAndre Oppermann 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
33856a4e45aSAndre Oppermann 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
33956a4e45aSAndre Oppermann 
340fcf90618SGleb Smirnoff 	/* Make jumbo frame zone too. Page size, 9k and 16k. */
341ec63cb90SAndre Oppermann 	zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
34256a5f52eSGleb Smirnoff 	    mb_ctor_clust,
343d5269a63SAndre Oppermann #ifdef INVARIANTS
34456a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
345d5269a63SAndre Oppermann #else
34656a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
347d5269a63SAndre Oppermann #endif
34856a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
34945fe0bf7SPawel Jakub Dawidek 	if (nmbjumbop > 0)
35045fe0bf7SPawel Jakub Dawidek 		nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
3516e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
352e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
353d5269a63SAndre Oppermann 
35456a4e45aSAndre Oppermann 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
35556a5f52eSGleb Smirnoff 	    mb_ctor_clust,
35656a4e45aSAndre Oppermann #ifdef INVARIANTS
35756a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
35856a4e45aSAndre Oppermann #else
35956a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
36056a4e45aSAndre Oppermann #endif
36156a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
362ba63339aSAlan Cox 	uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
36345fe0bf7SPawel Jakub Dawidek 	if (nmbjumbo9 > 0)
36445fe0bf7SPawel Jakub Dawidek 		nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
3656e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
366e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
36756a4e45aSAndre Oppermann 
36856a4e45aSAndre Oppermann 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
36956a5f52eSGleb Smirnoff 	    mb_ctor_clust,
37056a4e45aSAndre Oppermann #ifdef INVARIANTS
37156a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
37256a4e45aSAndre Oppermann #else
37356a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
37456a4e45aSAndre Oppermann #endif
37556a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
376ba63339aSAlan Cox 	uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
37745fe0bf7SPawel Jakub Dawidek 	if (nmbjumbo16 > 0)
37845fe0bf7SPawel Jakub Dawidek 		nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
3796e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
380e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
38156a4e45aSAndre Oppermann 
382099a0e58SBosko Milekic 	/*
383099a0e58SBosko Milekic 	 * Hook event handler for low-memory situation, used to
384099a0e58SBosko Milekic 	 * drain protocols and push data back to the caches (UMA
385099a0e58SBosko Milekic 	 * later pushes it back to VM).
386099a0e58SBosko Milekic 	 */
387099a0e58SBosko Milekic 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
388099a0e58SBosko Milekic 	    EVENTHANDLER_PRI_FIRST);
389*fb3bc596SJohn Baldwin 
390*fb3bc596SJohn Baldwin 	snd_tag_count = counter_u64_alloc(M_WAITOK);
391099a0e58SBosko Milekic }
39237140716SAndre Oppermann SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
393099a0e58SBosko Milekic 
3945475ca5aSMark Johnston #ifdef NETDUMP
3955475ca5aSMark Johnston /*
3965475ca5aSMark Johnston  * netdump makes use of a pre-allocated pool of mbufs and clusters.  When
3975475ca5aSMark Johnston  * netdump is configured, we initialize a set of UMA cache zones which return
3985475ca5aSMark Johnston  * items from this pool.  At panic-time, the regular UMA zone pointers are
3995475ca5aSMark Johnston  * overwritten with those of the cache zones so that drivers may allocate and
4005475ca5aSMark Johnston  * free mbufs and clusters without attempting to allocate physical memory.
4015475ca5aSMark Johnston  *
4025475ca5aSMark Johnston  * We keep mbufs and clusters in a pair of mbuf queues.  In particular, for
4035475ca5aSMark Johnston  * the purpose of caching clusters, we treat them as mbufs.
4045475ca5aSMark Johnston  */
4055475ca5aSMark Johnston static struct mbufq nd_mbufq =
4065475ca5aSMark Johnston     { STAILQ_HEAD_INITIALIZER(nd_mbufq.mq_head), 0, INT_MAX };
4075475ca5aSMark Johnston static struct mbufq nd_clustq =
4085475ca5aSMark Johnston     { STAILQ_HEAD_INITIALIZER(nd_clustq.mq_head), 0, INT_MAX };
4095475ca5aSMark Johnston 
4105475ca5aSMark Johnston static int nd_clsize;
4115475ca5aSMark Johnston static uma_zone_t nd_zone_mbuf;
4125475ca5aSMark Johnston static uma_zone_t nd_zone_clust;
4135475ca5aSMark Johnston static uma_zone_t nd_zone_pack;
4145475ca5aSMark Johnston 
4155475ca5aSMark Johnston static int
4165475ca5aSMark Johnston nd_buf_import(void *arg, void **store, int count, int domain __unused,
4175475ca5aSMark Johnston     int flags)
4185475ca5aSMark Johnston {
4195475ca5aSMark Johnston 	struct mbufq *q;
4205475ca5aSMark Johnston 	struct mbuf *m;
4215475ca5aSMark Johnston 	int i;
4225475ca5aSMark Johnston 
4235475ca5aSMark Johnston 	q = arg;
4245475ca5aSMark Johnston 
4255475ca5aSMark Johnston 	for (i = 0; i < count; i++) {
4265475ca5aSMark Johnston 		m = mbufq_dequeue(q);
4275475ca5aSMark Johnston 		if (m == NULL)
4285475ca5aSMark Johnston 			break;
4295475ca5aSMark Johnston 		trash_init(m, q == &nd_mbufq ? MSIZE : nd_clsize, flags);
4305475ca5aSMark Johnston 		store[i] = m;
4315475ca5aSMark Johnston 	}
4320d1467b1SConrad Meyer 	KASSERT((flags & M_WAITOK) == 0 || i == count,
4330d1467b1SConrad Meyer 	    ("%s: ran out of pre-allocated mbufs", __func__));
4345475ca5aSMark Johnston 	return (i);
4355475ca5aSMark Johnston }
4365475ca5aSMark Johnston 
4375475ca5aSMark Johnston static void
4385475ca5aSMark Johnston nd_buf_release(void *arg, void **store, int count)
4395475ca5aSMark Johnston {
4405475ca5aSMark Johnston 	struct mbufq *q;
4415475ca5aSMark Johnston 	struct mbuf *m;
4425475ca5aSMark Johnston 	int i;
4435475ca5aSMark Johnston 
4445475ca5aSMark Johnston 	q = arg;
4455475ca5aSMark Johnston 
4465475ca5aSMark Johnston 	for (i = 0; i < count; i++) {
4475475ca5aSMark Johnston 		m = store[i];
4485475ca5aSMark Johnston 		(void)mbufq_enqueue(q, m);
4495475ca5aSMark Johnston 	}
4505475ca5aSMark Johnston }
4515475ca5aSMark Johnston 
4525475ca5aSMark Johnston static int
4535475ca5aSMark Johnston nd_pack_import(void *arg __unused, void **store, int count, int domain __unused,
4545475ca5aSMark Johnston     int flags __unused)
4555475ca5aSMark Johnston {
4565475ca5aSMark Johnston 	struct mbuf *m;
4575475ca5aSMark Johnston 	void *clust;
4585475ca5aSMark Johnston 	int i;
4595475ca5aSMark Johnston 
4605475ca5aSMark Johnston 	for (i = 0; i < count; i++) {
4615475ca5aSMark Johnston 		m = m_get(MT_DATA, M_NOWAIT);
4625475ca5aSMark Johnston 		if (m == NULL)
4635475ca5aSMark Johnston 			break;
4645475ca5aSMark Johnston 		clust = uma_zalloc(nd_zone_clust, M_NOWAIT);
4655475ca5aSMark Johnston 		if (clust == NULL) {
4665475ca5aSMark Johnston 			m_free(m);
4675475ca5aSMark Johnston 			break;
4685475ca5aSMark Johnston 		}
4695475ca5aSMark Johnston 		mb_ctor_clust(clust, nd_clsize, m, 0);
4705475ca5aSMark Johnston 		store[i] = m;
4715475ca5aSMark Johnston 	}
4720d1467b1SConrad Meyer 	KASSERT((flags & M_WAITOK) == 0 || i == count,
4730d1467b1SConrad Meyer 	    ("%s: ran out of pre-allocated mbufs", __func__));
4745475ca5aSMark Johnston 	return (i);
4755475ca5aSMark Johnston }
4765475ca5aSMark Johnston 
4775475ca5aSMark Johnston static void
4785475ca5aSMark Johnston nd_pack_release(void *arg __unused, void **store, int count)
4795475ca5aSMark Johnston {
4805475ca5aSMark Johnston 	struct mbuf *m;
4815475ca5aSMark Johnston 	void *clust;
4825475ca5aSMark Johnston 	int i;
4835475ca5aSMark Johnston 
4845475ca5aSMark Johnston 	for (i = 0; i < count; i++) {
4855475ca5aSMark Johnston 		m = store[i];
4865475ca5aSMark Johnston 		clust = m->m_ext.ext_buf;
4875475ca5aSMark Johnston 		uma_zfree(nd_zone_clust, clust);
4885475ca5aSMark Johnston 		uma_zfree(nd_zone_mbuf, m);
4895475ca5aSMark Johnston 	}
4905475ca5aSMark Johnston }
4915475ca5aSMark Johnston 
4925475ca5aSMark Johnston /*
4935475ca5aSMark Johnston  * Free the pre-allocated mbufs and clusters reserved for netdump, and destroy
4945475ca5aSMark Johnston  * the corresponding UMA cache zones.
4955475ca5aSMark Johnston  */
4965475ca5aSMark Johnston void
4975475ca5aSMark Johnston netdump_mbuf_drain(void)
4985475ca5aSMark Johnston {
4995475ca5aSMark Johnston 	struct mbuf *m;
5005475ca5aSMark Johnston 	void *item;
5015475ca5aSMark Johnston 
5025475ca5aSMark Johnston 	if (nd_zone_mbuf != NULL) {
5035475ca5aSMark Johnston 		uma_zdestroy(nd_zone_mbuf);
5045475ca5aSMark Johnston 		nd_zone_mbuf = NULL;
5055475ca5aSMark Johnston 	}
5065475ca5aSMark Johnston 	if (nd_zone_clust != NULL) {
5075475ca5aSMark Johnston 		uma_zdestroy(nd_zone_clust);
5085475ca5aSMark Johnston 		nd_zone_clust = NULL;
5095475ca5aSMark Johnston 	}
5105475ca5aSMark Johnston 	if (nd_zone_pack != NULL) {
5115475ca5aSMark Johnston 		uma_zdestroy(nd_zone_pack);
5125475ca5aSMark Johnston 		nd_zone_pack = NULL;
5135475ca5aSMark Johnston 	}
5145475ca5aSMark Johnston 
5155475ca5aSMark Johnston 	while ((m = mbufq_dequeue(&nd_mbufq)) != NULL)
5165475ca5aSMark Johnston 		m_free(m);
5175475ca5aSMark Johnston 	while ((item = mbufq_dequeue(&nd_clustq)) != NULL)
5185475ca5aSMark Johnston 		uma_zfree(m_getzone(nd_clsize), item);
5195475ca5aSMark Johnston }
5205475ca5aSMark Johnston 
5215475ca5aSMark Johnston /*
5225475ca5aSMark Johnston  * Callback invoked immediately prior to starting a netdump.
5235475ca5aSMark Johnston  */
5245475ca5aSMark Johnston void
5255475ca5aSMark Johnston netdump_mbuf_dump(void)
5265475ca5aSMark Johnston {
5275475ca5aSMark Johnston 
5285475ca5aSMark Johnston 	/*
5295475ca5aSMark Johnston 	 * All cluster zones return buffers of the size requested by the
5305475ca5aSMark Johnston 	 * drivers.  It's up to the driver to reinitialize the zones if the
5315475ca5aSMark Johnston 	 * MTU of a netdump-enabled interface changes.
5325475ca5aSMark Johnston 	 */
5335475ca5aSMark Johnston 	printf("netdump: overwriting mbuf zone pointers\n");
5345475ca5aSMark Johnston 	zone_mbuf = nd_zone_mbuf;
5355475ca5aSMark Johnston 	zone_clust = nd_zone_clust;
5365475ca5aSMark Johnston 	zone_pack = nd_zone_pack;
5375475ca5aSMark Johnston 	zone_jumbop = nd_zone_clust;
5385475ca5aSMark Johnston 	zone_jumbo9 = nd_zone_clust;
5395475ca5aSMark Johnston 	zone_jumbo16 = nd_zone_clust;
5405475ca5aSMark Johnston }
5415475ca5aSMark Johnston 
5425475ca5aSMark Johnston /*
5435475ca5aSMark Johnston  * Reinitialize the netdump mbuf+cluster pool and cache zones.
5445475ca5aSMark Johnston  */
5455475ca5aSMark Johnston void
5465475ca5aSMark Johnston netdump_mbuf_reinit(int nmbuf, int nclust, int clsize)
5475475ca5aSMark Johnston {
5485475ca5aSMark Johnston 	struct mbuf *m;
5495475ca5aSMark Johnston 	void *item;
5505475ca5aSMark Johnston 
5515475ca5aSMark Johnston 	netdump_mbuf_drain();
5525475ca5aSMark Johnston 
5535475ca5aSMark Johnston 	nd_clsize = clsize;
5545475ca5aSMark Johnston 
5555475ca5aSMark Johnston 	nd_zone_mbuf = uma_zcache_create("netdump_" MBUF_MEM_NAME,
5565475ca5aSMark Johnston 	    MSIZE, mb_ctor_mbuf, mb_dtor_mbuf,
5575475ca5aSMark Johnston #ifdef INVARIANTS
5585475ca5aSMark Johnston 	    trash_init, trash_fini,
5595475ca5aSMark Johnston #else
5605475ca5aSMark Johnston 	    NULL, NULL,
5615475ca5aSMark Johnston #endif
5625475ca5aSMark Johnston 	    nd_buf_import, nd_buf_release,
5635475ca5aSMark Johnston 	    &nd_mbufq, UMA_ZONE_NOBUCKET);
5645475ca5aSMark Johnston 
5655475ca5aSMark Johnston 	nd_zone_clust = uma_zcache_create("netdump_" MBUF_CLUSTER_MEM_NAME,
5665475ca5aSMark Johnston 	    clsize, mb_ctor_clust,
5675475ca5aSMark Johnston #ifdef INVARIANTS
5685475ca5aSMark Johnston 	    trash_dtor, trash_init, trash_fini,
5695475ca5aSMark Johnston #else
5705475ca5aSMark Johnston 	    NULL, NULL, NULL,
5715475ca5aSMark Johnston #endif
5725475ca5aSMark Johnston 	    nd_buf_import, nd_buf_release,
5735475ca5aSMark Johnston 	    &nd_clustq, UMA_ZONE_NOBUCKET);
5745475ca5aSMark Johnston 
5755475ca5aSMark Johnston 	nd_zone_pack = uma_zcache_create("netdump_" MBUF_PACKET_MEM_NAME,
5765475ca5aSMark Johnston 	    MCLBYTES, mb_ctor_pack, mb_dtor_pack, NULL, NULL,
5775475ca5aSMark Johnston 	    nd_pack_import, nd_pack_release,
5785475ca5aSMark Johnston 	    NULL, UMA_ZONE_NOBUCKET);
5795475ca5aSMark Johnston 
5805475ca5aSMark Johnston 	while (nmbuf-- > 0) {
5815475ca5aSMark Johnston 		m = m_get(MT_DATA, M_WAITOK);
5825475ca5aSMark Johnston 		uma_zfree(nd_zone_mbuf, m);
5835475ca5aSMark Johnston 	}
5845475ca5aSMark Johnston 	while (nclust-- > 0) {
5855475ca5aSMark Johnston 		item = uma_zalloc(m_getzone(nd_clsize), M_WAITOK);
5865475ca5aSMark Johnston 		uma_zfree(nd_zone_clust, item);
5875475ca5aSMark Johnston 	}
5885475ca5aSMark Johnston }
5895475ca5aSMark Johnston #endif /* NETDUMP */
5905475ca5aSMark Johnston 
591099a0e58SBosko Milekic /*
592ba63339aSAlan Cox  * UMA backend page allocator for the jumbo frame zones.
593ba63339aSAlan Cox  *
594ba63339aSAlan Cox  * Allocates kernel virtual memory that is backed by contiguous physical
595ba63339aSAlan Cox  * pages.
596ba63339aSAlan Cox  */
597ba63339aSAlan Cox static void *
598ab3185d1SJeff Roberson mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags,
599ab3185d1SJeff Roberson     int wait)
600ba63339aSAlan Cox {
601ba63339aSAlan Cox 
6027630c265SAlan Cox 	/* Inform UMA that this allocator uses kernel_map/object. */
6037630c265SAlan Cox 	*flags = UMA_SLAB_KERNEL;
6049978bd99SMark Johnston 	return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain),
6059978bd99SMark Johnston 	    bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0,
6069978bd99SMark Johnston 	    VM_MEMATTR_DEFAULT));
607ba63339aSAlan Cox }
608ba63339aSAlan Cox 
609ba63339aSAlan Cox /*
610099a0e58SBosko Milekic  * Constructor for Mbuf master zone.
611099a0e58SBosko Milekic  *
612099a0e58SBosko Milekic  * The 'arg' pointer points to a mb_args structure which
613099a0e58SBosko Milekic  * contains call-specific information required to support the
61456a4e45aSAndre Oppermann  * mbuf allocation API.  See mbuf.h.
615099a0e58SBosko Milekic  */
616b23f72e9SBrian Feldman static int
617b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how)
618099a0e58SBosko Milekic {
619099a0e58SBosko Milekic 	struct mbuf *m;
620099a0e58SBosko Milekic 	struct mb_args *args;
621b23f72e9SBrian Feldman 	int error;
622099a0e58SBosko Milekic 	int flags;
623099a0e58SBosko Milekic 	short type;
624099a0e58SBosko Milekic 
625121f0509SMike Silbersack #ifdef INVARIANTS
626121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
627121f0509SMike Silbersack #endif
628099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
629099a0e58SBosko Milekic 	type = args->type;
630099a0e58SBosko Milekic 
63156a4e45aSAndre Oppermann 	/*
63256a4e45aSAndre Oppermann 	 * The mbuf is initialized later.  The caller has the
633fcf90618SGleb Smirnoff 	 * responsibility to set up any MAC labels too.
63456a4e45aSAndre Oppermann 	 */
63556a4e45aSAndre Oppermann 	if (type == MT_NOINIT)
63656a4e45aSAndre Oppermann 		return (0);
63756a4e45aSAndre Oppermann 
638afb295ccSAndre Oppermann 	m = (struct mbuf *)mem;
639afb295ccSAndre Oppermann 	flags = args->flags;
640fddd4f62SNavdeep Parhar 	MPASS((flags & M_NOFREE) == 0);
641afb295ccSAndre Oppermann 
642b4b12e52SGleb Smirnoff 	error = m_init(m, how, type, flags);
643afb295ccSAndre Oppermann 
644b23f72e9SBrian Feldman 	return (error);
645099a0e58SBosko Milekic }
646099a0e58SBosko Milekic 
647099a0e58SBosko Milekic /*
64856a4e45aSAndre Oppermann  * The Mbuf master zone destructor.
649099a0e58SBosko Milekic  */
650099a0e58SBosko Milekic static void
651099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg)
652099a0e58SBosko Milekic {
653099a0e58SBosko Milekic 	struct mbuf *m;
654629b9e08SKip Macy 	unsigned long flags;
655099a0e58SBosko Milekic 
656099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
657629b9e08SKip Macy 	flags = (unsigned long)arg;
658629b9e08SKip Macy 
659a9fa76f2SNavdeep Parhar 	KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
6604c7070dbSScott Long 	if (!(flags & MB_DTOR_SKIP) && (m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags))
661099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
662121f0509SMike Silbersack #ifdef INVARIANTS
663121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
664121f0509SMike Silbersack #endif
665099a0e58SBosko Milekic }
666099a0e58SBosko Milekic 
66756a4e45aSAndre Oppermann /*
66856a4e45aSAndre Oppermann  * The Mbuf Packet zone destructor.
66956a4e45aSAndre Oppermann  */
670099a0e58SBosko Milekic static void
671099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg)
672099a0e58SBosko Milekic {
673099a0e58SBosko Milekic 	struct mbuf *m;
674099a0e58SBosko Milekic 
675099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
676099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
677099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
67856a4e45aSAndre Oppermann 
67956a4e45aSAndre Oppermann 	/* Make sure we've got a clean cluster back. */
68056a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
68156a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
68256a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
683cf827063SPoul-Henning Kamp 	KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
684cf827063SPoul-Henning Kamp 	KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
68556a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
68649d46b61SGleb Smirnoff 	KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
687121f0509SMike Silbersack #ifdef INVARIANTS
688121f0509SMike Silbersack 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
689121f0509SMike Silbersack #endif
6906c125b8dSMohan Srinivasan 	/*
691ef44c8d2SDavid E. O'Brien 	 * If there are processes blocked on zone_clust, waiting for pages
692ef44c8d2SDavid E. O'Brien 	 * to be freed up, * cause them to be woken up by draining the
693ef44c8d2SDavid E. O'Brien 	 * packet zone.  We are exposed to a race here * (in the check for
694ef44c8d2SDavid E. O'Brien 	 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
695ef44c8d2SDavid E. O'Brien 	 * is deliberate. We don't want to acquire the zone lock for every
696ef44c8d2SDavid E. O'Brien 	 * mbuf free.
6976c125b8dSMohan Srinivasan 	 */
6986c125b8dSMohan Srinivasan 	if (uma_zone_exhausted_nolock(zone_clust))
6996c125b8dSMohan Srinivasan 		zone_drain(zone_pack);
700099a0e58SBosko Milekic }
701099a0e58SBosko Milekic 
702099a0e58SBosko Milekic /*
703ec63cb90SAndre Oppermann  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
704099a0e58SBosko Milekic  *
705099a0e58SBosko Milekic  * Here the 'arg' pointer points to the Mbuf which we
70656a4e45aSAndre Oppermann  * are configuring cluster storage for.  If 'arg' is
70756a4e45aSAndre Oppermann  * empty we allocate just the cluster without setting
70856a4e45aSAndre Oppermann  * the mbuf to it.  See mbuf.h.
709099a0e58SBosko Milekic  */
710b23f72e9SBrian Feldman static int
711b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how)
712099a0e58SBosko Milekic {
713099a0e58SBosko Milekic 	struct mbuf *m;
714099a0e58SBosko Milekic 
715121f0509SMike Silbersack #ifdef INVARIANTS
716121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
717121f0509SMike Silbersack #endif
7180f4d9d04SKip Macy 	m = (struct mbuf *)arg;
7190f4d9d04SKip Macy 	if (m != NULL) {
720e8fd18f3SGleb Smirnoff 		m->m_ext.ext_buf = (char *)mem;
721099a0e58SBosko Milekic 		m->m_data = m->m_ext.ext_buf;
722099a0e58SBosko Milekic 		m->m_flags |= M_EXT;
723099a0e58SBosko Milekic 		m->m_ext.ext_free = NULL;
724cf827063SPoul-Henning Kamp 		m->m_ext.ext_arg1 = NULL;
725cf827063SPoul-Henning Kamp 		m->m_ext.ext_arg2 = NULL;
72656a4e45aSAndre Oppermann 		m->m_ext.ext_size = size;
72756a5f52eSGleb Smirnoff 		m->m_ext.ext_type = m_gettype(size);
72856a5f52eSGleb Smirnoff 		m->m_ext.ext_flags = EXT_FLAG_EMBREF;
72956a5f52eSGleb Smirnoff 		m->m_ext.ext_count = 1;
73056a4e45aSAndre Oppermann 	}
7310f4d9d04SKip Macy 
732b23f72e9SBrian Feldman 	return (0);
733099a0e58SBosko Milekic }
734099a0e58SBosko Milekic 
73556a4e45aSAndre Oppermann /*
736099a0e58SBosko Milekic  * The Packet secondary zone's init routine, executed on the
73756a4e45aSAndre Oppermann  * object's transition from mbuf keg slab to zone cache.
738099a0e58SBosko Milekic  */
739b23f72e9SBrian Feldman static int
74056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how)
741099a0e58SBosko Milekic {
742099a0e58SBosko Milekic 	struct mbuf *m;
743099a0e58SBosko Milekic 
74456a4e45aSAndre Oppermann 	m = (struct mbuf *)mem;		/* m is virgin. */
745a7bd90efSAndre Oppermann 	if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
746a7bd90efSAndre Oppermann 	    m->m_ext.ext_buf == NULL)
747b23f72e9SBrian Feldman 		return (ENOMEM);
748cd5bb63bSAndre Oppermann 	m->m_ext.ext_type = EXT_PACKET;	/* Override. */
749121f0509SMike Silbersack #ifdef INVARIANTS
750121f0509SMike Silbersack 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
751121f0509SMike Silbersack #endif
752b23f72e9SBrian Feldman 	return (0);
753099a0e58SBosko Milekic }
754099a0e58SBosko Milekic 
755099a0e58SBosko Milekic /*
756099a0e58SBosko Milekic  * The Packet secondary zone's fini routine, executed on the
757099a0e58SBosko Milekic  * object's transition from zone cache to keg slab.
758099a0e58SBosko Milekic  */
759099a0e58SBosko Milekic static void
76056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size)
761099a0e58SBosko Milekic {
762099a0e58SBosko Milekic 	struct mbuf *m;
763099a0e58SBosko Milekic 
764099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
765121f0509SMike Silbersack #ifdef INVARIANTS
766121f0509SMike Silbersack 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
767121f0509SMike Silbersack #endif
768099a0e58SBosko Milekic 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
769a7b844d2SMike Silbersack #ifdef INVARIANTS
770a7b844d2SMike Silbersack 	trash_dtor(mem, size, NULL);
771a7b844d2SMike Silbersack #endif
772099a0e58SBosko Milekic }
773099a0e58SBosko Milekic 
774099a0e58SBosko Milekic /*
775099a0e58SBosko Milekic  * The "packet" keg constructor.
776099a0e58SBosko Milekic  */
777b23f72e9SBrian Feldman static int
778b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how)
779099a0e58SBosko Milekic {
780099a0e58SBosko Milekic 	struct mbuf *m;
781099a0e58SBosko Milekic 	struct mb_args *args;
782ce28636bSAndre Oppermann 	int error, flags;
783099a0e58SBosko Milekic 	short type;
784099a0e58SBosko Milekic 
785099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
786099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
787099a0e58SBosko Milekic 	flags = args->flags;
788099a0e58SBosko Milekic 	type = args->type;
789fddd4f62SNavdeep Parhar 	MPASS((flags & M_NOFREE) == 0);
790099a0e58SBosko Milekic 
791121f0509SMike Silbersack #ifdef INVARIANTS
792121f0509SMike Silbersack 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
793121f0509SMike Silbersack #endif
794099a0e58SBosko Milekic 
795b4b12e52SGleb Smirnoff 	error = m_init(m, how, type, flags);
796afb295ccSAndre Oppermann 
79756a4e45aSAndre Oppermann 	/* m_ext is already initialized. */
798afb295ccSAndre Oppermann 	m->m_data = m->m_ext.ext_buf;
799afb295ccSAndre Oppermann  	m->m_flags = (flags | M_EXT);
80056a4e45aSAndre Oppermann 
801afb295ccSAndre Oppermann 	return (error);
802099a0e58SBosko Milekic }
803099a0e58SBosko Milekic 
804099a0e58SBosko Milekic /*
805e60b2fcbSGleb Smirnoff  * This is the protocol drain routine.  Called by UMA whenever any of the
806e60b2fcbSGleb Smirnoff  * mbuf zones is closed to its limit.
807099a0e58SBosko Milekic  *
808099a0e58SBosko Milekic  * No locks should be held when this is called.  The drain routines have to
809099a0e58SBosko Milekic  * presently acquire some locks which raises the possibility of lock order
810099a0e58SBosko Milekic  * reversal.
811099a0e58SBosko Milekic  */
812099a0e58SBosko Milekic static void
813e60b2fcbSGleb Smirnoff mb_reclaim(uma_zone_t zone __unused, int pending __unused)
814099a0e58SBosko Milekic {
815099a0e58SBosko Milekic 	struct domain *dp;
816099a0e58SBosko Milekic 	struct protosw *pr;
817099a0e58SBosko Milekic 
818e60b2fcbSGleb Smirnoff 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
819099a0e58SBosko Milekic 
820099a0e58SBosko Milekic 	for (dp = domains; dp != NULL; dp = dp->dom_next)
821099a0e58SBosko Milekic 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
822099a0e58SBosko Milekic 			if (pr->pr_drain != NULL)
823099a0e58SBosko Milekic 				(*pr->pr_drain)();
824099a0e58SBosko Milekic }
8255e4bc63bSGleb Smirnoff 
8265e4bc63bSGleb Smirnoff /*
8275e4bc63bSGleb Smirnoff  * Clean up after mbufs with M_EXT storage attached to them if the
8285e4bc63bSGleb Smirnoff  * reference count hits 1.
8295e4bc63bSGleb Smirnoff  */
8305e4bc63bSGleb Smirnoff void
8315e4bc63bSGleb Smirnoff mb_free_ext(struct mbuf *m)
8325e4bc63bSGleb Smirnoff {
83356a5f52eSGleb Smirnoff 	volatile u_int *refcnt;
83456a5f52eSGleb Smirnoff 	struct mbuf *mref;
8355e4bc63bSGleb Smirnoff 	int freembuf;
8365e4bc63bSGleb Smirnoff 
8375e4bc63bSGleb Smirnoff 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
8385e4bc63bSGleb Smirnoff 
83956a5f52eSGleb Smirnoff 	/* See if this is the mbuf that holds the embedded refcount. */
84056a5f52eSGleb Smirnoff 	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
84156a5f52eSGleb Smirnoff 		refcnt = &m->m_ext.ext_count;
84256a5f52eSGleb Smirnoff 		mref = m;
84356a5f52eSGleb Smirnoff 	} else {
84456a5f52eSGleb Smirnoff 		KASSERT(m->m_ext.ext_cnt != NULL,
84556a5f52eSGleb Smirnoff 		    ("%s: no refcounting pointer on %p", __func__, m));
84656a5f52eSGleb Smirnoff 		refcnt = m->m_ext.ext_cnt;
84756a5f52eSGleb Smirnoff 		mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
84856a5f52eSGleb Smirnoff 	}
84956a5f52eSGleb Smirnoff 
8505e4bc63bSGleb Smirnoff 	/*
85156a5f52eSGleb Smirnoff 	 * Check if the header is embedded in the cluster.  It is
85256a5f52eSGleb Smirnoff 	 * important that we can't touch any of the mbuf fields
85356a5f52eSGleb Smirnoff 	 * after we have freed the external storage, since mbuf
85417cd649fSGleb Smirnoff 	 * could have been embedded in it.  For now, the mbufs
85517cd649fSGleb Smirnoff 	 * embedded into the cluster are always of type EXT_EXTREF,
85617cd649fSGleb Smirnoff 	 * and for this type we won't free the mref.
8575e4bc63bSGleb Smirnoff 	 */
85817cd649fSGleb Smirnoff 	if (m->m_flags & M_NOFREE) {
85917cd649fSGleb Smirnoff 		freembuf = 0;
860eec189c7SGleb Smirnoff 		KASSERT(m->m_ext.ext_type == EXT_EXTREF ||
861eec189c7SGleb Smirnoff 		    m->m_ext.ext_type == EXT_RXRING,
86217cd649fSGleb Smirnoff 		    ("%s: no-free mbuf %p has wrong type", __func__, m));
86317cd649fSGleb Smirnoff 	} else
86417cd649fSGleb Smirnoff 		freembuf = 1;
8655e4bc63bSGleb Smirnoff 
86656a5f52eSGleb Smirnoff 	/* Free attached storage if this mbuf is the only reference to it. */
86756a5f52eSGleb Smirnoff 	if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) {
8685e4bc63bSGleb Smirnoff 		switch (m->m_ext.ext_type) {
86956a5f52eSGleb Smirnoff 		case EXT_PACKET:
87056a5f52eSGleb Smirnoff 			/* The packet zone is special. */
87156a5f52eSGleb Smirnoff 			if (*refcnt == 0)
87256a5f52eSGleb Smirnoff 				*refcnt = 1;
87356a5f52eSGleb Smirnoff 			uma_zfree(zone_pack, mref);
8745e4bc63bSGleb Smirnoff 			break;
8755e4bc63bSGleb Smirnoff 		case EXT_CLUSTER:
8765e4bc63bSGleb Smirnoff 			uma_zfree(zone_clust, m->m_ext.ext_buf);
87756a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
8785e4bc63bSGleb Smirnoff 			break;
8795e4bc63bSGleb Smirnoff 		case EXT_JUMBOP:
8805e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
88156a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
8825e4bc63bSGleb Smirnoff 			break;
8835e4bc63bSGleb Smirnoff 		case EXT_JUMBO9:
8845e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
88556a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
8865e4bc63bSGleb Smirnoff 			break;
8875e4bc63bSGleb Smirnoff 		case EXT_JUMBO16:
8885e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
88956a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
89056a5f52eSGleb Smirnoff 			break;
89156a5f52eSGleb Smirnoff 		case EXT_SFBUF:
8925e4bc63bSGleb Smirnoff 		case EXT_NET_DRV:
8935e4bc63bSGleb Smirnoff 		case EXT_MOD_TYPE:
8945e4bc63bSGleb Smirnoff 		case EXT_DISPOSABLE:
89507e87a1dSGleb Smirnoff 			KASSERT(mref->m_ext.ext_free != NULL,
8960ea37a86SGleb Smirnoff 			    ("%s: ext_free not set", __func__));
89707e87a1dSGleb Smirnoff 			mref->m_ext.ext_free(mref);
89856a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
8990ea37a86SGleb Smirnoff 			break;
9005e4bc63bSGleb Smirnoff 		case EXT_EXTREF:
9015e4bc63bSGleb Smirnoff 			KASSERT(m->m_ext.ext_free != NULL,
9025e4bc63bSGleb Smirnoff 			    ("%s: ext_free not set", __func__));
903e8fd18f3SGleb Smirnoff 			m->m_ext.ext_free(m);
9045e4bc63bSGleb Smirnoff 			break;
905eec189c7SGleb Smirnoff 		case EXT_RXRING:
906eec189c7SGleb Smirnoff 			KASSERT(m->m_ext.ext_free == NULL,
907eec189c7SGleb Smirnoff 			    ("%s: ext_free is set", __func__));
908eec189c7SGleb Smirnoff 			break;
9095e4bc63bSGleb Smirnoff 		default:
9105e4bc63bSGleb Smirnoff 			KASSERT(m->m_ext.ext_type == 0,
9115e4bc63bSGleb Smirnoff 			    ("%s: unknown ext_type", __func__));
9125e4bc63bSGleb Smirnoff 		}
9135e4bc63bSGleb Smirnoff 	}
9145e4bc63bSGleb Smirnoff 
91556a5f52eSGleb Smirnoff 	if (freembuf && m != mref)
9165e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
9175e4bc63bSGleb Smirnoff }
9185e4bc63bSGleb Smirnoff 
9195e4bc63bSGleb Smirnoff /*
9205e4bc63bSGleb Smirnoff  * Official mbuf(9) allocation KPI for stack and drivers:
9215e4bc63bSGleb Smirnoff  *
9225e4bc63bSGleb Smirnoff  * m_get()	- a single mbuf without any attachments, sys/mbuf.h.
9235e4bc63bSGleb Smirnoff  * m_gethdr()	- a single mbuf initialized as M_PKTHDR, sys/mbuf.h.
9245e4bc63bSGleb Smirnoff  * m_getcl()	- an mbuf + 2k cluster, sys/mbuf.h.
9255e4bc63bSGleb Smirnoff  * m_clget()	- attach cluster to already allocated mbuf.
9265e4bc63bSGleb Smirnoff  * m_cljget()	- attach jumbo cluster to already allocated mbuf.
9275e4bc63bSGleb Smirnoff  * m_get2()	- allocate minimum mbuf that would fit size argument.
9285e4bc63bSGleb Smirnoff  * m_getm2()	- allocate a chain of mbufs/clusters.
9295e4bc63bSGleb Smirnoff  * m_extadd()	- attach external cluster to mbuf.
9305e4bc63bSGleb Smirnoff  *
9315e4bc63bSGleb Smirnoff  * m_free()	- free single mbuf with its tags and ext, sys/mbuf.h.
9325e4bc63bSGleb Smirnoff  * m_freem()	- free chain of mbufs.
9335e4bc63bSGleb Smirnoff  */
9345e4bc63bSGleb Smirnoff 
9355e4bc63bSGleb Smirnoff int
9365e4bc63bSGleb Smirnoff m_clget(struct mbuf *m, int how)
9375e4bc63bSGleb Smirnoff {
9385e4bc63bSGleb Smirnoff 
9395e4bc63bSGleb Smirnoff 	KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
9405e4bc63bSGleb Smirnoff 	    __func__, m));
9415e4bc63bSGleb Smirnoff 	m->m_ext.ext_buf = (char *)NULL;
9425e4bc63bSGleb Smirnoff 	uma_zalloc_arg(zone_clust, m, how);
9435e4bc63bSGleb Smirnoff 	/*
9445e4bc63bSGleb Smirnoff 	 * On a cluster allocation failure, drain the packet zone and retry,
9455e4bc63bSGleb Smirnoff 	 * we might be able to loosen a few clusters up on the drain.
9465e4bc63bSGleb Smirnoff 	 */
9475e4bc63bSGleb Smirnoff 	if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
9485e4bc63bSGleb Smirnoff 		zone_drain(zone_pack);
9495e4bc63bSGleb Smirnoff 		uma_zalloc_arg(zone_clust, m, how);
9505e4bc63bSGleb Smirnoff 	}
951480f4e94SGeorge V. Neville-Neil 	MBUF_PROBE2(m__clget, m, how);
9525e4bc63bSGleb Smirnoff 	return (m->m_flags & M_EXT);
9535e4bc63bSGleb Smirnoff }
9545e4bc63bSGleb Smirnoff 
9555e4bc63bSGleb Smirnoff /*
9565e4bc63bSGleb Smirnoff  * m_cljget() is different from m_clget() as it can allocate clusters without
9575e4bc63bSGleb Smirnoff  * attaching them to an mbuf.  In that case the return value is the pointer
9585e4bc63bSGleb Smirnoff  * to the cluster of the requested size.  If an mbuf was specified, it gets
9595e4bc63bSGleb Smirnoff  * the cluster attached to it and the return value can be safely ignored.
9605e4bc63bSGleb Smirnoff  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
9615e4bc63bSGleb Smirnoff  */
9625e4bc63bSGleb Smirnoff void *
9635e4bc63bSGleb Smirnoff m_cljget(struct mbuf *m, int how, int size)
9645e4bc63bSGleb Smirnoff {
9655e4bc63bSGleb Smirnoff 	uma_zone_t zone;
966480f4e94SGeorge V. Neville-Neil 	void *retval;
9675e4bc63bSGleb Smirnoff 
9685e4bc63bSGleb Smirnoff 	if (m != NULL) {
9695e4bc63bSGleb Smirnoff 		KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
9705e4bc63bSGleb Smirnoff 		    __func__, m));
9715e4bc63bSGleb Smirnoff 		m->m_ext.ext_buf = NULL;
9725e4bc63bSGleb Smirnoff 	}
9735e4bc63bSGleb Smirnoff 
9745e4bc63bSGleb Smirnoff 	zone = m_getzone(size);
975480f4e94SGeorge V. Neville-Neil 	retval = uma_zalloc_arg(zone, m, how);
976480f4e94SGeorge V. Neville-Neil 
977480f4e94SGeorge V. Neville-Neil 	MBUF_PROBE4(m__cljget, m, how, size, retval);
978480f4e94SGeorge V. Neville-Neil 
979480f4e94SGeorge V. Neville-Neil 	return (retval);
9805e4bc63bSGleb Smirnoff }
9815e4bc63bSGleb Smirnoff 
9825e4bc63bSGleb Smirnoff /*
9835e4bc63bSGleb Smirnoff  * m_get2() allocates minimum mbuf that would fit "size" argument.
9845e4bc63bSGleb Smirnoff  */
9855e4bc63bSGleb Smirnoff struct mbuf *
9865e4bc63bSGleb Smirnoff m_get2(int size, int how, short type, int flags)
9875e4bc63bSGleb Smirnoff {
9885e4bc63bSGleb Smirnoff 	struct mb_args args;
9895e4bc63bSGleb Smirnoff 	struct mbuf *m, *n;
9905e4bc63bSGleb Smirnoff 
9915e4bc63bSGleb Smirnoff 	args.flags = flags;
9925e4bc63bSGleb Smirnoff 	args.type = type;
9935e4bc63bSGleb Smirnoff 
9945e4bc63bSGleb Smirnoff 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
9955e4bc63bSGleb Smirnoff 		return (uma_zalloc_arg(zone_mbuf, &args, how));
9965e4bc63bSGleb Smirnoff 	if (size <= MCLBYTES)
9975e4bc63bSGleb Smirnoff 		return (uma_zalloc_arg(zone_pack, &args, how));
9985e4bc63bSGleb Smirnoff 
9995e4bc63bSGleb Smirnoff 	if (size > MJUMPAGESIZE)
10005e4bc63bSGleb Smirnoff 		return (NULL);
10015e4bc63bSGleb Smirnoff 
10025e4bc63bSGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
10035e4bc63bSGleb Smirnoff 	if (m == NULL)
10045e4bc63bSGleb Smirnoff 		return (NULL);
10055e4bc63bSGleb Smirnoff 
10065e4bc63bSGleb Smirnoff 	n = uma_zalloc_arg(zone_jumbop, m, how);
10075e4bc63bSGleb Smirnoff 	if (n == NULL) {
10085e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
10095e4bc63bSGleb Smirnoff 		return (NULL);
10105e4bc63bSGleb Smirnoff 	}
10115e4bc63bSGleb Smirnoff 
10125e4bc63bSGleb Smirnoff 	return (m);
10135e4bc63bSGleb Smirnoff }
10145e4bc63bSGleb Smirnoff 
10155e4bc63bSGleb Smirnoff /*
10165e4bc63bSGleb Smirnoff  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
10175e4bc63bSGleb Smirnoff  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
10185e4bc63bSGleb Smirnoff  */
10195e4bc63bSGleb Smirnoff struct mbuf *
10205e4bc63bSGleb Smirnoff m_getjcl(int how, short type, int flags, int size)
10215e4bc63bSGleb Smirnoff {
10225e4bc63bSGleb Smirnoff 	struct mb_args args;
10235e4bc63bSGleb Smirnoff 	struct mbuf *m, *n;
10245e4bc63bSGleb Smirnoff 	uma_zone_t zone;
10255e4bc63bSGleb Smirnoff 
10265e4bc63bSGleb Smirnoff 	if (size == MCLBYTES)
10275e4bc63bSGleb Smirnoff 		return m_getcl(how, type, flags);
10285e4bc63bSGleb Smirnoff 
10295e4bc63bSGleb Smirnoff 	args.flags = flags;
10305e4bc63bSGleb Smirnoff 	args.type = type;
10315e4bc63bSGleb Smirnoff 
10325e4bc63bSGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
10335e4bc63bSGleb Smirnoff 	if (m == NULL)
10345e4bc63bSGleb Smirnoff 		return (NULL);
10355e4bc63bSGleb Smirnoff 
10365e4bc63bSGleb Smirnoff 	zone = m_getzone(size);
10375e4bc63bSGleb Smirnoff 	n = uma_zalloc_arg(zone, m, how);
10385e4bc63bSGleb Smirnoff 	if (n == NULL) {
10395e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
10405e4bc63bSGleb Smirnoff 		return (NULL);
10415e4bc63bSGleb Smirnoff 	}
10425e4bc63bSGleb Smirnoff 	return (m);
10435e4bc63bSGleb Smirnoff }
10445e4bc63bSGleb Smirnoff 
10455e4bc63bSGleb Smirnoff /*
10465e4bc63bSGleb Smirnoff  * Allocate a given length worth of mbufs and/or clusters (whatever fits
10475e4bc63bSGleb Smirnoff  * best) and return a pointer to the top of the allocated chain.  If an
10485e4bc63bSGleb Smirnoff  * existing mbuf chain is provided, then we will append the new chain
104958c43838SAndriy Voskoboinyk  * to the existing one and return a pointer to the provided mbuf.
10505e4bc63bSGleb Smirnoff  */
10515e4bc63bSGleb Smirnoff struct mbuf *
10525e4bc63bSGleb Smirnoff m_getm2(struct mbuf *m, int len, int how, short type, int flags)
10535e4bc63bSGleb Smirnoff {
10545e4bc63bSGleb Smirnoff 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
10555e4bc63bSGleb Smirnoff 
10565e4bc63bSGleb Smirnoff 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
10575e4bc63bSGleb Smirnoff 
10585e4bc63bSGleb Smirnoff 	/* Validate flags. */
10595e4bc63bSGleb Smirnoff 	flags &= (M_PKTHDR | M_EOR);
10605e4bc63bSGleb Smirnoff 
10615e4bc63bSGleb Smirnoff 	/* Packet header mbuf must be first in chain. */
10625e4bc63bSGleb Smirnoff 	if ((flags & M_PKTHDR) && m != NULL)
10635e4bc63bSGleb Smirnoff 		flags &= ~M_PKTHDR;
10645e4bc63bSGleb Smirnoff 
10655e4bc63bSGleb Smirnoff 	/* Loop and append maximum sized mbufs to the chain tail. */
10665e4bc63bSGleb Smirnoff 	while (len > 0) {
10675e4bc63bSGleb Smirnoff 		if (len > MCLBYTES)
10685e4bc63bSGleb Smirnoff 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
10695e4bc63bSGleb Smirnoff 			    MJUMPAGESIZE);
10705e4bc63bSGleb Smirnoff 		else if (len >= MINCLSIZE)
10715e4bc63bSGleb Smirnoff 			mb = m_getcl(how, type, (flags & M_PKTHDR));
10725e4bc63bSGleb Smirnoff 		else if (flags & M_PKTHDR)
10735e4bc63bSGleb Smirnoff 			mb = m_gethdr(how, type);
10745e4bc63bSGleb Smirnoff 		else
10755e4bc63bSGleb Smirnoff 			mb = m_get(how, type);
10765e4bc63bSGleb Smirnoff 
10775e4bc63bSGleb Smirnoff 		/* Fail the whole operation if one mbuf can't be allocated. */
10785e4bc63bSGleb Smirnoff 		if (mb == NULL) {
10795e4bc63bSGleb Smirnoff 			if (nm != NULL)
10805e4bc63bSGleb Smirnoff 				m_freem(nm);
10815e4bc63bSGleb Smirnoff 			return (NULL);
10825e4bc63bSGleb Smirnoff 		}
10835e4bc63bSGleb Smirnoff 
10845e4bc63bSGleb Smirnoff 		/* Book keeping. */
10855e4bc63bSGleb Smirnoff 		len -= M_SIZE(mb);
10865e4bc63bSGleb Smirnoff 		if (mtail != NULL)
10875e4bc63bSGleb Smirnoff 			mtail->m_next = mb;
10885e4bc63bSGleb Smirnoff 		else
10895e4bc63bSGleb Smirnoff 			nm = mb;
10905e4bc63bSGleb Smirnoff 		mtail = mb;
10915e4bc63bSGleb Smirnoff 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
10925e4bc63bSGleb Smirnoff 	}
10935e4bc63bSGleb Smirnoff 	if (flags & M_EOR)
10945e4bc63bSGleb Smirnoff 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
10955e4bc63bSGleb Smirnoff 
10965e4bc63bSGleb Smirnoff 	/* If mbuf was supplied, append new chain to the end of it. */
10975e4bc63bSGleb Smirnoff 	if (m != NULL) {
10985e4bc63bSGleb Smirnoff 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
10995e4bc63bSGleb Smirnoff 			;
11005e4bc63bSGleb Smirnoff 		mtail->m_next = nm;
11015e4bc63bSGleb Smirnoff 		mtail->m_flags &= ~M_EOR;
11025e4bc63bSGleb Smirnoff 	} else
11035e4bc63bSGleb Smirnoff 		m = nm;
11045e4bc63bSGleb Smirnoff 
11055e4bc63bSGleb Smirnoff 	return (m);
11065e4bc63bSGleb Smirnoff }
11075e4bc63bSGleb Smirnoff 
11085e4bc63bSGleb Smirnoff /*-
11095e4bc63bSGleb Smirnoff  * Configure a provided mbuf to refer to the provided external storage
111056a5f52eSGleb Smirnoff  * buffer and setup a reference count for said buffer.
11115e4bc63bSGleb Smirnoff  *
11125e4bc63bSGleb Smirnoff  * Arguments:
11135e4bc63bSGleb Smirnoff  *    mb     The existing mbuf to which to attach the provided buffer.
11145e4bc63bSGleb Smirnoff  *    buf    The address of the provided external storage buffer.
11155e4bc63bSGleb Smirnoff  *    size   The size of the provided buffer.
11165e4bc63bSGleb Smirnoff  *    freef  A pointer to a routine that is responsible for freeing the
11175e4bc63bSGleb Smirnoff  *           provided external storage buffer.
11185e4bc63bSGleb Smirnoff  *    args   A pointer to an argument structure (of any type) to be passed
11195e4bc63bSGleb Smirnoff  *           to the provided freef routine (may be NULL).
11205e4bc63bSGleb Smirnoff  *    flags  Any other flags to be passed to the provided mbuf.
11215e4bc63bSGleb Smirnoff  *    type   The type that the external storage buffer should be
11225e4bc63bSGleb Smirnoff  *           labeled with.
11235e4bc63bSGleb Smirnoff  *
11245e4bc63bSGleb Smirnoff  * Returns:
11255e4bc63bSGleb Smirnoff  *    Nothing.
11265e4bc63bSGleb Smirnoff  */
112756a5f52eSGleb Smirnoff void
1128e8fd18f3SGleb Smirnoff m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef,
1129e8fd18f3SGleb Smirnoff     void *arg1, void *arg2, int flags, int type)
11305e4bc63bSGleb Smirnoff {
113156a5f52eSGleb Smirnoff 
11325e4bc63bSGleb Smirnoff 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
11335e4bc63bSGleb Smirnoff 
11345e4bc63bSGleb Smirnoff 	mb->m_flags |= (M_EXT | flags);
11355e4bc63bSGleb Smirnoff 	mb->m_ext.ext_buf = buf;
11365e4bc63bSGleb Smirnoff 	mb->m_data = mb->m_ext.ext_buf;
11375e4bc63bSGleb Smirnoff 	mb->m_ext.ext_size = size;
11385e4bc63bSGleb Smirnoff 	mb->m_ext.ext_free = freef;
11395e4bc63bSGleb Smirnoff 	mb->m_ext.ext_arg1 = arg1;
11405e4bc63bSGleb Smirnoff 	mb->m_ext.ext_arg2 = arg2;
11415e4bc63bSGleb Smirnoff 	mb->m_ext.ext_type = type;
11425e4bc63bSGleb Smirnoff 
114356a5f52eSGleb Smirnoff 	if (type != EXT_EXTREF) {
114456a5f52eSGleb Smirnoff 		mb->m_ext.ext_count = 1;
114556a5f52eSGleb Smirnoff 		mb->m_ext.ext_flags = EXT_FLAG_EMBREF;
114656a5f52eSGleb Smirnoff 	} else
114756a5f52eSGleb Smirnoff 		mb->m_ext.ext_flags = 0;
11485e4bc63bSGleb Smirnoff }
11495e4bc63bSGleb Smirnoff 
11505e4bc63bSGleb Smirnoff /*
11515e4bc63bSGleb Smirnoff  * Free an entire chain of mbufs and associated external buffers, if
11525e4bc63bSGleb Smirnoff  * applicable.
11535e4bc63bSGleb Smirnoff  */
11545e4bc63bSGleb Smirnoff void
11555e4bc63bSGleb Smirnoff m_freem(struct mbuf *mb)
11565e4bc63bSGleb Smirnoff {
11575e4bc63bSGleb Smirnoff 
1158c8f59118SGleb Smirnoff 	MBUF_PROBE1(m__freem, mb);
11595e4bc63bSGleb Smirnoff 	while (mb != NULL)
11605e4bc63bSGleb Smirnoff 		mb = m_free(mb);
11615e4bc63bSGleb Smirnoff }
1162*fb3bc596SJohn Baldwin 
1163*fb3bc596SJohn Baldwin void
1164*fb3bc596SJohn Baldwin m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp)
1165*fb3bc596SJohn Baldwin {
1166*fb3bc596SJohn Baldwin 
1167*fb3bc596SJohn Baldwin 	if_ref(ifp);
1168*fb3bc596SJohn Baldwin 	mst->ifp = ifp;
1169*fb3bc596SJohn Baldwin 	refcount_init(&mst->refcount, 1);
1170*fb3bc596SJohn Baldwin 	counter_u64_add(snd_tag_count, 1);
1171*fb3bc596SJohn Baldwin }
1172*fb3bc596SJohn Baldwin 
1173*fb3bc596SJohn Baldwin void
1174*fb3bc596SJohn Baldwin m_snd_tag_destroy(struct m_snd_tag *mst)
1175*fb3bc596SJohn Baldwin {
1176*fb3bc596SJohn Baldwin 	struct ifnet *ifp;
1177*fb3bc596SJohn Baldwin 
1178*fb3bc596SJohn Baldwin 	ifp = mst->ifp;
1179*fb3bc596SJohn Baldwin 	ifp->if_snd_tag_free(mst);
1180*fb3bc596SJohn Baldwin 	if_rele(ifp);
1181*fb3bc596SJohn Baldwin 	counter_u64_add(snd_tag_count, -1);
1182*fb3bc596SJohn Baldwin }
1183