xref: /freebsd/sys/kern/kern_mbuf.c (revision e8fd18f306915c411b08ab4664ec0aa3bc03d4d4)
1099a0e58SBosko Milekic /*-
28076cb52SBosko Milekic  * Copyright (c) 2004, 2005,
38076cb52SBosko Milekic  *	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
4099a0e58SBosko Milekic  *
5099a0e58SBosko Milekic  * Redistribution and use in source and binary forms, with or without
6099a0e58SBosko Milekic  * modification, are permitted provided that the following conditions
7099a0e58SBosko Milekic  * are met:
8099a0e58SBosko Milekic  * 1. Redistributions of source code must retain the above copyright
9099a0e58SBosko Milekic  *    notice unmodified, this list of conditions and the following
10099a0e58SBosko Milekic  *    disclaimer.
11099a0e58SBosko Milekic  * 2. Redistributions in binary form must reproduce the above copyright
12099a0e58SBosko Milekic  *    notice, this list of conditions and the following disclaimer in the
13099a0e58SBosko Milekic  *    documentation and/or other materials provided with the distribution.
14099a0e58SBosko Milekic  *
15099a0e58SBosko Milekic  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16099a0e58SBosko Milekic  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17099a0e58SBosko Milekic  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18099a0e58SBosko Milekic  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19099a0e58SBosko Milekic  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20099a0e58SBosko Milekic  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21099a0e58SBosko Milekic  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22099a0e58SBosko Milekic  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23099a0e58SBosko Milekic  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24099a0e58SBosko Milekic  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25099a0e58SBosko Milekic  * SUCH DAMAGE.
26099a0e58SBosko Milekic  */
27099a0e58SBosko Milekic 
28099a0e58SBosko Milekic #include <sys/cdefs.h>
29099a0e58SBosko Milekic __FBSDID("$FreeBSD$");
30099a0e58SBosko Milekic 
31099a0e58SBosko Milekic #include "opt_param.h"
32099a0e58SBosko Milekic 
33099a0e58SBosko Milekic #include <sys/param.h>
34099a0e58SBosko Milekic #include <sys/malloc.h>
3554503a13SJonathan T. Looney #include <sys/types.h>
36099a0e58SBosko Milekic #include <sys/systm.h>
37099a0e58SBosko Milekic #include <sys/mbuf.h>
38099a0e58SBosko Milekic #include <sys/domain.h>
39099a0e58SBosko Milekic #include <sys/eventhandler.h>
40099a0e58SBosko Milekic #include <sys/kernel.h>
4154503a13SJonathan T. Looney #include <sys/lock.h>
4254503a13SJonathan T. Looney #include <sys/mutex.h>
43099a0e58SBosko Milekic #include <sys/protosw.h>
44099a0e58SBosko Milekic #include <sys/smp.h>
45099a0e58SBosko Milekic #include <sys/sysctl.h>
46099a0e58SBosko Milekic 
47099a0e58SBosko Milekic #include <vm/vm.h>
48c45c0034SAlan Cox #include <vm/vm_extern.h>
49c45c0034SAlan Cox #include <vm/vm_kern.h>
50099a0e58SBosko Milekic #include <vm/vm_page.h>
5137140716SAndre Oppermann #include <vm/vm_map.h>
52099a0e58SBosko Milekic #include <vm/uma.h>
53121f0509SMike Silbersack #include <vm/uma_dbg.h>
54099a0e58SBosko Milekic 
55099a0e58SBosko Milekic /*
56099a0e58SBosko Milekic  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
57099a0e58SBosko Milekic  * Zones.
58099a0e58SBosko Milekic  *
59099a0e58SBosko Milekic  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
60099a0e58SBosko Milekic  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
61099a0e58SBosko Milekic  * administrator so desires.
62099a0e58SBosko Milekic  *
63099a0e58SBosko Milekic  * Mbufs are allocated from a UMA Master Zone called the Mbuf
64099a0e58SBosko Milekic  * Zone.
65099a0e58SBosko Milekic  *
66099a0e58SBosko Milekic  * Additionally, FreeBSD provides a Packet Zone, which it
67099a0e58SBosko Milekic  * configures as a Secondary Zone to the Mbuf Master Zone,
68099a0e58SBosko Milekic  * thus sharing backend Slab kegs with the Mbuf Master Zone.
69099a0e58SBosko Milekic  *
70099a0e58SBosko Milekic  * Thus common-case allocations and locking are simplified:
71099a0e58SBosko Milekic  *
72099a0e58SBosko Milekic  *  m_clget()                m_getcl()
73099a0e58SBosko Milekic  *    |                         |
74099a0e58SBosko Milekic  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
75099a0e58SBosko Milekic  *    |   |             [     Packet   ]            |
76099a0e58SBosko Milekic  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
77099a0e58SBosko Milekic  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
78099a0e58SBosko Milekic  *        |                       \________         |
79099a0e58SBosko Milekic  *  [ Cluster Keg   ]                      \       /
80099a0e58SBosko Milekic  *        |	                         [ Mbuf Keg   ]
81099a0e58SBosko Milekic  *  [ Cluster Slabs ]                         |
82099a0e58SBosko Milekic  *        |                              [ Mbuf Slabs ]
83099a0e58SBosko Milekic  *         \____________(VM)_________________/
8456a4e45aSAndre Oppermann  *
8556a4e45aSAndre Oppermann  *
86fcf90618SGleb Smirnoff  * Whenever an object is allocated with uma_zalloc() out of
8756a4e45aSAndre Oppermann  * one of the Zones its _ctor_ function is executed.  The same
88fcf90618SGleb Smirnoff  * for any deallocation through uma_zfree() the _dtor_ function
8956a4e45aSAndre Oppermann  * is executed.
9056a4e45aSAndre Oppermann  *
9156a4e45aSAndre Oppermann  * Caches are per-CPU and are filled from the Master Zone.
9256a4e45aSAndre Oppermann  *
93fcf90618SGleb Smirnoff  * Whenever an object is allocated from the underlying global
9456a4e45aSAndre Oppermann  * memory pool it gets pre-initialized with the _zinit_ functions.
95e3043798SPedro F. Giffuni  * When the Keg's are overfull objects get decommissioned with
9656a4e45aSAndre Oppermann  * _zfini_ functions and free'd back to the global memory pool.
9756a4e45aSAndre Oppermann  *
98099a0e58SBosko Milekic  */
99099a0e58SBosko Milekic 
100ead46972SAndre Oppermann int nmbufs;			/* limits number of mbufs */
10156a4e45aSAndre Oppermann int nmbclusters;		/* limits number of mbuf clusters */
102ec63cb90SAndre Oppermann int nmbjumbop;			/* limits number of page size jumbo clusters */
10356a4e45aSAndre Oppermann int nmbjumbo9;			/* limits number of 9k jumbo clusters */
10456a4e45aSAndre Oppermann int nmbjumbo16;			/* limits number of 16k jumbo clusters */
105099a0e58SBosko Milekic 
106e0c00addSAndre Oppermann static quad_t maxmbufmem;	/* overall real memory limit for all mbufs */
107e0c00addSAndre Oppermann 
108af3b2549SHans Petter Selasky SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0,
109b6f49c23SHiren Panchasara     "Maximum real memory allocatable to various mbuf types");
110e0c00addSAndre Oppermann 
11162938659SBjoern A. Zeeb /*
11237140716SAndre Oppermann  * tunable_mbinit() has to be run before any mbuf allocations are done.
11362938659SBjoern A. Zeeb  */
114099a0e58SBosko Milekic static void
115099a0e58SBosko Milekic tunable_mbinit(void *dummy)
116099a0e58SBosko Milekic {
117e0c00addSAndre Oppermann 	quad_t realmem;
11837140716SAndre Oppermann 
11937140716SAndre Oppermann 	/*
12037140716SAndre Oppermann 	 * The default limit for all mbuf related memory is 1/2 of all
12137140716SAndre Oppermann 	 * available kernel memory (physical or kmem).
12237140716SAndre Oppermann 	 * At most it can be 3/4 of available kernel memory.
12337140716SAndre Oppermann 	 */
1245df87b21SJeff Roberson 	realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size);
12537140716SAndre Oppermann 	maxmbufmem = realmem / 2;
126e0c00addSAndre Oppermann 	TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem);
12737140716SAndre Oppermann 	if (maxmbufmem > realmem / 4 * 3)
12837140716SAndre Oppermann 		maxmbufmem = realmem / 4 * 3;
129099a0e58SBosko Milekic 
130812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
131416a434cSAndre Oppermann 	if (nmbclusters == 0)
132416a434cSAndre Oppermann 		nmbclusters = maxmbufmem / MCLBYTES / 4;
133812302c3SNavdeep Parhar 
134812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop);
135812302c3SNavdeep Parhar 	if (nmbjumbop == 0)
136416a434cSAndre Oppermann 		nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4;
137812302c3SNavdeep Parhar 
138812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9);
139812302c3SNavdeep Parhar 	if (nmbjumbo9 == 0)
140416a434cSAndre Oppermann 		nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6;
141812302c3SNavdeep Parhar 
142812302c3SNavdeep Parhar 	TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16);
143812302c3SNavdeep Parhar 	if (nmbjumbo16 == 0)
144416a434cSAndre Oppermann 		nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6;
145416a434cSAndre Oppermann 
146416a434cSAndre Oppermann 	/*
147416a434cSAndre Oppermann 	 * We need at least as many mbufs as we have clusters of
148416a434cSAndre Oppermann 	 * the various types added together.
149416a434cSAndre Oppermann 	 */
150416a434cSAndre Oppermann 	TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs);
151416a434cSAndre Oppermann 	if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16)
152416a434cSAndre Oppermann 		nmbufs = lmax(maxmbufmem / MSIZE / 5,
153416a434cSAndre Oppermann 		    nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16);
154099a0e58SBosko Milekic }
15537140716SAndre Oppermann SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL);
156099a0e58SBosko Milekic 
1574f590175SPaul Saab static int
1584f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
1594f590175SPaul Saab {
1604f590175SPaul Saab 	int error, newnmbclusters;
1614f590175SPaul Saab 
1624f590175SPaul Saab 	newnmbclusters = nmbclusters;
163041b706bSDavid Malone 	error = sysctl_handle_int(oidp, &newnmbclusters, 0, req);
164d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbclusters != nmbclusters) {
165ead46972SAndre Oppermann 		if (newnmbclusters > nmbclusters &&
166ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
1674f590175SPaul Saab 			nmbclusters = newnmbclusters;
168bc4a1b8cSAndre Oppermann 			nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
1694f590175SPaul Saab 			EVENTHANDLER_INVOKE(nmbclusters_change);
1704f590175SPaul Saab 		} else
1714f590175SPaul Saab 			error = EINVAL;
1724f590175SPaul Saab 	}
1734f590175SPaul Saab 	return (error);
1744f590175SPaul Saab }
1754f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
1764f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU",
177099a0e58SBosko Milekic     "Maximum number of mbuf clusters allowed");
178cf70a46bSRandall Stewart 
179cf70a46bSRandall Stewart static int
180cf70a46bSRandall Stewart sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS)
181cf70a46bSRandall Stewart {
182cf70a46bSRandall Stewart 	int error, newnmbjumbop;
183cf70a46bSRandall Stewart 
184cf70a46bSRandall Stewart 	newnmbjumbop = nmbjumbop;
185cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req);
186d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) {
187ead46972SAndre Oppermann 		if (newnmbjumbop > nmbjumbop &&
188ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
189cf70a46bSRandall Stewart 			nmbjumbop = newnmbjumbop;
190bc4a1b8cSAndre Oppermann 			nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
191cf70a46bSRandall Stewart 		} else
192cf70a46bSRandall Stewart 			error = EINVAL;
193cf70a46bSRandall Stewart 	}
194cf70a46bSRandall Stewart 	return (error);
195cf70a46bSRandall Stewart }
196cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW,
197cf70a46bSRandall Stewart &nmbjumbop, 0, sysctl_nmbjumbop, "IU",
198ec63cb90SAndre Oppermann     "Maximum number of mbuf page size jumbo clusters allowed");
199cf70a46bSRandall Stewart 
200cf70a46bSRandall Stewart static int
201cf70a46bSRandall Stewart sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS)
202cf70a46bSRandall Stewart {
203cf70a46bSRandall Stewart 	int error, newnmbjumbo9;
204cf70a46bSRandall Stewart 
205cf70a46bSRandall Stewart 	newnmbjumbo9 = nmbjumbo9;
206cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req);
207d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) {
208ead46972SAndre Oppermann 		if (newnmbjumbo9 > nmbjumbo9 &&
209ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
210cf70a46bSRandall Stewart 			nmbjumbo9 = newnmbjumbo9;
211bc4a1b8cSAndre Oppermann 			nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
212cf70a46bSRandall Stewart 		} else
213cf70a46bSRandall Stewart 			error = EINVAL;
214cf70a46bSRandall Stewart 	}
215cf70a46bSRandall Stewart 	return (error);
216cf70a46bSRandall Stewart }
217cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW,
218cf70a46bSRandall Stewart &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU",
21956a4e45aSAndre Oppermann     "Maximum number of mbuf 9k jumbo clusters allowed");
220cf70a46bSRandall Stewart 
221cf70a46bSRandall Stewart static int
222cf70a46bSRandall Stewart sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS)
223cf70a46bSRandall Stewart {
224cf70a46bSRandall Stewart 	int error, newnmbjumbo16;
225cf70a46bSRandall Stewart 
226cf70a46bSRandall Stewart 	newnmbjumbo16 = nmbjumbo16;
227cf70a46bSRandall Stewart 	error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req);
228d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) {
229ead46972SAndre Oppermann 		if (newnmbjumbo16 > nmbjumbo16 &&
230ead46972SAndre Oppermann 		    nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) {
231cf70a46bSRandall Stewart 			nmbjumbo16 = newnmbjumbo16;
232bc4a1b8cSAndre Oppermann 			nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
233cf70a46bSRandall Stewart 		} else
234cf70a46bSRandall Stewart 			error = EINVAL;
235cf70a46bSRandall Stewart 	}
236cf70a46bSRandall Stewart 	return (error);
237cf70a46bSRandall Stewart }
238cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW,
239cf70a46bSRandall Stewart &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU",
24056a4e45aSAndre Oppermann     "Maximum number of mbuf 16k jumbo clusters allowed");
241cf70a46bSRandall Stewart 
242ead46972SAndre Oppermann static int
243ead46972SAndre Oppermann sysctl_nmbufs(SYSCTL_HANDLER_ARGS)
244ead46972SAndre Oppermann {
245ead46972SAndre Oppermann 	int error, newnmbufs;
246ead46972SAndre Oppermann 
247ead46972SAndre Oppermann 	newnmbufs = nmbufs;
248ead46972SAndre Oppermann 	error = sysctl_handle_int(oidp, &newnmbufs, 0, req);
249d251e700SJohn Baldwin 	if (error == 0 && req->newptr && newnmbufs != nmbufs) {
250ead46972SAndre Oppermann 		if (newnmbufs > nmbufs) {
251ead46972SAndre Oppermann 			nmbufs = newnmbufs;
252bc4a1b8cSAndre Oppermann 			nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
253ead46972SAndre Oppermann 			EVENTHANDLER_INVOKE(nmbufs_change);
254ead46972SAndre Oppermann 		} else
255ead46972SAndre Oppermann 			error = EINVAL;
256ead46972SAndre Oppermann 	}
257ead46972SAndre Oppermann 	return (error);
258ead46972SAndre Oppermann }
259e0c00addSAndre Oppermann SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW,
260ead46972SAndre Oppermann &nmbufs, 0, sysctl_nmbufs, "IU",
261ead46972SAndre Oppermann     "Maximum number of mbufs allowed");
262cf70a46bSRandall Stewart 
263099a0e58SBosko Milekic /*
264099a0e58SBosko Milekic  * Zones from which we allocate.
265099a0e58SBosko Milekic  */
266099a0e58SBosko Milekic uma_zone_t	zone_mbuf;
267099a0e58SBosko Milekic uma_zone_t	zone_clust;
268099a0e58SBosko Milekic uma_zone_t	zone_pack;
269ec63cb90SAndre Oppermann uma_zone_t	zone_jumbop;
27056a4e45aSAndre Oppermann uma_zone_t	zone_jumbo9;
27156a4e45aSAndre Oppermann uma_zone_t	zone_jumbo16;
272099a0e58SBosko Milekic 
273099a0e58SBosko Milekic /*
274099a0e58SBosko Milekic  * Local prototypes.
275099a0e58SBosko Milekic  */
276b23f72e9SBrian Feldman static int	mb_ctor_mbuf(void *, int, void *, int);
277b23f72e9SBrian Feldman static int	mb_ctor_clust(void *, int, void *, int);
278b23f72e9SBrian Feldman static int	mb_ctor_pack(void *, int, void *, int);
279099a0e58SBosko Milekic static void	mb_dtor_mbuf(void *, int, void *);
28056a4e45aSAndre Oppermann static void	mb_dtor_pack(void *, int, void *);
28156a4e45aSAndre Oppermann static int	mb_zinit_pack(void *, int, int);
28256a4e45aSAndre Oppermann static void	mb_zfini_pack(void *, int);
283e60b2fcbSGleb Smirnoff static void	mb_reclaim(uma_zone_t, int);
284f2c2231eSRyan Stone static void    *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
285099a0e58SBosko Milekic 
28637140716SAndre Oppermann /* Ensure that MSIZE is a power of 2. */
287a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
288a04946cfSBrian Somers 
289099a0e58SBosko Milekic /*
290099a0e58SBosko Milekic  * Initialize FreeBSD Network buffer allocation.
291099a0e58SBosko Milekic  */
292099a0e58SBosko Milekic static void
293099a0e58SBosko Milekic mbuf_init(void *dummy)
294099a0e58SBosko Milekic {
295099a0e58SBosko Milekic 
296099a0e58SBosko Milekic 	/*
297099a0e58SBosko Milekic 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
298099a0e58SBosko Milekic 	 */
29956a4e45aSAndre Oppermann 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
30056a4e45aSAndre Oppermann 	    mb_ctor_mbuf, mb_dtor_mbuf,
301121f0509SMike Silbersack #ifdef INVARIANTS
30256a4e45aSAndre Oppermann 	    trash_init, trash_fini,
303121f0509SMike Silbersack #else
30456a4e45aSAndre Oppermann 	    NULL, NULL,
305121f0509SMike Silbersack #endif
30656a4e45aSAndre Oppermann 	    MSIZE - 1, UMA_ZONE_MAXBUCKET);
30745fe0bf7SPawel Jakub Dawidek 	if (nmbufs > 0)
30845fe0bf7SPawel Jakub Dawidek 		nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
3096e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
310e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
31156a4e45aSAndre Oppermann 
31268352adfSRobert Watson 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
31356a5f52eSGleb Smirnoff 	    mb_ctor_clust,
314121f0509SMike Silbersack #ifdef INVARIANTS
31556a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
316121f0509SMike Silbersack #else
31756a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
318121f0509SMike Silbersack #endif
31956a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
32045fe0bf7SPawel Jakub Dawidek 	if (nmbclusters > 0)
32145fe0bf7SPawel Jakub Dawidek 		nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
3226e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
323e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_clust, mb_reclaim);
324099a0e58SBosko Milekic 
32556a4e45aSAndre Oppermann 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
32656a4e45aSAndre Oppermann 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
32756a4e45aSAndre Oppermann 
328fcf90618SGleb Smirnoff 	/* Make jumbo frame zone too. Page size, 9k and 16k. */
329ec63cb90SAndre Oppermann 	zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
33056a5f52eSGleb Smirnoff 	    mb_ctor_clust,
331d5269a63SAndre Oppermann #ifdef INVARIANTS
33256a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
333d5269a63SAndre Oppermann #else
33456a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
335d5269a63SAndre Oppermann #endif
33656a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
33745fe0bf7SPawel Jakub Dawidek 	if (nmbjumbop > 0)
33845fe0bf7SPawel Jakub Dawidek 		nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
3396e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
340e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
341d5269a63SAndre Oppermann 
34256a4e45aSAndre Oppermann 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
34356a5f52eSGleb Smirnoff 	    mb_ctor_clust,
34456a4e45aSAndre Oppermann #ifdef INVARIANTS
34556a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
34656a4e45aSAndre Oppermann #else
34756a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
34856a4e45aSAndre Oppermann #endif
34956a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
350ba63339aSAlan Cox 	uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc);
35145fe0bf7SPawel Jakub Dawidek 	if (nmbjumbo9 > 0)
35245fe0bf7SPawel Jakub Dawidek 		nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
3536e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
354e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
35556a4e45aSAndre Oppermann 
35656a4e45aSAndre Oppermann 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
35756a5f52eSGleb Smirnoff 	    mb_ctor_clust,
35856a4e45aSAndre Oppermann #ifdef INVARIANTS
35956a5f52eSGleb Smirnoff 	    trash_dtor, trash_init, trash_fini,
36056a4e45aSAndre Oppermann #else
36156a5f52eSGleb Smirnoff 	    NULL, NULL, NULL,
36256a4e45aSAndre Oppermann #endif
36356a5f52eSGleb Smirnoff 	    UMA_ALIGN_PTR, 0);
364ba63339aSAlan Cox 	uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc);
36545fe0bf7SPawel Jakub Dawidek 	if (nmbjumbo16 > 0)
36645fe0bf7SPawel Jakub Dawidek 		nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
3676e0b6746SPawel Jakub Dawidek 	uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
368e60b2fcbSGleb Smirnoff 	uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
36956a4e45aSAndre Oppermann 
370099a0e58SBosko Milekic 	/*
371099a0e58SBosko Milekic 	 * Hook event handler for low-memory situation, used to
372099a0e58SBosko Milekic 	 * drain protocols and push data back to the caches (UMA
373099a0e58SBosko Milekic 	 * later pushes it back to VM).
374099a0e58SBosko Milekic 	 */
375099a0e58SBosko Milekic 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
376099a0e58SBosko Milekic 	    EVENTHANDLER_PRI_FIRST);
377099a0e58SBosko Milekic }
37837140716SAndre Oppermann SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL);
379099a0e58SBosko Milekic 
380099a0e58SBosko Milekic /*
381ba63339aSAlan Cox  * UMA backend page allocator for the jumbo frame zones.
382ba63339aSAlan Cox  *
383ba63339aSAlan Cox  * Allocates kernel virtual memory that is backed by contiguous physical
384ba63339aSAlan Cox  * pages.
385ba63339aSAlan Cox  */
386ba63339aSAlan Cox static void *
387f2c2231eSRyan Stone mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
388ba63339aSAlan Cox {
389ba63339aSAlan Cox 
3907630c265SAlan Cox 	/* Inform UMA that this allocator uses kernel_map/object. */
3917630c265SAlan Cox 	*flags = UMA_SLAB_KERNEL;
3925df87b21SJeff Roberson 	return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait,
3933153e878SAlan Cox 	    (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT));
394ba63339aSAlan Cox }
395ba63339aSAlan Cox 
396ba63339aSAlan Cox /*
397099a0e58SBosko Milekic  * Constructor for Mbuf master zone.
398099a0e58SBosko Milekic  *
399099a0e58SBosko Milekic  * The 'arg' pointer points to a mb_args structure which
400099a0e58SBosko Milekic  * contains call-specific information required to support the
40156a4e45aSAndre Oppermann  * mbuf allocation API.  See mbuf.h.
402099a0e58SBosko Milekic  */
403b23f72e9SBrian Feldman static int
404b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how)
405099a0e58SBosko Milekic {
406099a0e58SBosko Milekic 	struct mbuf *m;
407099a0e58SBosko Milekic 	struct mb_args *args;
408b23f72e9SBrian Feldman 	int error;
409099a0e58SBosko Milekic 	int flags;
410099a0e58SBosko Milekic 	short type;
411099a0e58SBosko Milekic 
412121f0509SMike Silbersack #ifdef INVARIANTS
413121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
414121f0509SMike Silbersack #endif
415099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
416099a0e58SBosko Milekic 	type = args->type;
417099a0e58SBosko Milekic 
41856a4e45aSAndre Oppermann 	/*
41956a4e45aSAndre Oppermann 	 * The mbuf is initialized later.  The caller has the
420fcf90618SGleb Smirnoff 	 * responsibility to set up any MAC labels too.
42156a4e45aSAndre Oppermann 	 */
42256a4e45aSAndre Oppermann 	if (type == MT_NOINIT)
42356a4e45aSAndre Oppermann 		return (0);
42456a4e45aSAndre Oppermann 
425afb295ccSAndre Oppermann 	m = (struct mbuf *)mem;
426afb295ccSAndre Oppermann 	flags = args->flags;
427fddd4f62SNavdeep Parhar 	MPASS((flags & M_NOFREE) == 0);
428afb295ccSAndre Oppermann 
429b4b12e52SGleb Smirnoff 	error = m_init(m, how, type, flags);
430afb295ccSAndre Oppermann 
431b23f72e9SBrian Feldman 	return (error);
432099a0e58SBosko Milekic }
433099a0e58SBosko Milekic 
434099a0e58SBosko Milekic /*
43556a4e45aSAndre Oppermann  * The Mbuf master zone destructor.
436099a0e58SBosko Milekic  */
437099a0e58SBosko Milekic static void
438099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg)
439099a0e58SBosko Milekic {
440099a0e58SBosko Milekic 	struct mbuf *m;
441629b9e08SKip Macy 	unsigned long flags;
442099a0e58SBosko Milekic 
443099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
444629b9e08SKip Macy 	flags = (unsigned long)arg;
445629b9e08SKip Macy 
446a9fa76f2SNavdeep Parhar 	KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__));
4474c7070dbSScott Long 	if (!(flags & MB_DTOR_SKIP) && (m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags))
448099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
449121f0509SMike Silbersack #ifdef INVARIANTS
450121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
451121f0509SMike Silbersack #endif
452099a0e58SBosko Milekic }
453099a0e58SBosko Milekic 
45456a4e45aSAndre Oppermann /*
45556a4e45aSAndre Oppermann  * The Mbuf Packet zone destructor.
45656a4e45aSAndre Oppermann  */
457099a0e58SBosko Milekic static void
458099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg)
459099a0e58SBosko Milekic {
460099a0e58SBosko Milekic 	struct mbuf *m;
461099a0e58SBosko Milekic 
462099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
463099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
464099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
46556a4e45aSAndre Oppermann 
46656a4e45aSAndre Oppermann 	/* Make sure we've got a clean cluster back. */
46756a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
46856a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
46956a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
470cf827063SPoul-Henning Kamp 	KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__));
471cf827063SPoul-Henning Kamp 	KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__));
47256a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
47349d46b61SGleb Smirnoff 	KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
474121f0509SMike Silbersack #ifdef INVARIANTS
475121f0509SMike Silbersack 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
476121f0509SMike Silbersack #endif
4776c125b8dSMohan Srinivasan 	/*
478ef44c8d2SDavid E. O'Brien 	 * If there are processes blocked on zone_clust, waiting for pages
479ef44c8d2SDavid E. O'Brien 	 * to be freed up, * cause them to be woken up by draining the
480ef44c8d2SDavid E. O'Brien 	 * packet zone.  We are exposed to a race here * (in the check for
481ef44c8d2SDavid E. O'Brien 	 * the UMA_ZFLAG_FULL) where we might miss the flag set, but that
482ef44c8d2SDavid E. O'Brien 	 * is deliberate. We don't want to acquire the zone lock for every
483ef44c8d2SDavid E. O'Brien 	 * mbuf free.
4846c125b8dSMohan Srinivasan 	 */
4856c125b8dSMohan Srinivasan 	if (uma_zone_exhausted_nolock(zone_clust))
4866c125b8dSMohan Srinivasan 		zone_drain(zone_pack);
487099a0e58SBosko Milekic }
488099a0e58SBosko Milekic 
489099a0e58SBosko Milekic /*
490ec63cb90SAndre Oppermann  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
491099a0e58SBosko Milekic  *
492099a0e58SBosko Milekic  * Here the 'arg' pointer points to the Mbuf which we
49356a4e45aSAndre Oppermann  * are configuring cluster storage for.  If 'arg' is
49456a4e45aSAndre Oppermann  * empty we allocate just the cluster without setting
49556a4e45aSAndre Oppermann  * the mbuf to it.  See mbuf.h.
496099a0e58SBosko Milekic  */
497b23f72e9SBrian Feldman static int
498b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how)
499099a0e58SBosko Milekic {
500099a0e58SBosko Milekic 	struct mbuf *m;
501099a0e58SBosko Milekic 
502121f0509SMike Silbersack #ifdef INVARIANTS
503121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
504121f0509SMike Silbersack #endif
5050f4d9d04SKip Macy 	m = (struct mbuf *)arg;
5060f4d9d04SKip Macy 	if (m != NULL) {
507*e8fd18f3SGleb Smirnoff 		m->m_ext.ext_buf = (char *)mem;
508099a0e58SBosko Milekic 		m->m_data = m->m_ext.ext_buf;
509099a0e58SBosko Milekic 		m->m_flags |= M_EXT;
510099a0e58SBosko Milekic 		m->m_ext.ext_free = NULL;
511cf827063SPoul-Henning Kamp 		m->m_ext.ext_arg1 = NULL;
512cf827063SPoul-Henning Kamp 		m->m_ext.ext_arg2 = NULL;
51356a4e45aSAndre Oppermann 		m->m_ext.ext_size = size;
51456a5f52eSGleb Smirnoff 		m->m_ext.ext_type = m_gettype(size);
51556a5f52eSGleb Smirnoff 		m->m_ext.ext_flags = EXT_FLAG_EMBREF;
51656a5f52eSGleb Smirnoff 		m->m_ext.ext_count = 1;
51756a4e45aSAndre Oppermann 	}
5180f4d9d04SKip Macy 
519b23f72e9SBrian Feldman 	return (0);
520099a0e58SBosko Milekic }
521099a0e58SBosko Milekic 
52256a4e45aSAndre Oppermann /*
523099a0e58SBosko Milekic  * The Packet secondary zone's init routine, executed on the
52456a4e45aSAndre Oppermann  * object's transition from mbuf keg slab to zone cache.
525099a0e58SBosko Milekic  */
526b23f72e9SBrian Feldman static int
52756a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how)
528099a0e58SBosko Milekic {
529099a0e58SBosko Milekic 	struct mbuf *m;
530099a0e58SBosko Milekic 
53156a4e45aSAndre Oppermann 	m = (struct mbuf *)mem;		/* m is virgin. */
532a7bd90efSAndre Oppermann 	if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
533a7bd90efSAndre Oppermann 	    m->m_ext.ext_buf == NULL)
534b23f72e9SBrian Feldman 		return (ENOMEM);
535cd5bb63bSAndre Oppermann 	m->m_ext.ext_type = EXT_PACKET;	/* Override. */
536121f0509SMike Silbersack #ifdef INVARIANTS
537121f0509SMike Silbersack 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
538121f0509SMike Silbersack #endif
539b23f72e9SBrian Feldman 	return (0);
540099a0e58SBosko Milekic }
541099a0e58SBosko Milekic 
542099a0e58SBosko Milekic /*
543099a0e58SBosko Milekic  * The Packet secondary zone's fini routine, executed on the
544099a0e58SBosko Milekic  * object's transition from zone cache to keg slab.
545099a0e58SBosko Milekic  */
546099a0e58SBosko Milekic static void
54756a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size)
548099a0e58SBosko Milekic {
549099a0e58SBosko Milekic 	struct mbuf *m;
550099a0e58SBosko Milekic 
551099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
552121f0509SMike Silbersack #ifdef INVARIANTS
553121f0509SMike Silbersack 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
554121f0509SMike Silbersack #endif
555099a0e58SBosko Milekic 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
556a7b844d2SMike Silbersack #ifdef INVARIANTS
557a7b844d2SMike Silbersack 	trash_dtor(mem, size, NULL);
558a7b844d2SMike Silbersack #endif
559099a0e58SBosko Milekic }
560099a0e58SBosko Milekic 
561099a0e58SBosko Milekic /*
562099a0e58SBosko Milekic  * The "packet" keg constructor.
563099a0e58SBosko Milekic  */
564b23f72e9SBrian Feldman static int
565b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how)
566099a0e58SBosko Milekic {
567099a0e58SBosko Milekic 	struct mbuf *m;
568099a0e58SBosko Milekic 	struct mb_args *args;
569ce28636bSAndre Oppermann 	int error, flags;
570099a0e58SBosko Milekic 	short type;
571099a0e58SBosko Milekic 
572099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
573099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
574099a0e58SBosko Milekic 	flags = args->flags;
575099a0e58SBosko Milekic 	type = args->type;
576fddd4f62SNavdeep Parhar 	MPASS((flags & M_NOFREE) == 0);
577099a0e58SBosko Milekic 
578121f0509SMike Silbersack #ifdef INVARIANTS
579121f0509SMike Silbersack 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
580121f0509SMike Silbersack #endif
581099a0e58SBosko Milekic 
582b4b12e52SGleb Smirnoff 	error = m_init(m, how, type, flags);
583afb295ccSAndre Oppermann 
58456a4e45aSAndre Oppermann 	/* m_ext is already initialized. */
585afb295ccSAndre Oppermann 	m->m_data = m->m_ext.ext_buf;
586afb295ccSAndre Oppermann  	m->m_flags = (flags | M_EXT);
58756a4e45aSAndre Oppermann 
588afb295ccSAndre Oppermann 	return (error);
589099a0e58SBosko Milekic }
590099a0e58SBosko Milekic 
591099a0e58SBosko Milekic /*
592e60b2fcbSGleb Smirnoff  * This is the protocol drain routine.  Called by UMA whenever any of the
593e60b2fcbSGleb Smirnoff  * mbuf zones is closed to its limit.
594099a0e58SBosko Milekic  *
595099a0e58SBosko Milekic  * No locks should be held when this is called.  The drain routines have to
596099a0e58SBosko Milekic  * presently acquire some locks which raises the possibility of lock order
597099a0e58SBosko Milekic  * reversal.
598099a0e58SBosko Milekic  */
599099a0e58SBosko Milekic static void
600e60b2fcbSGleb Smirnoff mb_reclaim(uma_zone_t zone __unused, int pending __unused)
601099a0e58SBosko Milekic {
602099a0e58SBosko Milekic 	struct domain *dp;
603099a0e58SBosko Milekic 	struct protosw *pr;
604099a0e58SBosko Milekic 
605e60b2fcbSGleb Smirnoff 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
606099a0e58SBosko Milekic 
607099a0e58SBosko Milekic 	for (dp = domains; dp != NULL; dp = dp->dom_next)
608099a0e58SBosko Milekic 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
609099a0e58SBosko Milekic 			if (pr->pr_drain != NULL)
610099a0e58SBosko Milekic 				(*pr->pr_drain)();
611099a0e58SBosko Milekic }
6125e4bc63bSGleb Smirnoff 
6135e4bc63bSGleb Smirnoff /*
6145e4bc63bSGleb Smirnoff  * Clean up after mbufs with M_EXT storage attached to them if the
6155e4bc63bSGleb Smirnoff  * reference count hits 1.
6165e4bc63bSGleb Smirnoff  */
6175e4bc63bSGleb Smirnoff void
6185e4bc63bSGleb Smirnoff mb_free_ext(struct mbuf *m)
6195e4bc63bSGleb Smirnoff {
62056a5f52eSGleb Smirnoff 	volatile u_int *refcnt;
62156a5f52eSGleb Smirnoff 	struct mbuf *mref;
6225e4bc63bSGleb Smirnoff 	int freembuf;
6235e4bc63bSGleb Smirnoff 
6245e4bc63bSGleb Smirnoff 	KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m));
6255e4bc63bSGleb Smirnoff 
62656a5f52eSGleb Smirnoff 	/* See if this is the mbuf that holds the embedded refcount. */
62756a5f52eSGleb Smirnoff 	if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) {
62856a5f52eSGleb Smirnoff 		refcnt = &m->m_ext.ext_count;
62956a5f52eSGleb Smirnoff 		mref = m;
63056a5f52eSGleb Smirnoff 	} else {
63156a5f52eSGleb Smirnoff 		KASSERT(m->m_ext.ext_cnt != NULL,
63256a5f52eSGleb Smirnoff 		    ("%s: no refcounting pointer on %p", __func__, m));
63356a5f52eSGleb Smirnoff 		refcnt = m->m_ext.ext_cnt;
63456a5f52eSGleb Smirnoff 		mref = __containerof(refcnt, struct mbuf, m_ext.ext_count);
63556a5f52eSGleb Smirnoff 	}
63656a5f52eSGleb Smirnoff 
6375e4bc63bSGleb Smirnoff 	/*
63856a5f52eSGleb Smirnoff 	 * Check if the header is embedded in the cluster.  It is
63956a5f52eSGleb Smirnoff 	 * important that we can't touch any of the mbuf fields
64056a5f52eSGleb Smirnoff 	 * after we have freed the external storage, since mbuf
64117cd649fSGleb Smirnoff 	 * could have been embedded in it.  For now, the mbufs
64217cd649fSGleb Smirnoff 	 * embedded into the cluster are always of type EXT_EXTREF,
64317cd649fSGleb Smirnoff 	 * and for this type we won't free the mref.
6445e4bc63bSGleb Smirnoff 	 */
64517cd649fSGleb Smirnoff 	if (m->m_flags & M_NOFREE) {
64617cd649fSGleb Smirnoff 		freembuf = 0;
64717cd649fSGleb Smirnoff 		KASSERT(m->m_ext.ext_type == EXT_EXTREF,
64817cd649fSGleb Smirnoff 		    ("%s: no-free mbuf %p has wrong type", __func__, m));
64917cd649fSGleb Smirnoff 	} else
65017cd649fSGleb Smirnoff 		freembuf = 1;
6515e4bc63bSGleb Smirnoff 
65256a5f52eSGleb Smirnoff 	/* Free attached storage if this mbuf is the only reference to it. */
65356a5f52eSGleb Smirnoff 	if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) {
6545e4bc63bSGleb Smirnoff 		switch (m->m_ext.ext_type) {
65556a5f52eSGleb Smirnoff 		case EXT_PACKET:
65656a5f52eSGleb Smirnoff 			/* The packet zone is special. */
65756a5f52eSGleb Smirnoff 			if (*refcnt == 0)
65856a5f52eSGleb Smirnoff 				*refcnt = 1;
65956a5f52eSGleb Smirnoff 			uma_zfree(zone_pack, mref);
6605e4bc63bSGleb Smirnoff 			break;
6615e4bc63bSGleb Smirnoff 		case EXT_CLUSTER:
6625e4bc63bSGleb Smirnoff 			uma_zfree(zone_clust, m->m_ext.ext_buf);
66356a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
6645e4bc63bSGleb Smirnoff 			break;
6655e4bc63bSGleb Smirnoff 		case EXT_JUMBOP:
6665e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbop, m->m_ext.ext_buf);
66756a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
6685e4bc63bSGleb Smirnoff 			break;
6695e4bc63bSGleb Smirnoff 		case EXT_JUMBO9:
6705e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbo9, m->m_ext.ext_buf);
67156a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
6725e4bc63bSGleb Smirnoff 			break;
6735e4bc63bSGleb Smirnoff 		case EXT_JUMBO16:
6745e4bc63bSGleb Smirnoff 			uma_zfree(zone_jumbo16, m->m_ext.ext_buf);
67556a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
67656a5f52eSGleb Smirnoff 			break;
67756a5f52eSGleb Smirnoff 		case EXT_SFBUF:
67856a5f52eSGleb Smirnoff 			sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2);
67956a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
68056a5f52eSGleb Smirnoff 			break;
68156a5f52eSGleb Smirnoff 		case EXT_SFBUF_NOCACHE:
68256a5f52eSGleb Smirnoff 			sf_ext_free_nocache(m->m_ext.ext_arg1,
68356a5f52eSGleb Smirnoff 			    m->m_ext.ext_arg2);
68456a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
6855e4bc63bSGleb Smirnoff 			break;
6865e4bc63bSGleb Smirnoff 		case EXT_NET_DRV:
6875e4bc63bSGleb Smirnoff 		case EXT_MOD_TYPE:
6885e4bc63bSGleb Smirnoff 		case EXT_DISPOSABLE:
6890ea37a86SGleb Smirnoff 			KASSERT(m->m_ext.ext_free != NULL,
6900ea37a86SGleb Smirnoff 				("%s: ext_free not set", __func__));
691*e8fd18f3SGleb Smirnoff 			m->m_ext.ext_free(m);
69256a5f52eSGleb Smirnoff 			uma_zfree(zone_mbuf, mref);
6930ea37a86SGleb Smirnoff 			break;
6945e4bc63bSGleb Smirnoff 		case EXT_EXTREF:
6955e4bc63bSGleb Smirnoff 			KASSERT(m->m_ext.ext_free != NULL,
6965e4bc63bSGleb Smirnoff 				("%s: ext_free not set", __func__));
697*e8fd18f3SGleb Smirnoff 			m->m_ext.ext_free(m);
6985e4bc63bSGleb Smirnoff 			break;
6995e4bc63bSGleb Smirnoff 		default:
7005e4bc63bSGleb Smirnoff 			KASSERT(m->m_ext.ext_type == 0,
7015e4bc63bSGleb Smirnoff 				("%s: unknown ext_type", __func__));
7025e4bc63bSGleb Smirnoff 		}
7035e4bc63bSGleb Smirnoff 	}
7045e4bc63bSGleb Smirnoff 
70556a5f52eSGleb Smirnoff 	if (freembuf && m != mref)
7065e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
7075e4bc63bSGleb Smirnoff }
7085e4bc63bSGleb Smirnoff 
7095e4bc63bSGleb Smirnoff /*
7105e4bc63bSGleb Smirnoff  * Official mbuf(9) allocation KPI for stack and drivers:
7115e4bc63bSGleb Smirnoff  *
7125e4bc63bSGleb Smirnoff  * m_get()	- a single mbuf without any attachments, sys/mbuf.h.
7135e4bc63bSGleb Smirnoff  * m_gethdr()	- a single mbuf initialized as M_PKTHDR, sys/mbuf.h.
7145e4bc63bSGleb Smirnoff  * m_getcl()	- an mbuf + 2k cluster, sys/mbuf.h.
7155e4bc63bSGleb Smirnoff  * m_clget()	- attach cluster to already allocated mbuf.
7165e4bc63bSGleb Smirnoff  * m_cljget()	- attach jumbo cluster to already allocated mbuf.
7175e4bc63bSGleb Smirnoff  * m_get2()	- allocate minimum mbuf that would fit size argument.
7185e4bc63bSGleb Smirnoff  * m_getm2()	- allocate a chain of mbufs/clusters.
7195e4bc63bSGleb Smirnoff  * m_extadd()	- attach external cluster to mbuf.
7205e4bc63bSGleb Smirnoff  *
7215e4bc63bSGleb Smirnoff  * m_free()	- free single mbuf with its tags and ext, sys/mbuf.h.
7225e4bc63bSGleb Smirnoff  * m_freem()	- free chain of mbufs.
7235e4bc63bSGleb Smirnoff  */
7245e4bc63bSGleb Smirnoff 
7255e4bc63bSGleb Smirnoff int
7265e4bc63bSGleb Smirnoff m_clget(struct mbuf *m, int how)
7275e4bc63bSGleb Smirnoff {
7285e4bc63bSGleb Smirnoff 
7295e4bc63bSGleb Smirnoff 	KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
7305e4bc63bSGleb Smirnoff 	    __func__, m));
7315e4bc63bSGleb Smirnoff 	m->m_ext.ext_buf = (char *)NULL;
7325e4bc63bSGleb Smirnoff 	uma_zalloc_arg(zone_clust, m, how);
7335e4bc63bSGleb Smirnoff 	/*
7345e4bc63bSGleb Smirnoff 	 * On a cluster allocation failure, drain the packet zone and retry,
7355e4bc63bSGleb Smirnoff 	 * we might be able to loosen a few clusters up on the drain.
7365e4bc63bSGleb Smirnoff 	 */
7375e4bc63bSGleb Smirnoff 	if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) {
7385e4bc63bSGleb Smirnoff 		zone_drain(zone_pack);
7395e4bc63bSGleb Smirnoff 		uma_zalloc_arg(zone_clust, m, how);
7405e4bc63bSGleb Smirnoff 	}
741480f4e94SGeorge V. Neville-Neil 	MBUF_PROBE2(m__clget, m, how);
7425e4bc63bSGleb Smirnoff 	return (m->m_flags & M_EXT);
7435e4bc63bSGleb Smirnoff }
7445e4bc63bSGleb Smirnoff 
7455e4bc63bSGleb Smirnoff /*
7465e4bc63bSGleb Smirnoff  * m_cljget() is different from m_clget() as it can allocate clusters without
7475e4bc63bSGleb Smirnoff  * attaching them to an mbuf.  In that case the return value is the pointer
7485e4bc63bSGleb Smirnoff  * to the cluster of the requested size.  If an mbuf was specified, it gets
7495e4bc63bSGleb Smirnoff  * the cluster attached to it and the return value can be safely ignored.
7505e4bc63bSGleb Smirnoff  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
7515e4bc63bSGleb Smirnoff  */
7525e4bc63bSGleb Smirnoff void *
7535e4bc63bSGleb Smirnoff m_cljget(struct mbuf *m, int how, int size)
7545e4bc63bSGleb Smirnoff {
7555e4bc63bSGleb Smirnoff 	uma_zone_t zone;
756480f4e94SGeorge V. Neville-Neil 	void *retval;
7575e4bc63bSGleb Smirnoff 
7585e4bc63bSGleb Smirnoff 	if (m != NULL) {
7595e4bc63bSGleb Smirnoff 		KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT",
7605e4bc63bSGleb Smirnoff 		    __func__, m));
7615e4bc63bSGleb Smirnoff 		m->m_ext.ext_buf = NULL;
7625e4bc63bSGleb Smirnoff 	}
7635e4bc63bSGleb Smirnoff 
7645e4bc63bSGleb Smirnoff 	zone = m_getzone(size);
765480f4e94SGeorge V. Neville-Neil 	retval = uma_zalloc_arg(zone, m, how);
766480f4e94SGeorge V. Neville-Neil 
767480f4e94SGeorge V. Neville-Neil 	MBUF_PROBE4(m__cljget, m, how, size, retval);
768480f4e94SGeorge V. Neville-Neil 
769480f4e94SGeorge V. Neville-Neil 	return (retval);
7705e4bc63bSGleb Smirnoff }
7715e4bc63bSGleb Smirnoff 
7725e4bc63bSGleb Smirnoff /*
7735e4bc63bSGleb Smirnoff  * m_get2() allocates minimum mbuf that would fit "size" argument.
7745e4bc63bSGleb Smirnoff  */
7755e4bc63bSGleb Smirnoff struct mbuf *
7765e4bc63bSGleb Smirnoff m_get2(int size, int how, short type, int flags)
7775e4bc63bSGleb Smirnoff {
7785e4bc63bSGleb Smirnoff 	struct mb_args args;
7795e4bc63bSGleb Smirnoff 	struct mbuf *m, *n;
7805e4bc63bSGleb Smirnoff 
7815e4bc63bSGleb Smirnoff 	args.flags = flags;
7825e4bc63bSGleb Smirnoff 	args.type = type;
7835e4bc63bSGleb Smirnoff 
7845e4bc63bSGleb Smirnoff 	if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0))
7855e4bc63bSGleb Smirnoff 		return (uma_zalloc_arg(zone_mbuf, &args, how));
7865e4bc63bSGleb Smirnoff 	if (size <= MCLBYTES)
7875e4bc63bSGleb Smirnoff 		return (uma_zalloc_arg(zone_pack, &args, how));
7885e4bc63bSGleb Smirnoff 
7895e4bc63bSGleb Smirnoff 	if (size > MJUMPAGESIZE)
7905e4bc63bSGleb Smirnoff 		return (NULL);
7915e4bc63bSGleb Smirnoff 
7925e4bc63bSGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
7935e4bc63bSGleb Smirnoff 	if (m == NULL)
7945e4bc63bSGleb Smirnoff 		return (NULL);
7955e4bc63bSGleb Smirnoff 
7965e4bc63bSGleb Smirnoff 	n = uma_zalloc_arg(zone_jumbop, m, how);
7975e4bc63bSGleb Smirnoff 	if (n == NULL) {
7985e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
7995e4bc63bSGleb Smirnoff 		return (NULL);
8005e4bc63bSGleb Smirnoff 	}
8015e4bc63bSGleb Smirnoff 
8025e4bc63bSGleb Smirnoff 	return (m);
8035e4bc63bSGleb Smirnoff }
8045e4bc63bSGleb Smirnoff 
8055e4bc63bSGleb Smirnoff /*
8065e4bc63bSGleb Smirnoff  * m_getjcl() returns an mbuf with a cluster of the specified size attached.
8075e4bc63bSGleb Smirnoff  * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES.
8085e4bc63bSGleb Smirnoff  */
8095e4bc63bSGleb Smirnoff struct mbuf *
8105e4bc63bSGleb Smirnoff m_getjcl(int how, short type, int flags, int size)
8115e4bc63bSGleb Smirnoff {
8125e4bc63bSGleb Smirnoff 	struct mb_args args;
8135e4bc63bSGleb Smirnoff 	struct mbuf *m, *n;
8145e4bc63bSGleb Smirnoff 	uma_zone_t zone;
8155e4bc63bSGleb Smirnoff 
8165e4bc63bSGleb Smirnoff 	if (size == MCLBYTES)
8175e4bc63bSGleb Smirnoff 		return m_getcl(how, type, flags);
8185e4bc63bSGleb Smirnoff 
8195e4bc63bSGleb Smirnoff 	args.flags = flags;
8205e4bc63bSGleb Smirnoff 	args.type = type;
8215e4bc63bSGleb Smirnoff 
8225e4bc63bSGleb Smirnoff 	m = uma_zalloc_arg(zone_mbuf, &args, how);
8235e4bc63bSGleb Smirnoff 	if (m == NULL)
8245e4bc63bSGleb Smirnoff 		return (NULL);
8255e4bc63bSGleb Smirnoff 
8265e4bc63bSGleb Smirnoff 	zone = m_getzone(size);
8275e4bc63bSGleb Smirnoff 	n = uma_zalloc_arg(zone, m, how);
8285e4bc63bSGleb Smirnoff 	if (n == NULL) {
8295e4bc63bSGleb Smirnoff 		uma_zfree(zone_mbuf, m);
8305e4bc63bSGleb Smirnoff 		return (NULL);
8315e4bc63bSGleb Smirnoff 	}
8325e4bc63bSGleb Smirnoff 	return (m);
8335e4bc63bSGleb Smirnoff }
8345e4bc63bSGleb Smirnoff 
8355e4bc63bSGleb Smirnoff /*
8365e4bc63bSGleb Smirnoff  * Allocate a given length worth of mbufs and/or clusters (whatever fits
8375e4bc63bSGleb Smirnoff  * best) and return a pointer to the top of the allocated chain.  If an
8385e4bc63bSGleb Smirnoff  * existing mbuf chain is provided, then we will append the new chain
8395e4bc63bSGleb Smirnoff  * to the existing one but still return the top of the newly allocated
8405e4bc63bSGleb Smirnoff  * chain.
8415e4bc63bSGleb Smirnoff  */
8425e4bc63bSGleb Smirnoff struct mbuf *
8435e4bc63bSGleb Smirnoff m_getm2(struct mbuf *m, int len, int how, short type, int flags)
8445e4bc63bSGleb Smirnoff {
8455e4bc63bSGleb Smirnoff 	struct mbuf *mb, *nm = NULL, *mtail = NULL;
8465e4bc63bSGleb Smirnoff 
8475e4bc63bSGleb Smirnoff 	KASSERT(len >= 0, ("%s: len is < 0", __func__));
8485e4bc63bSGleb Smirnoff 
8495e4bc63bSGleb Smirnoff 	/* Validate flags. */
8505e4bc63bSGleb Smirnoff 	flags &= (M_PKTHDR | M_EOR);
8515e4bc63bSGleb Smirnoff 
8525e4bc63bSGleb Smirnoff 	/* Packet header mbuf must be first in chain. */
8535e4bc63bSGleb Smirnoff 	if ((flags & M_PKTHDR) && m != NULL)
8545e4bc63bSGleb Smirnoff 		flags &= ~M_PKTHDR;
8555e4bc63bSGleb Smirnoff 
8565e4bc63bSGleb Smirnoff 	/* Loop and append maximum sized mbufs to the chain tail. */
8575e4bc63bSGleb Smirnoff 	while (len > 0) {
8585e4bc63bSGleb Smirnoff 		if (len > MCLBYTES)
8595e4bc63bSGleb Smirnoff 			mb = m_getjcl(how, type, (flags & M_PKTHDR),
8605e4bc63bSGleb Smirnoff 			    MJUMPAGESIZE);
8615e4bc63bSGleb Smirnoff 		else if (len >= MINCLSIZE)
8625e4bc63bSGleb Smirnoff 			mb = m_getcl(how, type, (flags & M_PKTHDR));
8635e4bc63bSGleb Smirnoff 		else if (flags & M_PKTHDR)
8645e4bc63bSGleb Smirnoff 			mb = m_gethdr(how, type);
8655e4bc63bSGleb Smirnoff 		else
8665e4bc63bSGleb Smirnoff 			mb = m_get(how, type);
8675e4bc63bSGleb Smirnoff 
8685e4bc63bSGleb Smirnoff 		/* Fail the whole operation if one mbuf can't be allocated. */
8695e4bc63bSGleb Smirnoff 		if (mb == NULL) {
8705e4bc63bSGleb Smirnoff 			if (nm != NULL)
8715e4bc63bSGleb Smirnoff 				m_freem(nm);
8725e4bc63bSGleb Smirnoff 			return (NULL);
8735e4bc63bSGleb Smirnoff 		}
8745e4bc63bSGleb Smirnoff 
8755e4bc63bSGleb Smirnoff 		/* Book keeping. */
8765e4bc63bSGleb Smirnoff 		len -= M_SIZE(mb);
8775e4bc63bSGleb Smirnoff 		if (mtail != NULL)
8785e4bc63bSGleb Smirnoff 			mtail->m_next = mb;
8795e4bc63bSGleb Smirnoff 		else
8805e4bc63bSGleb Smirnoff 			nm = mb;
8815e4bc63bSGleb Smirnoff 		mtail = mb;
8825e4bc63bSGleb Smirnoff 		flags &= ~M_PKTHDR;	/* Only valid on the first mbuf. */
8835e4bc63bSGleb Smirnoff 	}
8845e4bc63bSGleb Smirnoff 	if (flags & M_EOR)
8855e4bc63bSGleb Smirnoff 		mtail->m_flags |= M_EOR;  /* Only valid on the last mbuf. */
8865e4bc63bSGleb Smirnoff 
8875e4bc63bSGleb Smirnoff 	/* If mbuf was supplied, append new chain to the end of it. */
8885e4bc63bSGleb Smirnoff 	if (m != NULL) {
8895e4bc63bSGleb Smirnoff 		for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next)
8905e4bc63bSGleb Smirnoff 			;
8915e4bc63bSGleb Smirnoff 		mtail->m_next = nm;
8925e4bc63bSGleb Smirnoff 		mtail->m_flags &= ~M_EOR;
8935e4bc63bSGleb Smirnoff 	} else
8945e4bc63bSGleb Smirnoff 		m = nm;
8955e4bc63bSGleb Smirnoff 
8965e4bc63bSGleb Smirnoff 	return (m);
8975e4bc63bSGleb Smirnoff }
8985e4bc63bSGleb Smirnoff 
8995e4bc63bSGleb Smirnoff /*-
9005e4bc63bSGleb Smirnoff  * Configure a provided mbuf to refer to the provided external storage
90156a5f52eSGleb Smirnoff  * buffer and setup a reference count for said buffer.
9025e4bc63bSGleb Smirnoff  *
9035e4bc63bSGleb Smirnoff  * Arguments:
9045e4bc63bSGleb Smirnoff  *    mb     The existing mbuf to which to attach the provided buffer.
9055e4bc63bSGleb Smirnoff  *    buf    The address of the provided external storage buffer.
9065e4bc63bSGleb Smirnoff  *    size   The size of the provided buffer.
9075e4bc63bSGleb Smirnoff  *    freef  A pointer to a routine that is responsible for freeing the
9085e4bc63bSGleb Smirnoff  *           provided external storage buffer.
9095e4bc63bSGleb Smirnoff  *    args   A pointer to an argument structure (of any type) to be passed
9105e4bc63bSGleb Smirnoff  *           to the provided freef routine (may be NULL).
9115e4bc63bSGleb Smirnoff  *    flags  Any other flags to be passed to the provided mbuf.
9125e4bc63bSGleb Smirnoff  *    type   The type that the external storage buffer should be
9135e4bc63bSGleb Smirnoff  *           labeled with.
9145e4bc63bSGleb Smirnoff  *
9155e4bc63bSGleb Smirnoff  * Returns:
9165e4bc63bSGleb Smirnoff  *    Nothing.
9175e4bc63bSGleb Smirnoff  */
91856a5f52eSGleb Smirnoff void
919*e8fd18f3SGleb Smirnoff m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef,
920*e8fd18f3SGleb Smirnoff     void *arg1, void *arg2, int flags, int type)
9215e4bc63bSGleb Smirnoff {
92256a5f52eSGleb Smirnoff 
9235e4bc63bSGleb Smirnoff 	KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__));
9245e4bc63bSGleb Smirnoff 
9255e4bc63bSGleb Smirnoff 	mb->m_flags |= (M_EXT | flags);
9265e4bc63bSGleb Smirnoff 	mb->m_ext.ext_buf = buf;
9275e4bc63bSGleb Smirnoff 	mb->m_data = mb->m_ext.ext_buf;
9285e4bc63bSGleb Smirnoff 	mb->m_ext.ext_size = size;
9295e4bc63bSGleb Smirnoff 	mb->m_ext.ext_free = freef;
9305e4bc63bSGleb Smirnoff 	mb->m_ext.ext_arg1 = arg1;
9315e4bc63bSGleb Smirnoff 	mb->m_ext.ext_arg2 = arg2;
9325e4bc63bSGleb Smirnoff 	mb->m_ext.ext_type = type;
9335e4bc63bSGleb Smirnoff 
93456a5f52eSGleb Smirnoff 	if (type != EXT_EXTREF) {
93556a5f52eSGleb Smirnoff 		mb->m_ext.ext_count = 1;
93656a5f52eSGleb Smirnoff 		mb->m_ext.ext_flags = EXT_FLAG_EMBREF;
93756a5f52eSGleb Smirnoff 	} else
93856a5f52eSGleb Smirnoff 		mb->m_ext.ext_flags = 0;
9395e4bc63bSGleb Smirnoff }
9405e4bc63bSGleb Smirnoff 
9415e4bc63bSGleb Smirnoff /*
9425e4bc63bSGleb Smirnoff  * Free an entire chain of mbufs and associated external buffers, if
9435e4bc63bSGleb Smirnoff  * applicable.
9445e4bc63bSGleb Smirnoff  */
9455e4bc63bSGleb Smirnoff void
9465e4bc63bSGleb Smirnoff m_freem(struct mbuf *mb)
9475e4bc63bSGleb Smirnoff {
9485e4bc63bSGleb Smirnoff 
949c8f59118SGleb Smirnoff 	MBUF_PROBE1(m__freem, mb);
9505e4bc63bSGleb Smirnoff 	while (mb != NULL)
9515e4bc63bSGleb Smirnoff 		mb = m_free(mb);
9525e4bc63bSGleb Smirnoff }
953