xref: /freebsd/sys/kern/kern_mbuf.c (revision 6c125b8df6750dba0aa916ee7779fe5b20bf63b8)
1099a0e58SBosko Milekic /*-
28076cb52SBosko Milekic  * Copyright (c) 2004, 2005,
38076cb52SBosko Milekic  * 	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
4099a0e58SBosko Milekic  *
5099a0e58SBosko Milekic  * Redistribution and use in source and binary forms, with or without
6099a0e58SBosko Milekic  * modification, are permitted provided that the following conditions
7099a0e58SBosko Milekic  * are met:
8099a0e58SBosko Milekic  * 1. Redistributions of source code must retain the above copyright
9099a0e58SBosko Milekic  *    notice unmodified, this list of conditions and the following
10099a0e58SBosko Milekic  *    disclaimer.
11099a0e58SBosko Milekic  * 2. Redistributions in binary form must reproduce the above copyright
12099a0e58SBosko Milekic  *    notice, this list of conditions and the following disclaimer in the
13099a0e58SBosko Milekic  *    documentation and/or other materials provided with the distribution.
14099a0e58SBosko Milekic  *
15099a0e58SBosko Milekic  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16099a0e58SBosko Milekic  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17099a0e58SBosko Milekic  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18099a0e58SBosko Milekic  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19099a0e58SBosko Milekic  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20099a0e58SBosko Milekic  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21099a0e58SBosko Milekic  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22099a0e58SBosko Milekic  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23099a0e58SBosko Milekic  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24099a0e58SBosko Milekic  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25099a0e58SBosko Milekic  * SUCH DAMAGE.
26099a0e58SBosko Milekic  */
27099a0e58SBosko Milekic 
28099a0e58SBosko Milekic #include <sys/cdefs.h>
29099a0e58SBosko Milekic __FBSDID("$FreeBSD$");
30099a0e58SBosko Milekic 
31099a0e58SBosko Milekic #include "opt_mac.h"
32099a0e58SBosko Milekic #include "opt_param.h"
33099a0e58SBosko Milekic 
34099a0e58SBosko Milekic #include <sys/param.h>
35099a0e58SBosko Milekic #include <sys/malloc.h>
36099a0e58SBosko Milekic #include <sys/systm.h>
37099a0e58SBosko Milekic #include <sys/mbuf.h>
38099a0e58SBosko Milekic #include <sys/domain.h>
39099a0e58SBosko Milekic #include <sys/eventhandler.h>
40099a0e58SBosko Milekic #include <sys/kernel.h>
41099a0e58SBosko Milekic #include <sys/protosw.h>
42099a0e58SBosko Milekic #include <sys/smp.h>
43099a0e58SBosko Milekic #include <sys/sysctl.h>
44099a0e58SBosko Milekic 
45aed55708SRobert Watson #include <security/mac/mac_framework.h>
46aed55708SRobert Watson 
47099a0e58SBosko Milekic #include <vm/vm.h>
48099a0e58SBosko Milekic #include <vm/vm_page.h>
49099a0e58SBosko Milekic #include <vm/uma.h>
50121f0509SMike Silbersack #include <vm/uma_int.h>
51121f0509SMike Silbersack #include <vm/uma_dbg.h>
52099a0e58SBosko Milekic 
53099a0e58SBosko Milekic /*
54099a0e58SBosko Milekic  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
55099a0e58SBosko Milekic  * Zones.
56099a0e58SBosko Milekic  *
57099a0e58SBosko Milekic  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
58099a0e58SBosko Milekic  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
59099a0e58SBosko Milekic  * administrator so desires.
60099a0e58SBosko Milekic  *
61099a0e58SBosko Milekic  * Mbufs are allocated from a UMA Master Zone called the Mbuf
62099a0e58SBosko Milekic  * Zone.
63099a0e58SBosko Milekic  *
64099a0e58SBosko Milekic  * Additionally, FreeBSD provides a Packet Zone, which it
65099a0e58SBosko Milekic  * configures as a Secondary Zone to the Mbuf Master Zone,
66099a0e58SBosko Milekic  * thus sharing backend Slab kegs with the Mbuf Master Zone.
67099a0e58SBosko Milekic  *
68099a0e58SBosko Milekic  * Thus common-case allocations and locking are simplified:
69099a0e58SBosko Milekic  *
70099a0e58SBosko Milekic  *  m_clget()                m_getcl()
71099a0e58SBosko Milekic  *    |                         |
72099a0e58SBosko Milekic  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
73099a0e58SBosko Milekic  *    |   |             [     Packet   ]            |
74099a0e58SBosko Milekic  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
75099a0e58SBosko Milekic  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
76099a0e58SBosko Milekic  *        |                       \________         |
77099a0e58SBosko Milekic  *  [ Cluster Keg   ]                      \       /
78099a0e58SBosko Milekic  *        |    	                         [ Mbuf Keg   ]
79099a0e58SBosko Milekic  *  [ Cluster Slabs ]                         |
80099a0e58SBosko Milekic  *        |                              [ Mbuf Slabs ]
81099a0e58SBosko Milekic  *         \____________(VM)_________________/
8256a4e45aSAndre Oppermann  *
8356a4e45aSAndre Oppermann  *
84fcf90618SGleb Smirnoff  * Whenever an object is allocated with uma_zalloc() out of
8556a4e45aSAndre Oppermann  * one of the Zones its _ctor_ function is executed.  The same
86fcf90618SGleb Smirnoff  * for any deallocation through uma_zfree() the _dtor_ function
8756a4e45aSAndre Oppermann  * is executed.
8856a4e45aSAndre Oppermann  *
8956a4e45aSAndre Oppermann  * Caches are per-CPU and are filled from the Master Zone.
9056a4e45aSAndre Oppermann  *
91fcf90618SGleb Smirnoff  * Whenever an object is allocated from the underlying global
9256a4e45aSAndre Oppermann  * memory pool it gets pre-initialized with the _zinit_ functions.
9356a4e45aSAndre Oppermann  * When the Keg's are overfull objects get decomissioned with
9456a4e45aSAndre Oppermann  * _zfini_ functions and free'd back to the global memory pool.
9556a4e45aSAndre Oppermann  *
96099a0e58SBosko Milekic  */
97099a0e58SBosko Milekic 
9856a4e45aSAndre Oppermann int nmbclusters;		/* limits number of mbuf clusters */
99ec63cb90SAndre Oppermann int nmbjumbop;			/* limits number of page size jumbo clusters */
10056a4e45aSAndre Oppermann int nmbjumbo9;			/* limits number of 9k jumbo clusters */
10156a4e45aSAndre Oppermann int nmbjumbo16;			/* limits number of 16k jumbo clusters */
102099a0e58SBosko Milekic struct mbstat mbstat;
103099a0e58SBosko Milekic 
104099a0e58SBosko Milekic static void
105099a0e58SBosko Milekic tunable_mbinit(void *dummy)
106099a0e58SBosko Milekic {
107099a0e58SBosko Milekic 
108099a0e58SBosko Milekic 	/* This has to be done before VM init. */
109099a0e58SBosko Milekic 	nmbclusters = 1024 + maxusers * 64;
110099a0e58SBosko Milekic 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
111099a0e58SBosko Milekic }
112099a0e58SBosko Milekic SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
113099a0e58SBosko Milekic 
11456a4e45aSAndre Oppermann /* XXX: These should be tuneables. Can't change UMA limits on the fly. */
1154f590175SPaul Saab static int
1164f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS)
1174f590175SPaul Saab {
1184f590175SPaul Saab 	int error, newnmbclusters;
1194f590175SPaul Saab 
1204f590175SPaul Saab 	newnmbclusters = nmbclusters;
1214f590175SPaul Saab 	error = sysctl_handle_int(oidp, &newnmbclusters, sizeof(int), req);
1224f590175SPaul Saab 	if (error == 0 && req->newptr) {
1234f590175SPaul Saab 		if (newnmbclusters > nmbclusters) {
1244f590175SPaul Saab 			nmbclusters = newnmbclusters;
1254f590175SPaul Saab 			uma_zone_set_max(zone_clust, nmbclusters);
1264f590175SPaul Saab 			EVENTHANDLER_INVOKE(nmbclusters_change);
1274f590175SPaul Saab 		} else
1284f590175SPaul Saab 			error = EINVAL;
1294f590175SPaul Saab 	}
1304f590175SPaul Saab 	return (error);
1314f590175SPaul Saab }
1324f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW,
1334f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU",
134099a0e58SBosko Milekic     "Maximum number of mbuf clusters allowed");
135ec63cb90SAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0,
136ec63cb90SAndre Oppermann     "Maximum number of mbuf page size jumbo clusters allowed");
13756a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0,
13856a4e45aSAndre Oppermann     "Maximum number of mbuf 9k jumbo clusters allowed");
13956a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0,
14056a4e45aSAndre Oppermann     "Maximum number of mbuf 16k jumbo clusters allowed");
141099a0e58SBosko Milekic SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
142099a0e58SBosko Milekic     "Mbuf general information and statistics");
143099a0e58SBosko Milekic 
144099a0e58SBosko Milekic /*
145099a0e58SBosko Milekic  * Zones from which we allocate.
146099a0e58SBosko Milekic  */
147099a0e58SBosko Milekic uma_zone_t	zone_mbuf;
148099a0e58SBosko Milekic uma_zone_t	zone_clust;
149099a0e58SBosko Milekic uma_zone_t	zone_pack;
150ec63cb90SAndre Oppermann uma_zone_t	zone_jumbop;
15156a4e45aSAndre Oppermann uma_zone_t	zone_jumbo9;
15256a4e45aSAndre Oppermann uma_zone_t	zone_jumbo16;
15356a4e45aSAndre Oppermann uma_zone_t	zone_ext_refcnt;
154099a0e58SBosko Milekic 
155099a0e58SBosko Milekic /*
156099a0e58SBosko Milekic  * Local prototypes.
157099a0e58SBosko Milekic  */
158b23f72e9SBrian Feldman static int	mb_ctor_mbuf(void *, int, void *, int);
159b23f72e9SBrian Feldman static int	mb_ctor_clust(void *, int, void *, int);
160b23f72e9SBrian Feldman static int	mb_ctor_pack(void *, int, void *, int);
161099a0e58SBosko Milekic static void	mb_dtor_mbuf(void *, int, void *);
16256a4e45aSAndre Oppermann static void	mb_dtor_clust(void *, int, void *);
16356a4e45aSAndre Oppermann static void	mb_dtor_pack(void *, int, void *);
16456a4e45aSAndre Oppermann static int	mb_zinit_pack(void *, int, int);
16556a4e45aSAndre Oppermann static void	mb_zfini_pack(void *, int);
166099a0e58SBosko Milekic 
167099a0e58SBosko Milekic static void	mb_reclaim(void *);
168099a0e58SBosko Milekic static void	mbuf_init(void *);
169099a0e58SBosko Milekic 
170a04946cfSBrian Somers /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
171a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
172a04946cfSBrian Somers 
173099a0e58SBosko Milekic /*
174099a0e58SBosko Milekic  * Initialize FreeBSD Network buffer allocation.
175099a0e58SBosko Milekic  */
176099a0e58SBosko Milekic SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
177099a0e58SBosko Milekic static void
178099a0e58SBosko Milekic mbuf_init(void *dummy)
179099a0e58SBosko Milekic {
180099a0e58SBosko Milekic 
181099a0e58SBosko Milekic 	/*
182099a0e58SBosko Milekic 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
183099a0e58SBosko Milekic 	 */
18456a4e45aSAndre Oppermann 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
18556a4e45aSAndre Oppermann 	    mb_ctor_mbuf, mb_dtor_mbuf,
186121f0509SMike Silbersack #ifdef INVARIANTS
18756a4e45aSAndre Oppermann 	    trash_init, trash_fini,
188121f0509SMike Silbersack #else
18956a4e45aSAndre Oppermann 	    NULL, NULL,
190121f0509SMike Silbersack #endif
19156a4e45aSAndre Oppermann 	    MSIZE - 1, UMA_ZONE_MAXBUCKET);
19256a4e45aSAndre Oppermann 
19368352adfSRobert Watson 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
19456a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
195121f0509SMike Silbersack #ifdef INVARIANTS
19656a4e45aSAndre Oppermann 	    trash_init, trash_fini,
197121f0509SMike Silbersack #else
19856a4e45aSAndre Oppermann 	    NULL, NULL,
199121f0509SMike Silbersack #endif
20056a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
201099a0e58SBosko Milekic 	if (nmbclusters > 0)
202099a0e58SBosko Milekic 		uma_zone_set_max(zone_clust, nmbclusters);
203099a0e58SBosko Milekic 
20456a4e45aSAndre Oppermann 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
20556a4e45aSAndre Oppermann 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
20656a4e45aSAndre Oppermann 
207fcf90618SGleb Smirnoff 	/* Make jumbo frame zone too. Page size, 9k and 16k. */
208ec63cb90SAndre Oppermann 	zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
209d5269a63SAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
210d5269a63SAndre Oppermann #ifdef INVARIANTS
211d5269a63SAndre Oppermann 	    trash_init, trash_fini,
212d5269a63SAndre Oppermann #else
213d5269a63SAndre Oppermann 	    NULL, NULL,
214d5269a63SAndre Oppermann #endif
215d5269a63SAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
216ec63cb90SAndre Oppermann 	if (nmbjumbop > 0)
217ec63cb90SAndre Oppermann 		uma_zone_set_max(zone_jumbop, nmbjumbop);
218d5269a63SAndre Oppermann 
21956a4e45aSAndre Oppermann 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
22056a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
22156a4e45aSAndre Oppermann #ifdef INVARIANTS
22256a4e45aSAndre Oppermann 	    trash_init, trash_fini,
22356a4e45aSAndre Oppermann #else
22456a4e45aSAndre Oppermann 	    NULL, NULL,
22556a4e45aSAndre Oppermann #endif
22656a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
22756a4e45aSAndre Oppermann 	if (nmbjumbo9 > 0)
22856a4e45aSAndre Oppermann 		uma_zone_set_max(zone_jumbo9, nmbjumbo9);
22956a4e45aSAndre Oppermann 
23056a4e45aSAndre Oppermann 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
23156a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
23256a4e45aSAndre Oppermann #ifdef INVARIANTS
23356a4e45aSAndre Oppermann 	    trash_init, trash_fini,
23456a4e45aSAndre Oppermann #else
23556a4e45aSAndre Oppermann 	    NULL, NULL,
23656a4e45aSAndre Oppermann #endif
23756a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
23856a4e45aSAndre Oppermann 	if (nmbjumbo16 > 0)
23956a4e45aSAndre Oppermann 		uma_zone_set_max(zone_jumbo16, nmbjumbo16);
24056a4e45aSAndre Oppermann 
24156a4e45aSAndre Oppermann 	zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
24256a4e45aSAndre Oppermann 	    NULL, NULL,
24356a4e45aSAndre Oppermann 	    NULL, NULL,
24456a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
24556a4e45aSAndre Oppermann 
24656a4e45aSAndre Oppermann 	/* uma_prealloc() goes here... */
247099a0e58SBosko Milekic 
248099a0e58SBosko Milekic 	/*
249099a0e58SBosko Milekic 	 * Hook event handler for low-memory situation, used to
250099a0e58SBosko Milekic 	 * drain protocols and push data back to the caches (UMA
251099a0e58SBosko Milekic 	 * later pushes it back to VM).
252099a0e58SBosko Milekic 	 */
253099a0e58SBosko Milekic 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
254099a0e58SBosko Milekic 	    EVENTHANDLER_PRI_FIRST);
255099a0e58SBosko Milekic 
256099a0e58SBosko Milekic 	/*
257099a0e58SBosko Milekic 	 * [Re]set counters and local statistics knobs.
258099a0e58SBosko Milekic 	 * XXX Some of these should go and be replaced, but UMA stat
259099a0e58SBosko Milekic 	 * gathering needs to be revised.
260099a0e58SBosko Milekic 	 */
261099a0e58SBosko Milekic 	mbstat.m_mbufs = 0;
262099a0e58SBosko Milekic 	mbstat.m_mclusts = 0;
263099a0e58SBosko Milekic 	mbstat.m_drain = 0;
264099a0e58SBosko Milekic 	mbstat.m_msize = MSIZE;
265099a0e58SBosko Milekic 	mbstat.m_mclbytes = MCLBYTES;
266099a0e58SBosko Milekic 	mbstat.m_minclsize = MINCLSIZE;
267099a0e58SBosko Milekic 	mbstat.m_mlen = MLEN;
268099a0e58SBosko Milekic 	mbstat.m_mhlen = MHLEN;
269099a0e58SBosko Milekic 	mbstat.m_numtypes = MT_NTYPES;
270099a0e58SBosko Milekic 
271099a0e58SBosko Milekic 	mbstat.m_mcfail = mbstat.m_mpfail = 0;
272099a0e58SBosko Milekic 	mbstat.sf_iocnt = 0;
273099a0e58SBosko Milekic 	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
274099a0e58SBosko Milekic }
275099a0e58SBosko Milekic 
276099a0e58SBosko Milekic /*
277099a0e58SBosko Milekic  * Constructor for Mbuf master zone.
278099a0e58SBosko Milekic  *
279099a0e58SBosko Milekic  * The 'arg' pointer points to a mb_args structure which
280099a0e58SBosko Milekic  * contains call-specific information required to support the
28156a4e45aSAndre Oppermann  * mbuf allocation API.  See mbuf.h.
282099a0e58SBosko Milekic  */
283b23f72e9SBrian Feldman static int
284b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how)
285099a0e58SBosko Milekic {
286099a0e58SBosko Milekic 	struct mbuf *m;
287099a0e58SBosko Milekic 	struct mb_args *args;
288b23f72e9SBrian Feldman #ifdef MAC
289b23f72e9SBrian Feldman 	int error;
290b23f72e9SBrian Feldman #endif
291099a0e58SBosko Milekic 	int flags;
292099a0e58SBosko Milekic 	short type;
293099a0e58SBosko Milekic 
294121f0509SMike Silbersack #ifdef INVARIANTS
295121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
296121f0509SMike Silbersack #endif
297099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
298099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
299099a0e58SBosko Milekic 	flags = args->flags;
300099a0e58SBosko Milekic 	type = args->type;
301099a0e58SBosko Milekic 
30256a4e45aSAndre Oppermann 	/*
30356a4e45aSAndre Oppermann 	 * The mbuf is initialized later.  The caller has the
304fcf90618SGleb Smirnoff 	 * responsibility to set up any MAC labels too.
30556a4e45aSAndre Oppermann 	 */
30656a4e45aSAndre Oppermann 	if (type == MT_NOINIT)
30756a4e45aSAndre Oppermann 		return (0);
30856a4e45aSAndre Oppermann 
309099a0e58SBosko Milekic 	m->m_next = NULL;
310099a0e58SBosko Milekic 	m->m_nextpkt = NULL;
31156a4e45aSAndre Oppermann 	m->m_len = 0;
3126bc72ab9SBosko Milekic 	m->m_flags = flags;
31356a4e45aSAndre Oppermann 	m->m_type = type;
314099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
315099a0e58SBosko Milekic 		m->m_data = m->m_pktdat;
316099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
31756a4e45aSAndre Oppermann 		m->m_pkthdr.len = 0;
31856a4e45aSAndre Oppermann 		m->m_pkthdr.header = NULL;
319099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
32056a4e45aSAndre Oppermann 		m->m_pkthdr.csum_data = 0;
321a855e2b4SAndre Oppermann 		m->m_pkthdr.tso_segsz = 0;
322a855e2b4SAndre Oppermann 		m->m_pkthdr.ether_vtag = 0;
323099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
324099a0e58SBosko Milekic #ifdef MAC
325099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
326b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
327b23f72e9SBrian Feldman 		if (error)
328b23f72e9SBrian Feldman 			return (error);
329099a0e58SBosko Milekic #endif
3306bc72ab9SBosko Milekic 	} else
331099a0e58SBosko Milekic 		m->m_data = m->m_dat;
332b23f72e9SBrian Feldman 	return (0);
333099a0e58SBosko Milekic }
334099a0e58SBosko Milekic 
335099a0e58SBosko Milekic /*
33656a4e45aSAndre Oppermann  * The Mbuf master zone destructor.
337099a0e58SBosko Milekic  */
338099a0e58SBosko Milekic static void
339099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg)
340099a0e58SBosko Milekic {
341099a0e58SBosko Milekic 	struct mbuf *m;
342099a0e58SBosko Milekic 
343099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
344099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
345099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
34656a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
347121f0509SMike Silbersack #ifdef INVARIANTS
348121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
349121f0509SMike Silbersack #endif
350099a0e58SBosko Milekic }
351099a0e58SBosko Milekic 
35256a4e45aSAndre Oppermann /*
35356a4e45aSAndre Oppermann  * The Mbuf Packet zone destructor.
35456a4e45aSAndre Oppermann  */
355099a0e58SBosko Milekic static void
356099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg)
357099a0e58SBosko Milekic {
358099a0e58SBosko Milekic 	struct mbuf *m;
359099a0e58SBosko Milekic 
360099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
361099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
362099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
36356a4e45aSAndre Oppermann 
36456a4e45aSAndre Oppermann 	/* Make sure we've got a clean cluster back. */
36556a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
36656a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
36756a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
36856a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__));
36956a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
37049d46b61SGleb Smirnoff 	KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
37156a4e45aSAndre Oppermann 	KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
372121f0509SMike Silbersack #ifdef INVARIANTS
373121f0509SMike Silbersack 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
374121f0509SMike Silbersack #endif
3756c125b8dSMohan Srinivasan 	/*
3766c125b8dSMohan Srinivasan 	 * If there are processes blocked on zone_clust, waiting for pages to be freed up,
3776c125b8dSMohan Srinivasan 	 * cause them to be woken up by draining the packet zone. We are exposed to a race here
3786c125b8dSMohan Srinivasan 	 * (in the check for the UMA_ZFLAG_FULL) where we might miss the flag set, but that is
3796c125b8dSMohan Srinivasan 	 * deliberate. We don't want to acquire the zone lock for every mbuf free.
3806c125b8dSMohan Srinivasan 	 */
3816c125b8dSMohan Srinivasan  	if (uma_zone_exhausted_nolock(zone_clust))
3826c125b8dSMohan Srinivasan  		zone_drain(zone_pack);
383099a0e58SBosko Milekic }
384099a0e58SBosko Milekic 
385099a0e58SBosko Milekic /*
386ec63cb90SAndre Oppermann  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
387099a0e58SBosko Milekic  *
388099a0e58SBosko Milekic  * Here the 'arg' pointer points to the Mbuf which we
38956a4e45aSAndre Oppermann  * are configuring cluster storage for.  If 'arg' is
39056a4e45aSAndre Oppermann  * empty we allocate just the cluster without setting
39156a4e45aSAndre Oppermann  * the mbuf to it.  See mbuf.h.
392099a0e58SBosko Milekic  */
393b23f72e9SBrian Feldman static int
394b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how)
395099a0e58SBosko Milekic {
396099a0e58SBosko Milekic 	struct mbuf *m;
39756a4e45aSAndre Oppermann 	u_int *refcnt;
39856a4e45aSAndre Oppermann 	int type = 0;
399099a0e58SBosko Milekic 
400121f0509SMike Silbersack #ifdef INVARIANTS
401121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
402121f0509SMike Silbersack #endif
403099a0e58SBosko Milekic 	m = (struct mbuf *)arg;
40456a4e45aSAndre Oppermann 	if (m != NULL) {
40556a4e45aSAndre Oppermann 		switch (size) {
40656a4e45aSAndre Oppermann 		case MCLBYTES:
40756a4e45aSAndre Oppermann 			type = EXT_CLUSTER;
40856a4e45aSAndre Oppermann 			break;
409ec63cb90SAndre Oppermann #if MJUMPAGESIZE != MCLBYTES
410ec63cb90SAndre Oppermann 		case MJUMPAGESIZE:
411ec63cb90SAndre Oppermann 			type = EXT_JUMBOP;
412d5269a63SAndre Oppermann 			break;
41336ae3fd3SAndre Oppermann #endif
41456a4e45aSAndre Oppermann 		case MJUM9BYTES:
41556a4e45aSAndre Oppermann 			type = EXT_JUMBO9;
41656a4e45aSAndre Oppermann 			break;
41756a4e45aSAndre Oppermann 		case MJUM16BYTES:
41856a4e45aSAndre Oppermann 			type = EXT_JUMBO16;
41956a4e45aSAndre Oppermann 			break;
42056a4e45aSAndre Oppermann 		default:
42156a4e45aSAndre Oppermann 			panic("unknown cluster size");
42256a4e45aSAndre Oppermann 			break;
42356a4e45aSAndre Oppermann 		}
424099a0e58SBosko Milekic 		m->m_ext.ext_buf = (caddr_t)mem;
425099a0e58SBosko Milekic 		m->m_data = m->m_ext.ext_buf;
426099a0e58SBosko Milekic 		m->m_flags |= M_EXT;
427099a0e58SBosko Milekic 		m->m_ext.ext_free = NULL;
428099a0e58SBosko Milekic 		m->m_ext.ext_args = NULL;
42956a4e45aSAndre Oppermann 		m->m_ext.ext_size = size;
43056a4e45aSAndre Oppermann 		m->m_ext.ext_type = type;
43156a4e45aSAndre Oppermann 		m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem);
43256a4e45aSAndre Oppermann 		*m->m_ext.ref_cnt = 1;
43356a4e45aSAndre Oppermann 	} else {
43456a4e45aSAndre Oppermann 		refcnt =  uma_find_refcnt(zone_clust, mem);
43556a4e45aSAndre Oppermann 		*refcnt = 1;
43656a4e45aSAndre Oppermann 	}
437b23f72e9SBrian Feldman 	return (0);
438099a0e58SBosko Milekic }
439099a0e58SBosko Milekic 
44056a4e45aSAndre Oppermann /*
44156a4e45aSAndre Oppermann  * The Mbuf Cluster zone destructor.
44256a4e45aSAndre Oppermann  */
443099a0e58SBosko Milekic static void
444099a0e58SBosko Milekic mb_dtor_clust(void *mem, int size, void *arg)
445099a0e58SBosko Milekic {
44656a4e45aSAndre Oppermann 
447a5f77087SAndre Oppermann 	KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1,
448a5f77087SAndre Oppermann 		("%s: refcnt incorrect %u", __func__,
449a5f77087SAndre Oppermann 		 *(uma_find_refcnt(zone_clust, mem))) );
450121f0509SMike Silbersack #ifdef INVARIANTS
451121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
452121f0509SMike Silbersack #endif
453099a0e58SBosko Milekic }
454099a0e58SBosko Milekic 
455099a0e58SBosko Milekic /*
456099a0e58SBosko Milekic  * The Packet secondary zone's init routine, executed on the
45756a4e45aSAndre Oppermann  * object's transition from mbuf keg slab to zone cache.
458099a0e58SBosko Milekic  */
459b23f72e9SBrian Feldman static int
46056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how)
461099a0e58SBosko Milekic {
462099a0e58SBosko Milekic 	struct mbuf *m;
463099a0e58SBosko Milekic 
46456a4e45aSAndre Oppermann 	m = (struct mbuf *)mem;		/* m is virgin. */
465a7bd90efSAndre Oppermann 	if (uma_zalloc_arg(zone_clust, m, how) == NULL ||
466a7bd90efSAndre Oppermann 	    m->m_ext.ext_buf == NULL)
467b23f72e9SBrian Feldman 		return (ENOMEM);
468cd5bb63bSAndre Oppermann 	m->m_ext.ext_type = EXT_PACKET;	/* Override. */
469121f0509SMike Silbersack #ifdef INVARIANTS
470121f0509SMike Silbersack 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
471121f0509SMike Silbersack #endif
472b23f72e9SBrian Feldman 	return (0);
473099a0e58SBosko Milekic }
474099a0e58SBosko Milekic 
475099a0e58SBosko Milekic /*
476099a0e58SBosko Milekic  * The Packet secondary zone's fini routine, executed on the
477099a0e58SBosko Milekic  * object's transition from zone cache to keg slab.
478099a0e58SBosko Milekic  */
479099a0e58SBosko Milekic static void
48056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size)
481099a0e58SBosko Milekic {
482099a0e58SBosko Milekic 	struct mbuf *m;
483099a0e58SBosko Milekic 
484099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
485121f0509SMike Silbersack #ifdef INVARIANTS
486121f0509SMike Silbersack 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
487121f0509SMike Silbersack #endif
488099a0e58SBosko Milekic 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
489a7b844d2SMike Silbersack #ifdef INVARIANTS
490a7b844d2SMike Silbersack 	trash_dtor(mem, size, NULL);
491a7b844d2SMike Silbersack #endif
492099a0e58SBosko Milekic }
493099a0e58SBosko Milekic 
494099a0e58SBosko Milekic /*
495099a0e58SBosko Milekic  * The "packet" keg constructor.
496099a0e58SBosko Milekic  */
497b23f72e9SBrian Feldman static int
498b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how)
499099a0e58SBosko Milekic {
500099a0e58SBosko Milekic 	struct mbuf *m;
501099a0e58SBosko Milekic 	struct mb_args *args;
502b23f72e9SBrian Feldman #ifdef MAC
503b23f72e9SBrian Feldman 	int error;
504b23f72e9SBrian Feldman #endif
505b23f72e9SBrian Feldman 	int flags;
506099a0e58SBosko Milekic 	short type;
507099a0e58SBosko Milekic 
508099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
509099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
510099a0e58SBosko Milekic 	flags = args->flags;
511099a0e58SBosko Milekic 	type = args->type;
512099a0e58SBosko Milekic 
513121f0509SMike Silbersack #ifdef INVARIANTS
514121f0509SMike Silbersack 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
515121f0509SMike Silbersack #endif
516099a0e58SBosko Milekic 	m->m_next = NULL;
5176bc72ab9SBosko Milekic 	m->m_nextpkt = NULL;
518099a0e58SBosko Milekic 	m->m_data = m->m_ext.ext_buf;
51956a4e45aSAndre Oppermann 	m->m_len = 0;
52056a4e45aSAndre Oppermann 	m->m_flags = (flags | M_EXT);
52156a4e45aSAndre Oppermann 	m->m_type = type;
522099a0e58SBosko Milekic 
523099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
524099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
52556a4e45aSAndre Oppermann 		m->m_pkthdr.len = 0;
52656a4e45aSAndre Oppermann 		m->m_pkthdr.header = NULL;
527099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
52856a4e45aSAndre Oppermann 		m->m_pkthdr.csum_data = 0;
529a855e2b4SAndre Oppermann 		m->m_pkthdr.tso_segsz = 0;
530a855e2b4SAndre Oppermann 		m->m_pkthdr.ether_vtag = 0;
531099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
532099a0e58SBosko Milekic #ifdef MAC
533099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
534b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
535b23f72e9SBrian Feldman 		if (error)
536b23f72e9SBrian Feldman 			return (error);
537099a0e58SBosko Milekic #endif
538099a0e58SBosko Milekic 	}
53956a4e45aSAndre Oppermann 	/* m_ext is already initialized. */
54056a4e45aSAndre Oppermann 
541b23f72e9SBrian Feldman 	return (0);
542099a0e58SBosko Milekic }
543099a0e58SBosko Milekic 
544099a0e58SBosko Milekic /*
545099a0e58SBosko Milekic  * This is the protocol drain routine.
546099a0e58SBosko Milekic  *
547099a0e58SBosko Milekic  * No locks should be held when this is called.  The drain routines have to
548099a0e58SBosko Milekic  * presently acquire some locks which raises the possibility of lock order
549099a0e58SBosko Milekic  * reversal.
550099a0e58SBosko Milekic  */
551099a0e58SBosko Milekic static void
552099a0e58SBosko Milekic mb_reclaim(void *junk)
553099a0e58SBosko Milekic {
554099a0e58SBosko Milekic 	struct domain *dp;
555099a0e58SBosko Milekic 	struct protosw *pr;
556099a0e58SBosko Milekic 
557099a0e58SBosko Milekic 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
558099a0e58SBosko Milekic 	    "mb_reclaim()");
559099a0e58SBosko Milekic 
560099a0e58SBosko Milekic 	for (dp = domains; dp != NULL; dp = dp->dom_next)
561099a0e58SBosko Milekic 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
562099a0e58SBosko Milekic 			if (pr->pr_drain != NULL)
563099a0e58SBosko Milekic 				(*pr->pr_drain)();
564099a0e58SBosko Milekic }
565