xref: /freebsd/sys/kern/kern_mbuf.c (revision ec63cb90a302eabc68ae1cc695c5b4f3a78b7ce3)
1099a0e58SBosko Milekic /*-
28076cb52SBosko Milekic  * Copyright (c) 2004, 2005,
38076cb52SBosko Milekic  * 	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
4099a0e58SBosko Milekic  *
5099a0e58SBosko Milekic  * Redistribution and use in source and binary forms, with or without
6099a0e58SBosko Milekic  * modification, are permitted provided that the following conditions
7099a0e58SBosko Milekic  * are met:
8099a0e58SBosko Milekic  * 1. Redistributions of source code must retain the above copyright
9099a0e58SBosko Milekic  *    notice unmodified, this list of conditions and the following
10099a0e58SBosko Milekic  *    disclaimer.
11099a0e58SBosko Milekic  * 2. Redistributions in binary form must reproduce the above copyright
12099a0e58SBosko Milekic  *    notice, this list of conditions and the following disclaimer in the
13099a0e58SBosko Milekic  *    documentation and/or other materials provided with the distribution.
14099a0e58SBosko Milekic  *
15099a0e58SBosko Milekic  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16099a0e58SBosko Milekic  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17099a0e58SBosko Milekic  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18099a0e58SBosko Milekic  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19099a0e58SBosko Milekic  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20099a0e58SBosko Milekic  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21099a0e58SBosko Milekic  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22099a0e58SBosko Milekic  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23099a0e58SBosko Milekic  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24099a0e58SBosko Milekic  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25099a0e58SBosko Milekic  * SUCH DAMAGE.
26099a0e58SBosko Milekic  */
27099a0e58SBosko Milekic 
28099a0e58SBosko Milekic #include <sys/cdefs.h>
29099a0e58SBosko Milekic __FBSDID("$FreeBSD$");
30099a0e58SBosko Milekic 
31099a0e58SBosko Milekic #include "opt_mac.h"
32099a0e58SBosko Milekic #include "opt_param.h"
33099a0e58SBosko Milekic 
34099a0e58SBosko Milekic #include <sys/param.h>
35099a0e58SBosko Milekic #include <sys/mac.h>
36099a0e58SBosko Milekic #include <sys/malloc.h>
37099a0e58SBosko Milekic #include <sys/systm.h>
38099a0e58SBosko Milekic #include <sys/mbuf.h>
39099a0e58SBosko Milekic #include <sys/domain.h>
40099a0e58SBosko Milekic #include <sys/eventhandler.h>
41099a0e58SBosko Milekic #include <sys/kernel.h>
42099a0e58SBosko Milekic #include <sys/protosw.h>
43099a0e58SBosko Milekic #include <sys/smp.h>
44099a0e58SBosko Milekic #include <sys/sysctl.h>
45099a0e58SBosko Milekic 
46099a0e58SBosko Milekic #include <vm/vm.h>
47099a0e58SBosko Milekic #include <vm/vm_page.h>
48099a0e58SBosko Milekic #include <vm/uma.h>
49121f0509SMike Silbersack #include <vm/uma_int.h>
50121f0509SMike Silbersack #include <vm/uma_dbg.h>
51099a0e58SBosko Milekic 
52099a0e58SBosko Milekic /*
53099a0e58SBosko Milekic  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
54099a0e58SBosko Milekic  * Zones.
55099a0e58SBosko Milekic  *
56099a0e58SBosko Milekic  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
57099a0e58SBosko Milekic  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
58099a0e58SBosko Milekic  * administrator so desires.
59099a0e58SBosko Milekic  *
60099a0e58SBosko Milekic  * Mbufs are allocated from a UMA Master Zone called the Mbuf
61099a0e58SBosko Milekic  * Zone.
62099a0e58SBosko Milekic  *
63099a0e58SBosko Milekic  * Additionally, FreeBSD provides a Packet Zone, which it
64099a0e58SBosko Milekic  * configures as a Secondary Zone to the Mbuf Master Zone,
65099a0e58SBosko Milekic  * thus sharing backend Slab kegs with the Mbuf Master Zone.
66099a0e58SBosko Milekic  *
67099a0e58SBosko Milekic  * Thus common-case allocations and locking are simplified:
68099a0e58SBosko Milekic  *
69099a0e58SBosko Milekic  *  m_clget()                m_getcl()
70099a0e58SBosko Milekic  *    |                         |
71099a0e58SBosko Milekic  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
72099a0e58SBosko Milekic  *    |   |             [     Packet   ]            |
73099a0e58SBosko Milekic  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
74099a0e58SBosko Milekic  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
75099a0e58SBosko Milekic  *        |                       \________         |
76099a0e58SBosko Milekic  *  [ Cluster Keg   ]                      \       /
77099a0e58SBosko Milekic  *        |    	                         [ Mbuf Keg   ]
78099a0e58SBosko Milekic  *  [ Cluster Slabs ]                         |
79099a0e58SBosko Milekic  *        |                              [ Mbuf Slabs ]
80099a0e58SBosko Milekic  *         \____________(VM)_________________/
8156a4e45aSAndre Oppermann  *
8256a4e45aSAndre Oppermann  *
8356a4e45aSAndre Oppermann  * Whenever a object is allocated with uma_zalloc() out of the
8456a4e45aSAndre Oppermann  * one of the Zones its _ctor_ function is executed.  The same
8556a4e45aSAndre Oppermann  * for any deallocation through uma_zfree() the _dror_ function
8656a4e45aSAndre Oppermann  * is executed.
8756a4e45aSAndre Oppermann  *
8856a4e45aSAndre Oppermann  * Caches are per-CPU and are filled from the Master Zone.
8956a4e45aSAndre Oppermann  *
9056a4e45aSAndre Oppermann  * Whenever a object is allocated from the underlying global
9156a4e45aSAndre Oppermann  * memory pool it gets pre-initialized with the _zinit_ functions.
9256a4e45aSAndre Oppermann  * When the Keg's are overfull objects get decomissioned with
9356a4e45aSAndre Oppermann  * _zfini_ functions and free'd back to the global memory pool.
9456a4e45aSAndre Oppermann  *
95099a0e58SBosko Milekic  */
96099a0e58SBosko Milekic 
9756a4e45aSAndre Oppermann int nmbclusters;		/* limits number of mbuf clusters */
98ec63cb90SAndre Oppermann int nmbjumbop;			/* limits number of page size jumbo clusters */
9956a4e45aSAndre Oppermann int nmbjumbo9;			/* limits number of 9k jumbo clusters */
10056a4e45aSAndre Oppermann int nmbjumbo16;			/* limits number of 16k jumbo clusters */
101099a0e58SBosko Milekic struct mbstat mbstat;
102099a0e58SBosko Milekic 
103099a0e58SBosko Milekic static void
104099a0e58SBosko Milekic tunable_mbinit(void *dummy)
105099a0e58SBosko Milekic {
106099a0e58SBosko Milekic 
107099a0e58SBosko Milekic 	/* This has to be done before VM init. */
108099a0e58SBosko Milekic 	nmbclusters = 1024 + maxusers * 64;
109099a0e58SBosko Milekic 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
110099a0e58SBosko Milekic }
111099a0e58SBosko Milekic SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
112099a0e58SBosko Milekic 
113099a0e58SBosko Milekic SYSCTL_DECL(_kern_ipc);
11456a4e45aSAndre Oppermann /* XXX: These should be tuneables. Can't change UMA limits on the fly. */
115099a0e58SBosko Milekic SYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RW, &nmbclusters, 0,
116099a0e58SBosko Milekic     "Maximum number of mbuf clusters allowed");
117ec63cb90SAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0,
118ec63cb90SAndre Oppermann     "Maximum number of mbuf page size jumbo clusters allowed");
11956a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0,
12056a4e45aSAndre Oppermann     "Maximum number of mbuf 9k jumbo clusters allowed");
12156a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0,
12256a4e45aSAndre Oppermann     "Maximum number of mbuf 16k jumbo clusters allowed");
123099a0e58SBosko Milekic SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
124099a0e58SBosko Milekic     "Mbuf general information and statistics");
125099a0e58SBosko Milekic 
126099a0e58SBosko Milekic /*
127099a0e58SBosko Milekic  * Zones from which we allocate.
128099a0e58SBosko Milekic  */
129099a0e58SBosko Milekic uma_zone_t	zone_mbuf;
130099a0e58SBosko Milekic uma_zone_t	zone_clust;
131099a0e58SBosko Milekic uma_zone_t	zone_pack;
132ec63cb90SAndre Oppermann uma_zone_t	zone_jumbop;
13356a4e45aSAndre Oppermann uma_zone_t	zone_jumbo9;
13456a4e45aSAndre Oppermann uma_zone_t	zone_jumbo16;
13556a4e45aSAndre Oppermann uma_zone_t	zone_ext_refcnt;
13675ee267cSGleb Smirnoff uma_zone_t	zone_mtag_vlan;
137099a0e58SBosko Milekic 
138099a0e58SBosko Milekic /*
139099a0e58SBosko Milekic  * Local prototypes.
140099a0e58SBosko Milekic  */
141b23f72e9SBrian Feldman static int	mb_ctor_mbuf(void *, int, void *, int);
142b23f72e9SBrian Feldman static int	mb_ctor_clust(void *, int, void *, int);
143b23f72e9SBrian Feldman static int	mb_ctor_pack(void *, int, void *, int);
144099a0e58SBosko Milekic static void	mb_dtor_mbuf(void *, int, void *);
14556a4e45aSAndre Oppermann static void	mb_dtor_clust(void *, int, void *);
14656a4e45aSAndre Oppermann static void	mb_dtor_pack(void *, int, void *);
14756a4e45aSAndre Oppermann static int	mb_zinit_pack(void *, int, int);
14856a4e45aSAndre Oppermann static void	mb_zfini_pack(void *, int);
14975ee267cSGleb Smirnoff static int	mt_zinit_vlan(void *, int, int);
150099a0e58SBosko Milekic 
151099a0e58SBosko Milekic static void	mb_reclaim(void *);
152099a0e58SBosko Milekic static void	mbuf_init(void *);
153099a0e58SBosko Milekic 
154a04946cfSBrian Somers /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
155a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
156a04946cfSBrian Somers 
157099a0e58SBosko Milekic /*
158099a0e58SBosko Milekic  * Initialize FreeBSD Network buffer allocation.
159099a0e58SBosko Milekic  */
160099a0e58SBosko Milekic SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
161099a0e58SBosko Milekic static void
162099a0e58SBosko Milekic mbuf_init(void *dummy)
163099a0e58SBosko Milekic {
164099a0e58SBosko Milekic 
165099a0e58SBosko Milekic 	/*
166099a0e58SBosko Milekic 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
167099a0e58SBosko Milekic 	 */
16856a4e45aSAndre Oppermann 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE,
16956a4e45aSAndre Oppermann 	    mb_ctor_mbuf, mb_dtor_mbuf,
170121f0509SMike Silbersack #ifdef INVARIANTS
17156a4e45aSAndre Oppermann 	    trash_init, trash_fini,
172121f0509SMike Silbersack #else
17356a4e45aSAndre Oppermann 	    NULL, NULL,
174121f0509SMike Silbersack #endif
17556a4e45aSAndre Oppermann 	    MSIZE - 1, UMA_ZONE_MAXBUCKET);
17656a4e45aSAndre Oppermann 
17768352adfSRobert Watson 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
17856a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
179121f0509SMike Silbersack #ifdef INVARIANTS
18056a4e45aSAndre Oppermann 	    trash_init, trash_fini,
181121f0509SMike Silbersack #else
18256a4e45aSAndre Oppermann 	    NULL, NULL,
183121f0509SMike Silbersack #endif
18456a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
185099a0e58SBosko Milekic 	if (nmbclusters > 0)
186099a0e58SBosko Milekic 		uma_zone_set_max(zone_clust, nmbclusters);
187099a0e58SBosko Milekic 
18856a4e45aSAndre Oppermann 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
18956a4e45aSAndre Oppermann 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
19056a4e45aSAndre Oppermann 
191d5269a63SAndre Oppermann 	/* Make jumbo frame zone too. 4k, 9k and 16k. */
192ec63cb90SAndre Oppermann 	zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE,
193d5269a63SAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
194d5269a63SAndre Oppermann #ifdef INVARIANTS
195d5269a63SAndre Oppermann 	    trash_init, trash_fini,
196d5269a63SAndre Oppermann #else
197d5269a63SAndre Oppermann 	    NULL, NULL,
198d5269a63SAndre Oppermann #endif
199d5269a63SAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
200ec63cb90SAndre Oppermann 	if (nmbjumbop > 0)
201ec63cb90SAndre Oppermann 		uma_zone_set_max(zone_jumbop, nmbjumbop);
202d5269a63SAndre Oppermann 
20356a4e45aSAndre Oppermann 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
20456a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
20556a4e45aSAndre Oppermann #ifdef INVARIANTS
20656a4e45aSAndre Oppermann 	    trash_init, trash_fini,
20756a4e45aSAndre Oppermann #else
20856a4e45aSAndre Oppermann 	    NULL, NULL,
20956a4e45aSAndre Oppermann #endif
21056a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
21156a4e45aSAndre Oppermann 	if (nmbjumbo9 > 0)
21256a4e45aSAndre Oppermann 		uma_zone_set_max(zone_jumbo9, nmbjumbo9);
21356a4e45aSAndre Oppermann 
21456a4e45aSAndre Oppermann 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
21556a4e45aSAndre Oppermann 	    mb_ctor_clust, mb_dtor_clust,
21656a4e45aSAndre Oppermann #ifdef INVARIANTS
21756a4e45aSAndre Oppermann 	    trash_init, trash_fini,
21856a4e45aSAndre Oppermann #else
21956a4e45aSAndre Oppermann 	    NULL, NULL,
22056a4e45aSAndre Oppermann #endif
22156a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
22256a4e45aSAndre Oppermann 	if (nmbjumbo16 > 0)
22356a4e45aSAndre Oppermann 		uma_zone_set_max(zone_jumbo16, nmbjumbo16);
22456a4e45aSAndre Oppermann 
22556a4e45aSAndre Oppermann 	zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
22656a4e45aSAndre Oppermann 	    NULL, NULL,
22756a4e45aSAndre Oppermann 	    NULL, NULL,
22856a4e45aSAndre Oppermann 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
22956a4e45aSAndre Oppermann 
23075ee267cSGleb Smirnoff 	zone_mtag_vlan = uma_zcreate("mtag_vlan",
23175ee267cSGleb Smirnoff 	    sizeof(struct m_tag) + sizeof(u_int),
23275ee267cSGleb Smirnoff 	    NULL, NULL,
23375ee267cSGleb Smirnoff 	    mt_zinit_vlan, NULL,
23475ee267cSGleb Smirnoff 	    UMA_ALIGN_INT, 0);
23575ee267cSGleb Smirnoff 
23656a4e45aSAndre Oppermann 	/* uma_prealloc() goes here... */
237099a0e58SBosko Milekic 
238099a0e58SBosko Milekic 	/*
239099a0e58SBosko Milekic 	 * Hook event handler for low-memory situation, used to
240099a0e58SBosko Milekic 	 * drain protocols and push data back to the caches (UMA
241099a0e58SBosko Milekic 	 * later pushes it back to VM).
242099a0e58SBosko Milekic 	 */
243099a0e58SBosko Milekic 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
244099a0e58SBosko Milekic 	    EVENTHANDLER_PRI_FIRST);
245099a0e58SBosko Milekic 
246099a0e58SBosko Milekic 	/*
247099a0e58SBosko Milekic 	 * [Re]set counters and local statistics knobs.
248099a0e58SBosko Milekic 	 * XXX Some of these should go and be replaced, but UMA stat
249099a0e58SBosko Milekic 	 * gathering needs to be revised.
250099a0e58SBosko Milekic 	 */
251099a0e58SBosko Milekic 	mbstat.m_mbufs = 0;
252099a0e58SBosko Milekic 	mbstat.m_mclusts = 0;
253099a0e58SBosko Milekic 	mbstat.m_drain = 0;
254099a0e58SBosko Milekic 	mbstat.m_msize = MSIZE;
255099a0e58SBosko Milekic 	mbstat.m_mclbytes = MCLBYTES;
256099a0e58SBosko Milekic 	mbstat.m_minclsize = MINCLSIZE;
257099a0e58SBosko Milekic 	mbstat.m_mlen = MLEN;
258099a0e58SBosko Milekic 	mbstat.m_mhlen = MHLEN;
259099a0e58SBosko Milekic 	mbstat.m_numtypes = MT_NTYPES;
260099a0e58SBosko Milekic 
261099a0e58SBosko Milekic 	mbstat.m_mcfail = mbstat.m_mpfail = 0;
262099a0e58SBosko Milekic 	mbstat.sf_iocnt = 0;
263099a0e58SBosko Milekic 	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
264099a0e58SBosko Milekic }
265099a0e58SBosko Milekic 
266099a0e58SBosko Milekic /*
267099a0e58SBosko Milekic  * Constructor for Mbuf master zone.
268099a0e58SBosko Milekic  *
269099a0e58SBosko Milekic  * The 'arg' pointer points to a mb_args structure which
270099a0e58SBosko Milekic  * contains call-specific information required to support the
27156a4e45aSAndre Oppermann  * mbuf allocation API.  See mbuf.h.
272099a0e58SBosko Milekic  */
273b23f72e9SBrian Feldman static int
274b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how)
275099a0e58SBosko Milekic {
276099a0e58SBosko Milekic 	struct mbuf *m;
277099a0e58SBosko Milekic 	struct mb_args *args;
278b23f72e9SBrian Feldman #ifdef MAC
279b23f72e9SBrian Feldman 	int error;
280b23f72e9SBrian Feldman #endif
281099a0e58SBosko Milekic 	int flags;
282099a0e58SBosko Milekic 	short type;
283099a0e58SBosko Milekic 
284121f0509SMike Silbersack #ifdef INVARIANTS
285121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
286121f0509SMike Silbersack #endif
287099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
288099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
289099a0e58SBosko Milekic 	flags = args->flags;
290099a0e58SBosko Milekic 	type = args->type;
291099a0e58SBosko Milekic 
29256a4e45aSAndre Oppermann 	/*
29356a4e45aSAndre Oppermann 	 * The mbuf is initialized later.  The caller has the
29456a4e45aSAndre Oppermann 	 * responseability to setup any MAC labels too.
29556a4e45aSAndre Oppermann 	 */
29656a4e45aSAndre Oppermann 	if (type == MT_NOINIT)
29756a4e45aSAndre Oppermann 		return (0);
29856a4e45aSAndre Oppermann 
299099a0e58SBosko Milekic 	m->m_next = NULL;
300099a0e58SBosko Milekic 	m->m_nextpkt = NULL;
30156a4e45aSAndre Oppermann 	m->m_len = 0;
3026bc72ab9SBosko Milekic 	m->m_flags = flags;
30356a4e45aSAndre Oppermann 	m->m_type = type;
304099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
305099a0e58SBosko Milekic 		m->m_data = m->m_pktdat;
306099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
30756a4e45aSAndre Oppermann 		m->m_pkthdr.len = 0;
30856a4e45aSAndre Oppermann 		m->m_pkthdr.header = NULL;
309099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
31056a4e45aSAndre Oppermann 		m->m_pkthdr.csum_data = 0;
311099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
312099a0e58SBosko Milekic #ifdef MAC
313099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
314b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
315b23f72e9SBrian Feldman 		if (error)
316b23f72e9SBrian Feldman 			return (error);
317099a0e58SBosko Milekic #endif
3186bc72ab9SBosko Milekic 	} else
319099a0e58SBosko Milekic 		m->m_data = m->m_dat;
320b23f72e9SBrian Feldman 	return (0);
321099a0e58SBosko Milekic }
322099a0e58SBosko Milekic 
323099a0e58SBosko Milekic /*
32456a4e45aSAndre Oppermann  * The Mbuf master zone destructor.
325099a0e58SBosko Milekic  */
326099a0e58SBosko Milekic static void
327099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg)
328099a0e58SBosko Milekic {
329099a0e58SBosko Milekic 	struct mbuf *m;
330099a0e58SBosko Milekic 
331099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
332099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
333099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
33456a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__));
335121f0509SMike Silbersack #ifdef INVARIANTS
336121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
337121f0509SMike Silbersack #endif
338099a0e58SBosko Milekic }
339099a0e58SBosko Milekic 
34056a4e45aSAndre Oppermann /*
34156a4e45aSAndre Oppermann  * The Mbuf Packet zone destructor.
34256a4e45aSAndre Oppermann  */
343099a0e58SBosko Milekic static void
344099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg)
345099a0e58SBosko Milekic {
346099a0e58SBosko Milekic 	struct mbuf *m;
347099a0e58SBosko Milekic 
348099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
349099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
350099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
35156a4e45aSAndre Oppermann 
35256a4e45aSAndre Oppermann 	/* Make sure we've got a clean cluster back. */
35356a4e45aSAndre Oppermann 	KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__));
35456a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__));
35556a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__));
35656a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__));
35756a4e45aSAndre Oppermann 	KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__));
35849d46b61SGleb Smirnoff 	KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__));
35956a4e45aSAndre Oppermann 	KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__));
360121f0509SMike Silbersack #ifdef INVARIANTS
361121f0509SMike Silbersack 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
362121f0509SMike Silbersack #endif
363099a0e58SBosko Milekic }
364099a0e58SBosko Milekic 
365099a0e58SBosko Milekic /*
366ec63cb90SAndre Oppermann  * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor.
367099a0e58SBosko Milekic  *
368099a0e58SBosko Milekic  * Here the 'arg' pointer points to the Mbuf which we
36956a4e45aSAndre Oppermann  * are configuring cluster storage for.  If 'arg' is
37056a4e45aSAndre Oppermann  * empty we allocate just the cluster without setting
37156a4e45aSAndre Oppermann  * the mbuf to it.  See mbuf.h.
372099a0e58SBosko Milekic  */
373b23f72e9SBrian Feldman static int
374b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how)
375099a0e58SBosko Milekic {
376099a0e58SBosko Milekic 	struct mbuf *m;
37756a4e45aSAndre Oppermann 	u_int *refcnt;
37856a4e45aSAndre Oppermann 	int type = 0;
379099a0e58SBosko Milekic 
380121f0509SMike Silbersack #ifdef INVARIANTS
381121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
382121f0509SMike Silbersack #endif
383099a0e58SBosko Milekic 	m = (struct mbuf *)arg;
38456a4e45aSAndre Oppermann 	if (m != NULL) {
38556a4e45aSAndre Oppermann 		switch (size) {
38656a4e45aSAndre Oppermann 		case MCLBYTES:
38756a4e45aSAndre Oppermann 			type = EXT_CLUSTER;
38856a4e45aSAndre Oppermann 			break;
389ec63cb90SAndre Oppermann #if MJUMPAGESIZE != MCLBYTES
390ec63cb90SAndre Oppermann 		case MJUMPAGESIZE:
391ec63cb90SAndre Oppermann 			type = EXT_JUMBOP;
392d5269a63SAndre Oppermann 			break;
39336ae3fd3SAndre Oppermann #endif
39456a4e45aSAndre Oppermann 		case MJUM9BYTES:
39556a4e45aSAndre Oppermann 			type = EXT_JUMBO9;
39656a4e45aSAndre Oppermann 			break;
39756a4e45aSAndre Oppermann 		case MJUM16BYTES:
39856a4e45aSAndre Oppermann 			type = EXT_JUMBO16;
39956a4e45aSAndre Oppermann 			break;
40056a4e45aSAndre Oppermann 		default:
40156a4e45aSAndre Oppermann 			panic("unknown cluster size");
40256a4e45aSAndre Oppermann 			break;
40356a4e45aSAndre Oppermann 		}
404099a0e58SBosko Milekic 		m->m_ext.ext_buf = (caddr_t)mem;
405099a0e58SBosko Milekic 		m->m_data = m->m_ext.ext_buf;
406099a0e58SBosko Milekic 		m->m_flags |= M_EXT;
407099a0e58SBosko Milekic 		m->m_ext.ext_free = NULL;
408099a0e58SBosko Milekic 		m->m_ext.ext_args = NULL;
40956a4e45aSAndre Oppermann 		m->m_ext.ext_size = size;
41056a4e45aSAndre Oppermann 		m->m_ext.ext_type = type;
41156a4e45aSAndre Oppermann 		m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem);
41256a4e45aSAndre Oppermann 		*m->m_ext.ref_cnt = 1;
41356a4e45aSAndre Oppermann 	} else {
41456a4e45aSAndre Oppermann 		refcnt =  uma_find_refcnt(zone_clust, mem);
41556a4e45aSAndre Oppermann 		*refcnt = 1;
41656a4e45aSAndre Oppermann 	}
417b23f72e9SBrian Feldman 	return (0);
418099a0e58SBosko Milekic }
419099a0e58SBosko Milekic 
42056a4e45aSAndre Oppermann /*
42156a4e45aSAndre Oppermann  * The Mbuf Cluster zone destructor.
42256a4e45aSAndre Oppermann  */
423099a0e58SBosko Milekic static void
424099a0e58SBosko Milekic mb_dtor_clust(void *mem, int size, void *arg)
425099a0e58SBosko Milekic {
42656a4e45aSAndre Oppermann 
427a5f77087SAndre Oppermann 	KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1,
428a5f77087SAndre Oppermann 		("%s: refcnt incorrect %u", __func__,
429a5f77087SAndre Oppermann 		 *(uma_find_refcnt(zone_clust, mem))) );
430121f0509SMike Silbersack #ifdef INVARIANTS
431121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
432121f0509SMike Silbersack #endif
433099a0e58SBosko Milekic }
434099a0e58SBosko Milekic 
435099a0e58SBosko Milekic /*
436099a0e58SBosko Milekic  * The Packet secondary zone's init routine, executed on the
43756a4e45aSAndre Oppermann  * object's transition from mbuf keg slab to zone cache.
438099a0e58SBosko Milekic  */
439b23f72e9SBrian Feldman static int
44056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how)
441099a0e58SBosko Milekic {
442099a0e58SBosko Milekic 	struct mbuf *m;
443099a0e58SBosko Milekic 
44456a4e45aSAndre Oppermann 	m = (struct mbuf *)mem;		/* m is virgin. */
445fd241339SAndre Oppermann 	(void)uma_zalloc_arg(zone_clust, m, how);
446b23f72e9SBrian Feldman 	if (m->m_ext.ext_buf == NULL)
447b23f72e9SBrian Feldman 		return (ENOMEM);
448cd5bb63bSAndre Oppermann 	m->m_ext.ext_type = EXT_PACKET;	/* Override. */
449121f0509SMike Silbersack #ifdef INVARIANTS
450121f0509SMike Silbersack 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
451121f0509SMike Silbersack #endif
452b23f72e9SBrian Feldman 	return (0);
453099a0e58SBosko Milekic }
454099a0e58SBosko Milekic 
455099a0e58SBosko Milekic /*
456099a0e58SBosko Milekic  * The Packet secondary zone's fini routine, executed on the
457099a0e58SBosko Milekic  * object's transition from zone cache to keg slab.
458099a0e58SBosko Milekic  */
459099a0e58SBosko Milekic static void
46056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size)
461099a0e58SBosko Milekic {
462099a0e58SBosko Milekic 	struct mbuf *m;
463099a0e58SBosko Milekic 
464099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
465121f0509SMike Silbersack #ifdef INVARIANTS
466121f0509SMike Silbersack 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
467121f0509SMike Silbersack #endif
468099a0e58SBosko Milekic 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
469a7b844d2SMike Silbersack #ifdef INVARIANTS
470a7b844d2SMike Silbersack 	trash_dtor(mem, size, NULL);
471a7b844d2SMike Silbersack #endif
472099a0e58SBosko Milekic }
473099a0e58SBosko Milekic 
474099a0e58SBosko Milekic /*
475099a0e58SBosko Milekic  * The "packet" keg constructor.
476099a0e58SBosko Milekic  */
477b23f72e9SBrian Feldman static int
478b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how)
479099a0e58SBosko Milekic {
480099a0e58SBosko Milekic 	struct mbuf *m;
481099a0e58SBosko Milekic 	struct mb_args *args;
482b23f72e9SBrian Feldman #ifdef MAC
483b23f72e9SBrian Feldman 	int error;
484b23f72e9SBrian Feldman #endif
485b23f72e9SBrian Feldman 	int flags;
486099a0e58SBosko Milekic 	short type;
487099a0e58SBosko Milekic 
488099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
489099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
490099a0e58SBosko Milekic 	flags = args->flags;
491099a0e58SBosko Milekic 	type = args->type;
492099a0e58SBosko Milekic 
493121f0509SMike Silbersack #ifdef INVARIANTS
494121f0509SMike Silbersack 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
495121f0509SMike Silbersack #endif
496099a0e58SBosko Milekic 	m->m_next = NULL;
4976bc72ab9SBosko Milekic 	m->m_nextpkt = NULL;
498099a0e58SBosko Milekic 	m->m_data = m->m_ext.ext_buf;
49956a4e45aSAndre Oppermann 	m->m_len = 0;
50056a4e45aSAndre Oppermann 	m->m_flags = (flags | M_EXT);
50156a4e45aSAndre Oppermann 	m->m_type = type;
502099a0e58SBosko Milekic 
503099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
504099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
50556a4e45aSAndre Oppermann 		m->m_pkthdr.len = 0;
50656a4e45aSAndre Oppermann 		m->m_pkthdr.header = NULL;
507099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
50856a4e45aSAndre Oppermann 		m->m_pkthdr.csum_data = 0;
509099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
510099a0e58SBosko Milekic #ifdef MAC
511099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
512b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
513b23f72e9SBrian Feldman 		if (error)
514b23f72e9SBrian Feldman 			return (error);
515099a0e58SBosko Milekic #endif
516099a0e58SBosko Milekic 	}
51756a4e45aSAndre Oppermann 	/* m_ext is already initialized. */
51856a4e45aSAndre Oppermann 
519b23f72e9SBrian Feldman 	return (0);
520099a0e58SBosko Milekic }
521099a0e58SBosko Milekic 
52275ee267cSGleb Smirnoff static void
52375ee267cSGleb Smirnoff mt_vlan_free(struct m_tag *mtag)
52475ee267cSGleb Smirnoff {
52575ee267cSGleb Smirnoff 	uma_zfree(zone_mtag_vlan, mtag);
52675ee267cSGleb Smirnoff }
52775ee267cSGleb Smirnoff 
52875ee267cSGleb Smirnoff static int
52975ee267cSGleb Smirnoff mt_zinit_vlan(void *mem, int size, int how)
53075ee267cSGleb Smirnoff {
53175ee267cSGleb Smirnoff 	struct m_tag *mtag = (struct m_tag *)mem;
53275ee267cSGleb Smirnoff 
53375ee267cSGleb Smirnoff 	m_tag_setup(mtag, MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int));
53475ee267cSGleb Smirnoff 	mtag->m_tag_free = mt_vlan_free;
53575ee267cSGleb Smirnoff 
53675ee267cSGleb Smirnoff 	return (0);
53775ee267cSGleb Smirnoff }
53875ee267cSGleb Smirnoff 
539099a0e58SBosko Milekic /*
540099a0e58SBosko Milekic  * This is the protocol drain routine.
541099a0e58SBosko Milekic  *
542099a0e58SBosko Milekic  * No locks should be held when this is called.  The drain routines have to
543099a0e58SBosko Milekic  * presently acquire some locks which raises the possibility of lock order
544099a0e58SBosko Milekic  * reversal.
545099a0e58SBosko Milekic  */
546099a0e58SBosko Milekic static void
547099a0e58SBosko Milekic mb_reclaim(void *junk)
548099a0e58SBosko Milekic {
549099a0e58SBosko Milekic 	struct domain *dp;
550099a0e58SBosko Milekic 	struct protosw *pr;
551099a0e58SBosko Milekic 
552099a0e58SBosko Milekic 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
553099a0e58SBosko Milekic 	    "mb_reclaim()");
554099a0e58SBosko Milekic 
555099a0e58SBosko Milekic 	for (dp = domains; dp != NULL; dp = dp->dom_next)
556099a0e58SBosko Milekic 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
557099a0e58SBosko Milekic 			if (pr->pr_drain != NULL)
558099a0e58SBosko Milekic 				(*pr->pr_drain)();
559099a0e58SBosko Milekic }
560