xref: /freebsd/sys/kern/kern_mbuf.c (revision 68352adfe7e035ea57d958a14149583630edff23)
1099a0e58SBosko Milekic /*-
28076cb52SBosko Milekic  * Copyright (c) 2004, 2005,
38076cb52SBosko Milekic  * 	Bosko Milekic <bmilekic@FreeBSD.org>.  All rights reserved.
4099a0e58SBosko Milekic  *
5099a0e58SBosko Milekic  * Redistribution and use in source and binary forms, with or without
6099a0e58SBosko Milekic  * modification, are permitted provided that the following conditions
7099a0e58SBosko Milekic  * are met:
8099a0e58SBosko Milekic  * 1. Redistributions of source code must retain the above copyright
9099a0e58SBosko Milekic  *    notice unmodified, this list of conditions and the following
10099a0e58SBosko Milekic  *    disclaimer.
11099a0e58SBosko Milekic  * 2. Redistributions in binary form must reproduce the above copyright
12099a0e58SBosko Milekic  *    notice, this list of conditions and the following disclaimer in the
13099a0e58SBosko Milekic  *    documentation and/or other materials provided with the distribution.
14099a0e58SBosko Milekic  *
15099a0e58SBosko Milekic  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16099a0e58SBosko Milekic  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17099a0e58SBosko Milekic  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18099a0e58SBosko Milekic  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19099a0e58SBosko Milekic  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20099a0e58SBosko Milekic  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21099a0e58SBosko Milekic  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22099a0e58SBosko Milekic  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23099a0e58SBosko Milekic  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24099a0e58SBosko Milekic  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25099a0e58SBosko Milekic  * SUCH DAMAGE.
26099a0e58SBosko Milekic  */
27099a0e58SBosko Milekic 
28099a0e58SBosko Milekic #include <sys/cdefs.h>
29099a0e58SBosko Milekic __FBSDID("$FreeBSD$");
30099a0e58SBosko Milekic 
31099a0e58SBosko Milekic #include "opt_mac.h"
32099a0e58SBosko Milekic #include "opt_param.h"
33099a0e58SBosko Milekic 
34099a0e58SBosko Milekic #include <sys/param.h>
35099a0e58SBosko Milekic #include <sys/mac.h>
36099a0e58SBosko Milekic #include <sys/malloc.h>
37099a0e58SBosko Milekic #include <sys/systm.h>
38099a0e58SBosko Milekic #include <sys/mbuf.h>
39099a0e58SBosko Milekic #include <sys/domain.h>
40099a0e58SBosko Milekic #include <sys/eventhandler.h>
41099a0e58SBosko Milekic #include <sys/kernel.h>
42099a0e58SBosko Milekic #include <sys/protosw.h>
43099a0e58SBosko Milekic #include <sys/smp.h>
44099a0e58SBosko Milekic #include <sys/sysctl.h>
45099a0e58SBosko Milekic 
46099a0e58SBosko Milekic #include <vm/vm.h>
47099a0e58SBosko Milekic #include <vm/vm_page.h>
48099a0e58SBosko Milekic #include <vm/uma.h>
49121f0509SMike Silbersack #include <vm/uma_int.h>
50121f0509SMike Silbersack #include <vm/uma_dbg.h>
51099a0e58SBosko Milekic 
52099a0e58SBosko Milekic /*
53099a0e58SBosko Milekic  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
54099a0e58SBosko Milekic  * Zones.
55099a0e58SBosko Milekic  *
56099a0e58SBosko Milekic  * Mbuf Clusters (2K, contiguous) are allocated from the Cluster
57099a0e58SBosko Milekic  * Zone.  The Zone can be capped at kern.ipc.nmbclusters, if the
58099a0e58SBosko Milekic  * administrator so desires.
59099a0e58SBosko Milekic  *
60099a0e58SBosko Milekic  * Mbufs are allocated from a UMA Master Zone called the Mbuf
61099a0e58SBosko Milekic  * Zone.
62099a0e58SBosko Milekic  *
63099a0e58SBosko Milekic  * Additionally, FreeBSD provides a Packet Zone, which it
64099a0e58SBosko Milekic  * configures as a Secondary Zone to the Mbuf Master Zone,
65099a0e58SBosko Milekic  * thus sharing backend Slab kegs with the Mbuf Master Zone.
66099a0e58SBosko Milekic  *
67099a0e58SBosko Milekic  * Thus common-case allocations and locking are simplified:
68099a0e58SBosko Milekic  *
69099a0e58SBosko Milekic  *  m_clget()                m_getcl()
70099a0e58SBosko Milekic  *    |                         |
71099a0e58SBosko Milekic  *    |   .------------>[(Packet Cache)]    m_get(), m_gethdr()
72099a0e58SBosko Milekic  *    |   |             [     Packet   ]            |
73099a0e58SBosko Milekic  *  [(Cluster Cache)]   [    Secondary ]   [ (Mbuf Cache)     ]
74099a0e58SBosko Milekic  *  [ Cluster Zone  ]   [     Zone     ]   [ Mbuf Master Zone ]
75099a0e58SBosko Milekic  *        |                       \________         |
76099a0e58SBosko Milekic  *  [ Cluster Keg   ]                      \       /
77099a0e58SBosko Milekic  *        |    	                         [ Mbuf Keg   ]
78099a0e58SBosko Milekic  *  [ Cluster Slabs ]                         |
79099a0e58SBosko Milekic  *        |                              [ Mbuf Slabs ]
80099a0e58SBosko Milekic  *         \____________(VM)_________________/
81099a0e58SBosko Milekic  */
82099a0e58SBosko Milekic 
83099a0e58SBosko Milekic int nmbclusters;
84099a0e58SBosko Milekic struct mbstat mbstat;
85099a0e58SBosko Milekic 
86099a0e58SBosko Milekic static void
87099a0e58SBosko Milekic tunable_mbinit(void *dummy)
88099a0e58SBosko Milekic {
89099a0e58SBosko Milekic 
90099a0e58SBosko Milekic 	/* This has to be done before VM init. */
91099a0e58SBosko Milekic 	nmbclusters = 1024 + maxusers * 64;
92099a0e58SBosko Milekic 	TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters);
93099a0e58SBosko Milekic }
94099a0e58SBosko Milekic SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL);
95099a0e58SBosko Milekic 
96099a0e58SBosko Milekic SYSCTL_DECL(_kern_ipc);
97099a0e58SBosko Milekic SYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RW, &nmbclusters, 0,
98099a0e58SBosko Milekic     "Maximum number of mbuf clusters allowed");
99099a0e58SBosko Milekic SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat,
100099a0e58SBosko Milekic     "Mbuf general information and statistics");
101099a0e58SBosko Milekic 
102099a0e58SBosko Milekic /*
103099a0e58SBosko Milekic  * Zones from which we allocate.
104099a0e58SBosko Milekic  */
105099a0e58SBosko Milekic uma_zone_t	zone_mbuf;
106099a0e58SBosko Milekic uma_zone_t	zone_clust;
107099a0e58SBosko Milekic uma_zone_t	zone_pack;
108099a0e58SBosko Milekic 
109099a0e58SBosko Milekic /*
110099a0e58SBosko Milekic  * Local prototypes.
111099a0e58SBosko Milekic  */
112b23f72e9SBrian Feldman static int	mb_ctor_mbuf(void *, int, void *, int);
113b23f72e9SBrian Feldman static int	mb_ctor_clust(void *, int, void *, int);
114b23f72e9SBrian Feldman static int	mb_ctor_pack(void *, int, void *, int);
115099a0e58SBosko Milekic static void	mb_dtor_mbuf(void *, int, void *);
116099a0e58SBosko Milekic static void	mb_dtor_clust(void *, int, void *);	/* XXX */
117099a0e58SBosko Milekic static void	mb_dtor_pack(void *, int, void *);	/* XXX */
118b23f72e9SBrian Feldman static int	mb_init_pack(void *, int, int);
119099a0e58SBosko Milekic static void	mb_fini_pack(void *, int);
120099a0e58SBosko Milekic 
121099a0e58SBosko Milekic static void	mb_reclaim(void *);
122099a0e58SBosko Milekic static void	mbuf_init(void *);
123099a0e58SBosko Milekic 
124a04946cfSBrian Somers /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */
125a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
126a04946cfSBrian Somers 
127099a0e58SBosko Milekic /*
128099a0e58SBosko Milekic  * Initialize FreeBSD Network buffer allocation.
129099a0e58SBosko Milekic  */
130099a0e58SBosko Milekic SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL)
131099a0e58SBosko Milekic static void
132099a0e58SBosko Milekic mbuf_init(void *dummy)
133099a0e58SBosko Milekic {
134099a0e58SBosko Milekic 
135099a0e58SBosko Milekic 	/*
136099a0e58SBosko Milekic 	 * Configure UMA zones for Mbufs, Clusters, and Packets.
137099a0e58SBosko Milekic 	 */
13868352adfSRobert Watson 	zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, mb_ctor_mbuf,
13968352adfSRobert Watson 	    mb_dtor_mbuf,
140121f0509SMike Silbersack #ifdef INVARIANTS
141121f0509SMike Silbersack 	    trash_init, trash_fini, MSIZE - 1, UMA_ZONE_MAXBUCKET);
142121f0509SMike Silbersack #else
143a04946cfSBrian Somers 	    NULL, NULL, MSIZE - 1, UMA_ZONE_MAXBUCKET);
144121f0509SMike Silbersack #endif
14568352adfSRobert Watson 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
14668352adfSRobert Watson 	    mb_ctor_clust,
147121f0509SMike Silbersack #ifdef INVARIANTS
148121f0509SMike Silbersack 	    mb_dtor_clust, trash_init, trash_fini, UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
149121f0509SMike Silbersack #else
150099a0e58SBosko Milekic 	    mb_dtor_clust, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_REFCNT);
151121f0509SMike Silbersack #endif
152099a0e58SBosko Milekic 	if (nmbclusters > 0)
153099a0e58SBosko Milekic 		uma_zone_set_max(zone_clust, nmbclusters);
15468352adfSRobert Watson 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
15568352adfSRobert Watson 	    mb_dtor_pack, mb_init_pack, mb_fini_pack, zone_mbuf);
156099a0e58SBosko Milekic 
157099a0e58SBosko Milekic 	/* uma_prealloc() goes here */
158099a0e58SBosko Milekic 
159099a0e58SBosko Milekic 	/*
160099a0e58SBosko Milekic 	 * Hook event handler for low-memory situation, used to
161099a0e58SBosko Milekic 	 * drain protocols and push data back to the caches (UMA
162099a0e58SBosko Milekic 	 * later pushes it back to VM).
163099a0e58SBosko Milekic 	 */
164099a0e58SBosko Milekic 	EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL,
165099a0e58SBosko Milekic 	    EVENTHANDLER_PRI_FIRST);
166099a0e58SBosko Milekic 
167099a0e58SBosko Milekic 	/*
168099a0e58SBosko Milekic 	 * [Re]set counters and local statistics knobs.
169099a0e58SBosko Milekic 	 * XXX Some of these should go and be replaced, but UMA stat
170099a0e58SBosko Milekic 	 * gathering needs to be revised.
171099a0e58SBosko Milekic 	 */
172099a0e58SBosko Milekic 	mbstat.m_mbufs = 0;
173099a0e58SBosko Milekic 	mbstat.m_mclusts = 0;
174099a0e58SBosko Milekic 	mbstat.m_drain = 0;
175099a0e58SBosko Milekic 	mbstat.m_msize = MSIZE;
176099a0e58SBosko Milekic 	mbstat.m_mclbytes = MCLBYTES;
177099a0e58SBosko Milekic 	mbstat.m_minclsize = MINCLSIZE;
178099a0e58SBosko Milekic 	mbstat.m_mlen = MLEN;
179099a0e58SBosko Milekic 	mbstat.m_mhlen = MHLEN;
180099a0e58SBosko Milekic 	mbstat.m_numtypes = MT_NTYPES;
181099a0e58SBosko Milekic 
182099a0e58SBosko Milekic 	mbstat.m_mcfail = mbstat.m_mpfail = 0;
183099a0e58SBosko Milekic 	mbstat.sf_iocnt = 0;
184099a0e58SBosko Milekic 	mbstat.sf_allocwait = mbstat.sf_allocfail = 0;
185099a0e58SBosko Milekic }
186099a0e58SBosko Milekic 
187099a0e58SBosko Milekic /*
188099a0e58SBosko Milekic  * Constructor for Mbuf master zone.
189099a0e58SBosko Milekic  *
190099a0e58SBosko Milekic  * The 'arg' pointer points to a mb_args structure which
191099a0e58SBosko Milekic  * contains call-specific information required to support the
192099a0e58SBosko Milekic  * mbuf allocation API.
193099a0e58SBosko Milekic  */
194b23f72e9SBrian Feldman static int
195b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how)
196099a0e58SBosko Milekic {
197099a0e58SBosko Milekic 	struct mbuf *m;
198099a0e58SBosko Milekic 	struct mb_args *args;
199b23f72e9SBrian Feldman #ifdef MAC
200b23f72e9SBrian Feldman 	int error;
201b23f72e9SBrian Feldman #endif
202099a0e58SBosko Milekic 	int flags;
203099a0e58SBosko Milekic 	short type;
204099a0e58SBosko Milekic 
205121f0509SMike Silbersack #ifdef INVARIANTS
206121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
207121f0509SMike Silbersack #endif
208099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
209099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
210099a0e58SBosko Milekic 	flags = args->flags;
211099a0e58SBosko Milekic 	type = args->type;
212099a0e58SBosko Milekic 
213099a0e58SBosko Milekic 	m->m_type = type;
214099a0e58SBosko Milekic 	m->m_next = NULL;
215099a0e58SBosko Milekic 	m->m_nextpkt = NULL;
2166bc72ab9SBosko Milekic 	m->m_flags = flags;
217099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
218099a0e58SBosko Milekic 		m->m_data = m->m_pktdat;
219099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
220099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
221099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
222099a0e58SBosko Milekic #ifdef MAC
223099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
224b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
225b23f72e9SBrian Feldman 		if (error)
226b23f72e9SBrian Feldman 			return (error);
227099a0e58SBosko Milekic #endif
2286bc72ab9SBosko Milekic 	} else
229099a0e58SBosko Milekic 		m->m_data = m->m_dat;
230099a0e58SBosko Milekic 	mbstat.m_mbufs += 1;	/* XXX */
231b23f72e9SBrian Feldman 	return (0);
232099a0e58SBosko Milekic }
233099a0e58SBosko Milekic 
234099a0e58SBosko Milekic /*
235099a0e58SBosko Milekic  * The Mbuf master zone and Packet secondary zone destructor.
236099a0e58SBosko Milekic  */
237099a0e58SBosko Milekic static void
238099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg)
239099a0e58SBosko Milekic {
240099a0e58SBosko Milekic 	struct mbuf *m;
241099a0e58SBosko Milekic 
242099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
243099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
244099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
245121f0509SMike Silbersack #ifdef INVARIANTS
246121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
247121f0509SMike Silbersack #endif
248099a0e58SBosko Milekic 	mbstat.m_mbufs -= 1;	/* XXX */
249099a0e58SBosko Milekic }
250099a0e58SBosko Milekic 
251099a0e58SBosko Milekic /* XXX Only because of stats */
252099a0e58SBosko Milekic static void
253099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg)
254099a0e58SBosko Milekic {
255099a0e58SBosko Milekic 	struct mbuf *m;
256099a0e58SBosko Milekic 
257099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
258099a0e58SBosko Milekic 	if ((m->m_flags & M_PKTHDR) != 0)
259099a0e58SBosko Milekic 		m_tag_delete_chain(m, NULL);
260121f0509SMike Silbersack #ifdef INVARIANTS
261121f0509SMike Silbersack 	trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg);
262121f0509SMike Silbersack #endif
263099a0e58SBosko Milekic 	mbstat.m_mbufs -= 1;	/* XXX */
264099a0e58SBosko Milekic 	mbstat.m_mclusts -= 1;	/* XXX */
265099a0e58SBosko Milekic }
266099a0e58SBosko Milekic 
267099a0e58SBosko Milekic /*
268099a0e58SBosko Milekic  * The Cluster zone constructor.
269099a0e58SBosko Milekic  *
270099a0e58SBosko Milekic  * Here the 'arg' pointer points to the Mbuf which we
271099a0e58SBosko Milekic  * are configuring cluster storage for.
272099a0e58SBosko Milekic  */
273b23f72e9SBrian Feldman static int
274b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how)
275099a0e58SBosko Milekic {
276099a0e58SBosko Milekic 	struct mbuf *m;
277099a0e58SBosko Milekic 
278121f0509SMike Silbersack #ifdef INVARIANTS
279121f0509SMike Silbersack 	trash_ctor(mem, size, arg, how);
280121f0509SMike Silbersack #endif
281099a0e58SBosko Milekic 	m = (struct mbuf *)arg;
282099a0e58SBosko Milekic 	m->m_ext.ext_buf = (caddr_t)mem;
283099a0e58SBosko Milekic 	m->m_data = m->m_ext.ext_buf;
284099a0e58SBosko Milekic 	m->m_flags |= M_EXT;
285099a0e58SBosko Milekic 	m->m_ext.ext_free = NULL;
286099a0e58SBosko Milekic 	m->m_ext.ext_args = NULL;
287099a0e58SBosko Milekic 	m->m_ext.ext_size = MCLBYTES;
288099a0e58SBosko Milekic 	m->m_ext.ext_type = EXT_CLUSTER;
2893d2a3ff2SBosko Milekic 	m->m_ext.ref_cnt = NULL;	/* Lazy counter assign. */
290099a0e58SBosko Milekic 	mbstat.m_mclusts += 1;	/* XXX */
291b23f72e9SBrian Feldman 	return (0);
292099a0e58SBosko Milekic }
293099a0e58SBosko Milekic 
294099a0e58SBosko Milekic /* XXX */
295099a0e58SBosko Milekic static void
296099a0e58SBosko Milekic mb_dtor_clust(void *mem, int size, void *arg)
297099a0e58SBosko Milekic {
298121f0509SMike Silbersack #ifdef INVARIANTS
299121f0509SMike Silbersack 	trash_dtor(mem, size, arg);
300121f0509SMike Silbersack #endif
301099a0e58SBosko Milekic 	mbstat.m_mclusts -= 1;	/* XXX */
302099a0e58SBosko Milekic }
303099a0e58SBosko Milekic 
304099a0e58SBosko Milekic /*
305099a0e58SBosko Milekic  * The Packet secondary zone's init routine, executed on the
306099a0e58SBosko Milekic  * object's transition from keg slab to zone cache.
307099a0e58SBosko Milekic  */
308b23f72e9SBrian Feldman static int
309b23f72e9SBrian Feldman mb_init_pack(void *mem, int size, int how)
310099a0e58SBosko Milekic {
311099a0e58SBosko Milekic 	struct mbuf *m;
312099a0e58SBosko Milekic 
313099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
314099a0e58SBosko Milekic 	m->m_ext.ext_buf = NULL;
315b23f72e9SBrian Feldman 	uma_zalloc_arg(zone_clust, m, how);
316b23f72e9SBrian Feldman 	if (m->m_ext.ext_buf == NULL)
317b23f72e9SBrian Feldman 		return (ENOMEM);
318121f0509SMike Silbersack #ifdef INVARIANTS
319121f0509SMike Silbersack 	trash_init(m->m_ext.ext_buf, MCLBYTES, how);
320121f0509SMike Silbersack #endif
321099a0e58SBosko Milekic 	mbstat.m_mclusts -= 1;	/* XXX */
322b23f72e9SBrian Feldman 	return (0);
323099a0e58SBosko Milekic }
324099a0e58SBosko Milekic 
325099a0e58SBosko Milekic /*
326099a0e58SBosko Milekic  * The Packet secondary zone's fini routine, executed on the
327099a0e58SBosko Milekic  * object's transition from zone cache to keg slab.
328099a0e58SBosko Milekic  */
329099a0e58SBosko Milekic static void
330099a0e58SBosko Milekic mb_fini_pack(void *mem, int size)
331099a0e58SBosko Milekic {
332099a0e58SBosko Milekic 	struct mbuf *m;
333099a0e58SBosko Milekic 
334099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
335121f0509SMike Silbersack #ifdef INVARIANTS
336121f0509SMike Silbersack 	trash_fini(m->m_ext.ext_buf, MCLBYTES);
337121f0509SMike Silbersack #endif
338099a0e58SBosko Milekic 	uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL);
339099a0e58SBosko Milekic 	m->m_ext.ext_buf = NULL;
340099a0e58SBosko Milekic 	mbstat.m_mclusts += 1;	/* XXX */
341a7b844d2SMike Silbersack #ifdef INVARIANTS
342a7b844d2SMike Silbersack 	trash_dtor(mem, size, NULL);
343a7b844d2SMike Silbersack #endif
344099a0e58SBosko Milekic }
345099a0e58SBosko Milekic 
346099a0e58SBosko Milekic /*
347099a0e58SBosko Milekic  * The "packet" keg constructor.
348099a0e58SBosko Milekic  */
349b23f72e9SBrian Feldman static int
350b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how)
351099a0e58SBosko Milekic {
352099a0e58SBosko Milekic 	struct mbuf *m;
353099a0e58SBosko Milekic 	struct mb_args *args;
354b23f72e9SBrian Feldman #ifdef MAC
355b23f72e9SBrian Feldman 	int error;
356b23f72e9SBrian Feldman #endif
357b23f72e9SBrian Feldman 	int flags;
358099a0e58SBosko Milekic 	short type;
359099a0e58SBosko Milekic 
360099a0e58SBosko Milekic 	m = (struct mbuf *)mem;
361099a0e58SBosko Milekic 	args = (struct mb_args *)arg;
362099a0e58SBosko Milekic 	flags = args->flags;
363099a0e58SBosko Milekic 	type = args->type;
364099a0e58SBosko Milekic 
365121f0509SMike Silbersack #ifdef INVARIANTS
366121f0509SMike Silbersack 	trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how);
367121f0509SMike Silbersack #endif
368099a0e58SBosko Milekic 	m->m_type = type;
369099a0e58SBosko Milekic 	m->m_next = NULL;
3706bc72ab9SBosko Milekic 	m->m_nextpkt = NULL;
371099a0e58SBosko Milekic 	m->m_data = m->m_ext.ext_buf;
372099a0e58SBosko Milekic 	m->m_flags = flags|M_EXT;
373099a0e58SBosko Milekic 	m->m_ext.ext_free = NULL;
374099a0e58SBosko Milekic 	m->m_ext.ext_args = NULL;
375099a0e58SBosko Milekic 	m->m_ext.ext_size = MCLBYTES;
376099a0e58SBosko Milekic 	m->m_ext.ext_type = EXT_PACKET;
3773d2a3ff2SBosko Milekic 	m->m_ext.ref_cnt = NULL;	/* Lazy counter assign. */
378099a0e58SBosko Milekic 
379099a0e58SBosko Milekic 	if (flags & M_PKTHDR) {
380099a0e58SBosko Milekic 		m->m_pkthdr.rcvif = NULL;
381099a0e58SBosko Milekic 		m->m_pkthdr.csum_flags = 0;
382099a0e58SBosko Milekic 		SLIST_INIT(&m->m_pkthdr.tags);
383099a0e58SBosko Milekic #ifdef MAC
384099a0e58SBosko Milekic 		/* If the label init fails, fail the alloc */
385b23f72e9SBrian Feldman 		error = mac_init_mbuf(m, how);
386b23f72e9SBrian Feldman 		if (error)
387b23f72e9SBrian Feldman 			return (error);
388099a0e58SBosko Milekic #endif
389099a0e58SBosko Milekic 	}
390099a0e58SBosko Milekic 	mbstat.m_mbufs += 1;	/* XXX */
391099a0e58SBosko Milekic 	mbstat.m_mclusts += 1;	/* XXX */
392b23f72e9SBrian Feldman 	return (0);
393099a0e58SBosko Milekic }
394099a0e58SBosko Milekic 
395099a0e58SBosko Milekic /*
396099a0e58SBosko Milekic  * This is the protocol drain routine.
397099a0e58SBosko Milekic  *
398099a0e58SBosko Milekic  * No locks should be held when this is called.  The drain routines have to
399099a0e58SBosko Milekic  * presently acquire some locks which raises the possibility of lock order
400099a0e58SBosko Milekic  * reversal.
401099a0e58SBosko Milekic  */
402099a0e58SBosko Milekic static void
403099a0e58SBosko Milekic mb_reclaim(void *junk)
404099a0e58SBosko Milekic {
405099a0e58SBosko Milekic 	struct domain *dp;
406099a0e58SBosko Milekic 	struct protosw *pr;
407099a0e58SBosko Milekic 
408099a0e58SBosko Milekic 	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
409099a0e58SBosko Milekic 	    "mb_reclaim()");
410099a0e58SBosko Milekic 
411099a0e58SBosko Milekic 	mbstat.m_drain++;
412099a0e58SBosko Milekic 	for (dp = domains; dp != NULL; dp = dp->dom_next)
413099a0e58SBosko Milekic 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
414099a0e58SBosko Milekic 			if (pr->pr_drain != NULL)
415099a0e58SBosko Milekic 				(*pr->pr_drain)();
416099a0e58SBosko Milekic }
417