1099a0e58SBosko Milekic /*- 28076cb52SBosko Milekic * Copyright (c) 2004, 2005, 38076cb52SBosko Milekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4099a0e58SBosko Milekic * 5099a0e58SBosko Milekic * Redistribution and use in source and binary forms, with or without 6099a0e58SBosko Milekic * modification, are permitted provided that the following conditions 7099a0e58SBosko Milekic * are met: 8099a0e58SBosko Milekic * 1. Redistributions of source code must retain the above copyright 9099a0e58SBosko Milekic * notice unmodified, this list of conditions and the following 10099a0e58SBosko Milekic * disclaimer. 11099a0e58SBosko Milekic * 2. Redistributions in binary form must reproduce the above copyright 12099a0e58SBosko Milekic * notice, this list of conditions and the following disclaimer in the 13099a0e58SBosko Milekic * documentation and/or other materials provided with the distribution. 14099a0e58SBosko Milekic * 15099a0e58SBosko Milekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16099a0e58SBosko Milekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17099a0e58SBosko Milekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18099a0e58SBosko Milekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19099a0e58SBosko Milekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20099a0e58SBosko Milekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21099a0e58SBosko Milekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22099a0e58SBosko Milekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23099a0e58SBosko Milekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24099a0e58SBosko Milekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25099a0e58SBosko Milekic * SUCH DAMAGE. 26099a0e58SBosko Milekic */ 27099a0e58SBosko Milekic 28099a0e58SBosko Milekic #include <sys/cdefs.h> 29099a0e58SBosko Milekic __FBSDID("$FreeBSD$"); 30099a0e58SBosko Milekic 31099a0e58SBosko Milekic #include "opt_mac.h" 32099a0e58SBosko Milekic #include "opt_param.h" 33099a0e58SBosko Milekic 34099a0e58SBosko Milekic #include <sys/param.h> 35099a0e58SBosko Milekic #include <sys/mac.h> 36099a0e58SBosko Milekic #include <sys/malloc.h> 37099a0e58SBosko Milekic #include <sys/systm.h> 38099a0e58SBosko Milekic #include <sys/mbuf.h> 39099a0e58SBosko Milekic #include <sys/domain.h> 40099a0e58SBosko Milekic #include <sys/eventhandler.h> 41099a0e58SBosko Milekic #include <sys/kernel.h> 42099a0e58SBosko Milekic #include <sys/protosw.h> 43099a0e58SBosko Milekic #include <sys/smp.h> 44099a0e58SBosko Milekic #include <sys/sysctl.h> 45099a0e58SBosko Milekic 46099a0e58SBosko Milekic #include <vm/vm.h> 47099a0e58SBosko Milekic #include <vm/vm_page.h> 48099a0e58SBosko Milekic #include <vm/uma.h> 49121f0509SMike Silbersack #include <vm/uma_int.h> 50121f0509SMike Silbersack #include <vm/uma_dbg.h> 51099a0e58SBosko Milekic 52099a0e58SBosko Milekic /* 53099a0e58SBosko Milekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 54099a0e58SBosko Milekic * Zones. 55099a0e58SBosko Milekic * 56099a0e58SBosko Milekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 57099a0e58SBosko Milekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 58099a0e58SBosko Milekic * administrator so desires. 59099a0e58SBosko Milekic * 60099a0e58SBosko Milekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 61099a0e58SBosko Milekic * Zone. 62099a0e58SBosko Milekic * 63099a0e58SBosko Milekic * Additionally, FreeBSD provides a Packet Zone, which it 64099a0e58SBosko Milekic * configures as a Secondary Zone to the Mbuf Master Zone, 65099a0e58SBosko Milekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 66099a0e58SBosko Milekic * 67099a0e58SBosko Milekic * Thus common-case allocations and locking are simplified: 68099a0e58SBosko Milekic * 69099a0e58SBosko Milekic * m_clget() m_getcl() 70099a0e58SBosko Milekic * | | 71099a0e58SBosko Milekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 72099a0e58SBosko Milekic * | | [ Packet ] | 73099a0e58SBosko Milekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 74099a0e58SBosko Milekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 75099a0e58SBosko Milekic * | \________ | 76099a0e58SBosko Milekic * [ Cluster Keg ] \ / 77099a0e58SBosko Milekic * | [ Mbuf Keg ] 78099a0e58SBosko Milekic * [ Cluster Slabs ] | 79099a0e58SBosko Milekic * | [ Mbuf Slabs ] 80099a0e58SBosko Milekic * \____________(VM)_________________/ 8156a4e45aSAndre Oppermann * 8256a4e45aSAndre Oppermann * 8356a4e45aSAndre Oppermann * Whenever a object is allocated with uma_zalloc() out of the 8456a4e45aSAndre Oppermann * one of the Zones its _ctor_ function is executed. The same 8556a4e45aSAndre Oppermann * for any deallocation through uma_zfree() the _dror_ function 8656a4e45aSAndre Oppermann * is executed. 8756a4e45aSAndre Oppermann * 8856a4e45aSAndre Oppermann * Caches are per-CPU and are filled from the Master Zone. 8956a4e45aSAndre Oppermann * 9056a4e45aSAndre Oppermann * Whenever a object is allocated from the underlying global 9156a4e45aSAndre Oppermann * memory pool it gets pre-initialized with the _zinit_ functions. 9256a4e45aSAndre Oppermann * When the Keg's are overfull objects get decomissioned with 9356a4e45aSAndre Oppermann * _zfini_ functions and free'd back to the global memory pool. 9456a4e45aSAndre Oppermann * 95099a0e58SBosko Milekic */ 96099a0e58SBosko Milekic 9756a4e45aSAndre Oppermann int nmbclusters; /* limits number of mbuf clusters */ 98ec63cb90SAndre Oppermann int nmbjumbop; /* limits number of page size jumbo clusters */ 9956a4e45aSAndre Oppermann int nmbjumbo9; /* limits number of 9k jumbo clusters */ 10056a4e45aSAndre Oppermann int nmbjumbo16; /* limits number of 16k jumbo clusters */ 101099a0e58SBosko Milekic struct mbstat mbstat; 102099a0e58SBosko Milekic 103099a0e58SBosko Milekic static void 104099a0e58SBosko Milekic tunable_mbinit(void *dummy) 105099a0e58SBosko Milekic { 106099a0e58SBosko Milekic 107099a0e58SBosko Milekic /* This has to be done before VM init. */ 108099a0e58SBosko Milekic nmbclusters = 1024 + maxusers * 64; 109099a0e58SBosko Milekic TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 110099a0e58SBosko Milekic } 111099a0e58SBosko Milekic SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL); 112099a0e58SBosko Milekic 113099a0e58SBosko Milekic SYSCTL_DECL(_kern_ipc); 11456a4e45aSAndre Oppermann /* XXX: These should be tuneables. Can't change UMA limits on the fly. */ 115099a0e58SBosko Milekic SYSCTL_INT(_kern_ipc, OID_AUTO, nmbclusters, CTLFLAG_RW, &nmbclusters, 0, 116099a0e58SBosko Milekic "Maximum number of mbuf clusters allowed"); 117ec63cb90SAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0, 118ec63cb90SAndre Oppermann "Maximum number of mbuf page size jumbo clusters allowed"); 11956a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0, 12056a4e45aSAndre Oppermann "Maximum number of mbuf 9k jumbo clusters allowed"); 12156a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0, 12256a4e45aSAndre Oppermann "Maximum number of mbuf 16k jumbo clusters allowed"); 123099a0e58SBosko Milekic SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat, 124099a0e58SBosko Milekic "Mbuf general information and statistics"); 125099a0e58SBosko Milekic 126099a0e58SBosko Milekic /* 127099a0e58SBosko Milekic * Zones from which we allocate. 128099a0e58SBosko Milekic */ 129099a0e58SBosko Milekic uma_zone_t zone_mbuf; 130099a0e58SBosko Milekic uma_zone_t zone_clust; 131099a0e58SBosko Milekic uma_zone_t zone_pack; 132ec63cb90SAndre Oppermann uma_zone_t zone_jumbop; 13356a4e45aSAndre Oppermann uma_zone_t zone_jumbo9; 13456a4e45aSAndre Oppermann uma_zone_t zone_jumbo16; 13556a4e45aSAndre Oppermann uma_zone_t zone_ext_refcnt; 13675ee267cSGleb Smirnoff uma_zone_t zone_mtag_vlan; 137099a0e58SBosko Milekic 138099a0e58SBosko Milekic /* 139099a0e58SBosko Milekic * Local prototypes. 140099a0e58SBosko Milekic */ 141b23f72e9SBrian Feldman static int mb_ctor_mbuf(void *, int, void *, int); 142b23f72e9SBrian Feldman static int mb_ctor_clust(void *, int, void *, int); 143b23f72e9SBrian Feldman static int mb_ctor_pack(void *, int, void *, int); 144099a0e58SBosko Milekic static void mb_dtor_mbuf(void *, int, void *); 14556a4e45aSAndre Oppermann static void mb_dtor_clust(void *, int, void *); 14656a4e45aSAndre Oppermann static void mb_dtor_pack(void *, int, void *); 14756a4e45aSAndre Oppermann static int mb_zinit_pack(void *, int, int); 14856a4e45aSAndre Oppermann static void mb_zfini_pack(void *, int); 14975ee267cSGleb Smirnoff static int mt_zinit_vlan(void *, int, int); 150099a0e58SBosko Milekic 151099a0e58SBosko Milekic static void mb_reclaim(void *); 152099a0e58SBosko Milekic static void mbuf_init(void *); 153099a0e58SBosko Milekic 154a04946cfSBrian Somers /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */ 155a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 156a04946cfSBrian Somers 157099a0e58SBosko Milekic /* 158099a0e58SBosko Milekic * Initialize FreeBSD Network buffer allocation. 159099a0e58SBosko Milekic */ 160099a0e58SBosko Milekic SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL) 161099a0e58SBosko Milekic static void 162099a0e58SBosko Milekic mbuf_init(void *dummy) 163099a0e58SBosko Milekic { 164099a0e58SBosko Milekic 165099a0e58SBosko Milekic /* 166099a0e58SBosko Milekic * Configure UMA zones for Mbufs, Clusters, and Packets. 167099a0e58SBosko Milekic */ 16856a4e45aSAndre Oppermann zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 16956a4e45aSAndre Oppermann mb_ctor_mbuf, mb_dtor_mbuf, 170121f0509SMike Silbersack #ifdef INVARIANTS 17156a4e45aSAndre Oppermann trash_init, trash_fini, 172121f0509SMike Silbersack #else 17356a4e45aSAndre Oppermann NULL, NULL, 174121f0509SMike Silbersack #endif 17556a4e45aSAndre Oppermann MSIZE - 1, UMA_ZONE_MAXBUCKET); 17656a4e45aSAndre Oppermann 17768352adfSRobert Watson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 17856a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 179121f0509SMike Silbersack #ifdef INVARIANTS 18056a4e45aSAndre Oppermann trash_init, trash_fini, 181121f0509SMike Silbersack #else 18256a4e45aSAndre Oppermann NULL, NULL, 183121f0509SMike Silbersack #endif 18456a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 185099a0e58SBosko Milekic if (nmbclusters > 0) 186099a0e58SBosko Milekic uma_zone_set_max(zone_clust, nmbclusters); 187099a0e58SBosko Milekic 18856a4e45aSAndre Oppermann zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 18956a4e45aSAndre Oppermann mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 19056a4e45aSAndre Oppermann 191d5269a63SAndre Oppermann /* Make jumbo frame zone too. 4k, 9k and 16k. */ 192ec63cb90SAndre Oppermann zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 193d5269a63SAndre Oppermann mb_ctor_clust, mb_dtor_clust, 194d5269a63SAndre Oppermann #ifdef INVARIANTS 195d5269a63SAndre Oppermann trash_init, trash_fini, 196d5269a63SAndre Oppermann #else 197d5269a63SAndre Oppermann NULL, NULL, 198d5269a63SAndre Oppermann #endif 199d5269a63SAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 200ec63cb90SAndre Oppermann if (nmbjumbop > 0) 201ec63cb90SAndre Oppermann uma_zone_set_max(zone_jumbop, nmbjumbop); 202d5269a63SAndre Oppermann 20356a4e45aSAndre Oppermann zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 20456a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 20556a4e45aSAndre Oppermann #ifdef INVARIANTS 20656a4e45aSAndre Oppermann trash_init, trash_fini, 20756a4e45aSAndre Oppermann #else 20856a4e45aSAndre Oppermann NULL, NULL, 20956a4e45aSAndre Oppermann #endif 21056a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 21156a4e45aSAndre Oppermann if (nmbjumbo9 > 0) 21256a4e45aSAndre Oppermann uma_zone_set_max(zone_jumbo9, nmbjumbo9); 21356a4e45aSAndre Oppermann 21456a4e45aSAndre Oppermann zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 21556a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 21656a4e45aSAndre Oppermann #ifdef INVARIANTS 21756a4e45aSAndre Oppermann trash_init, trash_fini, 21856a4e45aSAndre Oppermann #else 21956a4e45aSAndre Oppermann NULL, NULL, 22056a4e45aSAndre Oppermann #endif 22156a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 22256a4e45aSAndre Oppermann if (nmbjumbo16 > 0) 22356a4e45aSAndre Oppermann uma_zone_set_max(zone_jumbo16, nmbjumbo16); 22456a4e45aSAndre Oppermann 22556a4e45aSAndre Oppermann zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 22656a4e45aSAndre Oppermann NULL, NULL, 22756a4e45aSAndre Oppermann NULL, NULL, 22856a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 22956a4e45aSAndre Oppermann 23075ee267cSGleb Smirnoff zone_mtag_vlan = uma_zcreate("mtag_vlan", 23175ee267cSGleb Smirnoff sizeof(struct m_tag) + sizeof(u_int), 23275ee267cSGleb Smirnoff NULL, NULL, 23375ee267cSGleb Smirnoff mt_zinit_vlan, NULL, 23475ee267cSGleb Smirnoff UMA_ALIGN_INT, 0); 23575ee267cSGleb Smirnoff 23656a4e45aSAndre Oppermann /* uma_prealloc() goes here... */ 237099a0e58SBosko Milekic 238099a0e58SBosko Milekic /* 239099a0e58SBosko Milekic * Hook event handler for low-memory situation, used to 240099a0e58SBosko Milekic * drain protocols and push data back to the caches (UMA 241099a0e58SBosko Milekic * later pushes it back to VM). 242099a0e58SBosko Milekic */ 243099a0e58SBosko Milekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 244099a0e58SBosko Milekic EVENTHANDLER_PRI_FIRST); 245099a0e58SBosko Milekic 246099a0e58SBosko Milekic /* 247099a0e58SBosko Milekic * [Re]set counters and local statistics knobs. 248099a0e58SBosko Milekic * XXX Some of these should go and be replaced, but UMA stat 249099a0e58SBosko Milekic * gathering needs to be revised. 250099a0e58SBosko Milekic */ 251099a0e58SBosko Milekic mbstat.m_mbufs = 0; 252099a0e58SBosko Milekic mbstat.m_mclusts = 0; 253099a0e58SBosko Milekic mbstat.m_drain = 0; 254099a0e58SBosko Milekic mbstat.m_msize = MSIZE; 255099a0e58SBosko Milekic mbstat.m_mclbytes = MCLBYTES; 256099a0e58SBosko Milekic mbstat.m_minclsize = MINCLSIZE; 257099a0e58SBosko Milekic mbstat.m_mlen = MLEN; 258099a0e58SBosko Milekic mbstat.m_mhlen = MHLEN; 259099a0e58SBosko Milekic mbstat.m_numtypes = MT_NTYPES; 260099a0e58SBosko Milekic 261099a0e58SBosko Milekic mbstat.m_mcfail = mbstat.m_mpfail = 0; 262099a0e58SBosko Milekic mbstat.sf_iocnt = 0; 263099a0e58SBosko Milekic mbstat.sf_allocwait = mbstat.sf_allocfail = 0; 264099a0e58SBosko Milekic } 265099a0e58SBosko Milekic 266099a0e58SBosko Milekic /* 267099a0e58SBosko Milekic * Constructor for Mbuf master zone. 268099a0e58SBosko Milekic * 269099a0e58SBosko Milekic * The 'arg' pointer points to a mb_args structure which 270099a0e58SBosko Milekic * contains call-specific information required to support the 27156a4e45aSAndre Oppermann * mbuf allocation API. See mbuf.h. 272099a0e58SBosko Milekic */ 273b23f72e9SBrian Feldman static int 274b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how) 275099a0e58SBosko Milekic { 276099a0e58SBosko Milekic struct mbuf *m; 277099a0e58SBosko Milekic struct mb_args *args; 278b23f72e9SBrian Feldman #ifdef MAC 279b23f72e9SBrian Feldman int error; 280b23f72e9SBrian Feldman #endif 281099a0e58SBosko Milekic int flags; 282099a0e58SBosko Milekic short type; 283099a0e58SBosko Milekic 284121f0509SMike Silbersack #ifdef INVARIANTS 285121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 286121f0509SMike Silbersack #endif 287099a0e58SBosko Milekic m = (struct mbuf *)mem; 288099a0e58SBosko Milekic args = (struct mb_args *)arg; 289099a0e58SBosko Milekic flags = args->flags; 290099a0e58SBosko Milekic type = args->type; 291099a0e58SBosko Milekic 29256a4e45aSAndre Oppermann /* 29356a4e45aSAndre Oppermann * The mbuf is initialized later. The caller has the 29456a4e45aSAndre Oppermann * responseability to setup any MAC labels too. 29556a4e45aSAndre Oppermann */ 29656a4e45aSAndre Oppermann if (type == MT_NOINIT) 29756a4e45aSAndre Oppermann return (0); 29856a4e45aSAndre Oppermann 299099a0e58SBosko Milekic m->m_next = NULL; 300099a0e58SBosko Milekic m->m_nextpkt = NULL; 30156a4e45aSAndre Oppermann m->m_len = 0; 3026bc72ab9SBosko Milekic m->m_flags = flags; 30356a4e45aSAndre Oppermann m->m_type = type; 304099a0e58SBosko Milekic if (flags & M_PKTHDR) { 305099a0e58SBosko Milekic m->m_data = m->m_pktdat; 306099a0e58SBosko Milekic m->m_pkthdr.rcvif = NULL; 30756a4e45aSAndre Oppermann m->m_pkthdr.len = 0; 30856a4e45aSAndre Oppermann m->m_pkthdr.header = NULL; 309099a0e58SBosko Milekic m->m_pkthdr.csum_flags = 0; 31056a4e45aSAndre Oppermann m->m_pkthdr.csum_data = 0; 311099a0e58SBosko Milekic SLIST_INIT(&m->m_pkthdr.tags); 312099a0e58SBosko Milekic #ifdef MAC 313099a0e58SBosko Milekic /* If the label init fails, fail the alloc */ 314b23f72e9SBrian Feldman error = mac_init_mbuf(m, how); 315b23f72e9SBrian Feldman if (error) 316b23f72e9SBrian Feldman return (error); 317099a0e58SBosko Milekic #endif 3186bc72ab9SBosko Milekic } else 319099a0e58SBosko Milekic m->m_data = m->m_dat; 320b23f72e9SBrian Feldman return (0); 321099a0e58SBosko Milekic } 322099a0e58SBosko Milekic 323099a0e58SBosko Milekic /* 32456a4e45aSAndre Oppermann * The Mbuf master zone destructor. 325099a0e58SBosko Milekic */ 326099a0e58SBosko Milekic static void 327099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg) 328099a0e58SBosko Milekic { 329099a0e58SBosko Milekic struct mbuf *m; 330099a0e58SBosko Milekic 331099a0e58SBosko Milekic m = (struct mbuf *)mem; 332099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 333099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 33456a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 335121f0509SMike Silbersack #ifdef INVARIANTS 336121f0509SMike Silbersack trash_dtor(mem, size, arg); 337121f0509SMike Silbersack #endif 338099a0e58SBosko Milekic } 339099a0e58SBosko Milekic 34056a4e45aSAndre Oppermann /* 34156a4e45aSAndre Oppermann * The Mbuf Packet zone destructor. 34256a4e45aSAndre Oppermann */ 343099a0e58SBosko Milekic static void 344099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg) 345099a0e58SBosko Milekic { 346099a0e58SBosko Milekic struct mbuf *m; 347099a0e58SBosko Milekic 348099a0e58SBosko Milekic m = (struct mbuf *)mem; 349099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 350099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 35156a4e45aSAndre Oppermann 35256a4e45aSAndre Oppermann /* Make sure we've got a clean cluster back. */ 35356a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 35456a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 35556a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 35656a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__)); 35756a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 35849d46b61SGleb Smirnoff KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); 35956a4e45aSAndre Oppermann KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); 360121f0509SMike Silbersack #ifdef INVARIANTS 361121f0509SMike Silbersack trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 362121f0509SMike Silbersack #endif 363099a0e58SBosko Milekic } 364099a0e58SBosko Milekic 365099a0e58SBosko Milekic /* 366ec63cb90SAndre Oppermann * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. 367099a0e58SBosko Milekic * 368099a0e58SBosko Milekic * Here the 'arg' pointer points to the Mbuf which we 36956a4e45aSAndre Oppermann * are configuring cluster storage for. If 'arg' is 37056a4e45aSAndre Oppermann * empty we allocate just the cluster without setting 37156a4e45aSAndre Oppermann * the mbuf to it. See mbuf.h. 372099a0e58SBosko Milekic */ 373b23f72e9SBrian Feldman static int 374b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how) 375099a0e58SBosko Milekic { 376099a0e58SBosko Milekic struct mbuf *m; 37756a4e45aSAndre Oppermann u_int *refcnt; 37856a4e45aSAndre Oppermann int type = 0; 379099a0e58SBosko Milekic 380121f0509SMike Silbersack #ifdef INVARIANTS 381121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 382121f0509SMike Silbersack #endif 383099a0e58SBosko Milekic m = (struct mbuf *)arg; 38456a4e45aSAndre Oppermann if (m != NULL) { 38556a4e45aSAndre Oppermann switch (size) { 38656a4e45aSAndre Oppermann case MCLBYTES: 38756a4e45aSAndre Oppermann type = EXT_CLUSTER; 38856a4e45aSAndre Oppermann break; 389ec63cb90SAndre Oppermann #if MJUMPAGESIZE != MCLBYTES 390ec63cb90SAndre Oppermann case MJUMPAGESIZE: 391ec63cb90SAndre Oppermann type = EXT_JUMBOP; 392d5269a63SAndre Oppermann break; 39336ae3fd3SAndre Oppermann #endif 39456a4e45aSAndre Oppermann case MJUM9BYTES: 39556a4e45aSAndre Oppermann type = EXT_JUMBO9; 39656a4e45aSAndre Oppermann break; 39756a4e45aSAndre Oppermann case MJUM16BYTES: 39856a4e45aSAndre Oppermann type = EXT_JUMBO16; 39956a4e45aSAndre Oppermann break; 40056a4e45aSAndre Oppermann default: 40156a4e45aSAndre Oppermann panic("unknown cluster size"); 40256a4e45aSAndre Oppermann break; 40356a4e45aSAndre Oppermann } 404099a0e58SBosko Milekic m->m_ext.ext_buf = (caddr_t)mem; 405099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 406099a0e58SBosko Milekic m->m_flags |= M_EXT; 407099a0e58SBosko Milekic m->m_ext.ext_free = NULL; 408099a0e58SBosko Milekic m->m_ext.ext_args = NULL; 40956a4e45aSAndre Oppermann m->m_ext.ext_size = size; 41056a4e45aSAndre Oppermann m->m_ext.ext_type = type; 41156a4e45aSAndre Oppermann m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem); 41256a4e45aSAndre Oppermann *m->m_ext.ref_cnt = 1; 41356a4e45aSAndre Oppermann } else { 41456a4e45aSAndre Oppermann refcnt = uma_find_refcnt(zone_clust, mem); 41556a4e45aSAndre Oppermann *refcnt = 1; 41656a4e45aSAndre Oppermann } 417b23f72e9SBrian Feldman return (0); 418099a0e58SBosko Milekic } 419099a0e58SBosko Milekic 42056a4e45aSAndre Oppermann /* 42156a4e45aSAndre Oppermann * The Mbuf Cluster zone destructor. 42256a4e45aSAndre Oppermann */ 423099a0e58SBosko Milekic static void 424099a0e58SBosko Milekic mb_dtor_clust(void *mem, int size, void *arg) 425099a0e58SBosko Milekic { 42656a4e45aSAndre Oppermann 427a5f77087SAndre Oppermann KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1, 428a5f77087SAndre Oppermann ("%s: refcnt incorrect %u", __func__, 429a5f77087SAndre Oppermann *(uma_find_refcnt(zone_clust, mem))) ); 430121f0509SMike Silbersack #ifdef INVARIANTS 431121f0509SMike Silbersack trash_dtor(mem, size, arg); 432121f0509SMike Silbersack #endif 433099a0e58SBosko Milekic } 434099a0e58SBosko Milekic 435099a0e58SBosko Milekic /* 436099a0e58SBosko Milekic * The Packet secondary zone's init routine, executed on the 43756a4e45aSAndre Oppermann * object's transition from mbuf keg slab to zone cache. 438099a0e58SBosko Milekic */ 439b23f72e9SBrian Feldman static int 44056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how) 441099a0e58SBosko Milekic { 442099a0e58SBosko Milekic struct mbuf *m; 443099a0e58SBosko Milekic 44456a4e45aSAndre Oppermann m = (struct mbuf *)mem; /* m is virgin. */ 445fd241339SAndre Oppermann (void)uma_zalloc_arg(zone_clust, m, how); 446b23f72e9SBrian Feldman if (m->m_ext.ext_buf == NULL) 447b23f72e9SBrian Feldman return (ENOMEM); 448cd5bb63bSAndre Oppermann m->m_ext.ext_type = EXT_PACKET; /* Override. */ 449121f0509SMike Silbersack #ifdef INVARIANTS 450121f0509SMike Silbersack trash_init(m->m_ext.ext_buf, MCLBYTES, how); 451121f0509SMike Silbersack #endif 452b23f72e9SBrian Feldman return (0); 453099a0e58SBosko Milekic } 454099a0e58SBosko Milekic 455099a0e58SBosko Milekic /* 456099a0e58SBosko Milekic * The Packet secondary zone's fini routine, executed on the 457099a0e58SBosko Milekic * object's transition from zone cache to keg slab. 458099a0e58SBosko Milekic */ 459099a0e58SBosko Milekic static void 46056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size) 461099a0e58SBosko Milekic { 462099a0e58SBosko Milekic struct mbuf *m; 463099a0e58SBosko Milekic 464099a0e58SBosko Milekic m = (struct mbuf *)mem; 465121f0509SMike Silbersack #ifdef INVARIANTS 466121f0509SMike Silbersack trash_fini(m->m_ext.ext_buf, MCLBYTES); 467121f0509SMike Silbersack #endif 468099a0e58SBosko Milekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 469a7b844d2SMike Silbersack #ifdef INVARIANTS 470a7b844d2SMike Silbersack trash_dtor(mem, size, NULL); 471a7b844d2SMike Silbersack #endif 472099a0e58SBosko Milekic } 473099a0e58SBosko Milekic 474099a0e58SBosko Milekic /* 475099a0e58SBosko Milekic * The "packet" keg constructor. 476099a0e58SBosko Milekic */ 477b23f72e9SBrian Feldman static int 478b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how) 479099a0e58SBosko Milekic { 480099a0e58SBosko Milekic struct mbuf *m; 481099a0e58SBosko Milekic struct mb_args *args; 482b23f72e9SBrian Feldman #ifdef MAC 483b23f72e9SBrian Feldman int error; 484b23f72e9SBrian Feldman #endif 485b23f72e9SBrian Feldman int flags; 486099a0e58SBosko Milekic short type; 487099a0e58SBosko Milekic 488099a0e58SBosko Milekic m = (struct mbuf *)mem; 489099a0e58SBosko Milekic args = (struct mb_args *)arg; 490099a0e58SBosko Milekic flags = args->flags; 491099a0e58SBosko Milekic type = args->type; 492099a0e58SBosko Milekic 493121f0509SMike Silbersack #ifdef INVARIANTS 494121f0509SMike Silbersack trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 495121f0509SMike Silbersack #endif 496099a0e58SBosko Milekic m->m_next = NULL; 4976bc72ab9SBosko Milekic m->m_nextpkt = NULL; 498099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 49956a4e45aSAndre Oppermann m->m_len = 0; 50056a4e45aSAndre Oppermann m->m_flags = (flags | M_EXT); 50156a4e45aSAndre Oppermann m->m_type = type; 502099a0e58SBosko Milekic 503099a0e58SBosko Milekic if (flags & M_PKTHDR) { 504099a0e58SBosko Milekic m->m_pkthdr.rcvif = NULL; 50556a4e45aSAndre Oppermann m->m_pkthdr.len = 0; 50656a4e45aSAndre Oppermann m->m_pkthdr.header = NULL; 507099a0e58SBosko Milekic m->m_pkthdr.csum_flags = 0; 50856a4e45aSAndre Oppermann m->m_pkthdr.csum_data = 0; 509099a0e58SBosko Milekic SLIST_INIT(&m->m_pkthdr.tags); 510099a0e58SBosko Milekic #ifdef MAC 511099a0e58SBosko Milekic /* If the label init fails, fail the alloc */ 512b23f72e9SBrian Feldman error = mac_init_mbuf(m, how); 513b23f72e9SBrian Feldman if (error) 514b23f72e9SBrian Feldman return (error); 515099a0e58SBosko Milekic #endif 516099a0e58SBosko Milekic } 51756a4e45aSAndre Oppermann /* m_ext is already initialized. */ 51856a4e45aSAndre Oppermann 519b23f72e9SBrian Feldman return (0); 520099a0e58SBosko Milekic } 521099a0e58SBosko Milekic 52275ee267cSGleb Smirnoff static void 52375ee267cSGleb Smirnoff mt_vlan_free(struct m_tag *mtag) 52475ee267cSGleb Smirnoff { 52575ee267cSGleb Smirnoff uma_zfree(zone_mtag_vlan, mtag); 52675ee267cSGleb Smirnoff } 52775ee267cSGleb Smirnoff 52875ee267cSGleb Smirnoff static int 52975ee267cSGleb Smirnoff mt_zinit_vlan(void *mem, int size, int how) 53075ee267cSGleb Smirnoff { 53175ee267cSGleb Smirnoff struct m_tag *mtag = (struct m_tag *)mem; 53275ee267cSGleb Smirnoff 53375ee267cSGleb Smirnoff m_tag_setup(mtag, MTAG_VLAN, MTAG_VLAN_TAG, sizeof(u_int)); 53475ee267cSGleb Smirnoff mtag->m_tag_free = mt_vlan_free; 53575ee267cSGleb Smirnoff 53675ee267cSGleb Smirnoff return (0); 53775ee267cSGleb Smirnoff } 53875ee267cSGleb Smirnoff 539099a0e58SBosko Milekic /* 540099a0e58SBosko Milekic * This is the protocol drain routine. 541099a0e58SBosko Milekic * 542099a0e58SBosko Milekic * No locks should be held when this is called. The drain routines have to 543099a0e58SBosko Milekic * presently acquire some locks which raises the possibility of lock order 544099a0e58SBosko Milekic * reversal. 545099a0e58SBosko Milekic */ 546099a0e58SBosko Milekic static void 547099a0e58SBosko Milekic mb_reclaim(void *junk) 548099a0e58SBosko Milekic { 549099a0e58SBosko Milekic struct domain *dp; 550099a0e58SBosko Milekic struct protosw *pr; 551099a0e58SBosko Milekic 552099a0e58SBosko Milekic WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, 553099a0e58SBosko Milekic "mb_reclaim()"); 554099a0e58SBosko Milekic 555099a0e58SBosko Milekic for (dp = domains; dp != NULL; dp = dp->dom_next) 556099a0e58SBosko Milekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 557099a0e58SBosko Milekic if (pr->pr_drain != NULL) 558099a0e58SBosko Milekic (*pr->pr_drain)(); 559099a0e58SBosko Milekic } 560