1099a0e58SBosko Milekic /*- 28076cb52SBosko Milekic * Copyright (c) 2004, 2005, 38076cb52SBosko Milekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4099a0e58SBosko Milekic * 5099a0e58SBosko Milekic * Redistribution and use in source and binary forms, with or without 6099a0e58SBosko Milekic * modification, are permitted provided that the following conditions 7099a0e58SBosko Milekic * are met: 8099a0e58SBosko Milekic * 1. Redistributions of source code must retain the above copyright 9099a0e58SBosko Milekic * notice unmodified, this list of conditions and the following 10099a0e58SBosko Milekic * disclaimer. 11099a0e58SBosko Milekic * 2. Redistributions in binary form must reproduce the above copyright 12099a0e58SBosko Milekic * notice, this list of conditions and the following disclaimer in the 13099a0e58SBosko Milekic * documentation and/or other materials provided with the distribution. 14099a0e58SBosko Milekic * 15099a0e58SBosko Milekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16099a0e58SBosko Milekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17099a0e58SBosko Milekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18099a0e58SBosko Milekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19099a0e58SBosko Milekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20099a0e58SBosko Milekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21099a0e58SBosko Milekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22099a0e58SBosko Milekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23099a0e58SBosko Milekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24099a0e58SBosko Milekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25099a0e58SBosko Milekic * SUCH DAMAGE. 26099a0e58SBosko Milekic */ 27099a0e58SBosko Milekic 28099a0e58SBosko Milekic #include <sys/cdefs.h> 29099a0e58SBosko Milekic __FBSDID("$FreeBSD$"); 30099a0e58SBosko Milekic 31099a0e58SBosko Milekic #include "opt_mac.h" 32099a0e58SBosko Milekic #include "opt_param.h" 33099a0e58SBosko Milekic 34099a0e58SBosko Milekic #include <sys/param.h> 35099a0e58SBosko Milekic #include <sys/malloc.h> 36099a0e58SBosko Milekic #include <sys/systm.h> 37099a0e58SBosko Milekic #include <sys/mbuf.h> 38099a0e58SBosko Milekic #include <sys/domain.h> 39099a0e58SBosko Milekic #include <sys/eventhandler.h> 40099a0e58SBosko Milekic #include <sys/kernel.h> 41099a0e58SBosko Milekic #include <sys/protosw.h> 42099a0e58SBosko Milekic #include <sys/smp.h> 43099a0e58SBosko Milekic #include <sys/sysctl.h> 44099a0e58SBosko Milekic 45aed55708SRobert Watson #include <security/mac/mac_framework.h> 46aed55708SRobert Watson 47099a0e58SBosko Milekic #include <vm/vm.h> 48099a0e58SBosko Milekic #include <vm/vm_page.h> 49099a0e58SBosko Milekic #include <vm/uma.h> 50121f0509SMike Silbersack #include <vm/uma_int.h> 51121f0509SMike Silbersack #include <vm/uma_dbg.h> 52099a0e58SBosko Milekic 53099a0e58SBosko Milekic /* 54099a0e58SBosko Milekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 55099a0e58SBosko Milekic * Zones. 56099a0e58SBosko Milekic * 57099a0e58SBosko Milekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 58099a0e58SBosko Milekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 59099a0e58SBosko Milekic * administrator so desires. 60099a0e58SBosko Milekic * 61099a0e58SBosko Milekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 62099a0e58SBosko Milekic * Zone. 63099a0e58SBosko Milekic * 64099a0e58SBosko Milekic * Additionally, FreeBSD provides a Packet Zone, which it 65099a0e58SBosko Milekic * configures as a Secondary Zone to the Mbuf Master Zone, 66099a0e58SBosko Milekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 67099a0e58SBosko Milekic * 68099a0e58SBosko Milekic * Thus common-case allocations and locking are simplified: 69099a0e58SBosko Milekic * 70099a0e58SBosko Milekic * m_clget() m_getcl() 71099a0e58SBosko Milekic * | | 72099a0e58SBosko Milekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 73099a0e58SBosko Milekic * | | [ Packet ] | 74099a0e58SBosko Milekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 75099a0e58SBosko Milekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 76099a0e58SBosko Milekic * | \________ | 77099a0e58SBosko Milekic * [ Cluster Keg ] \ / 78099a0e58SBosko Milekic * | [ Mbuf Keg ] 79099a0e58SBosko Milekic * [ Cluster Slabs ] | 80099a0e58SBosko Milekic * | [ Mbuf Slabs ] 81099a0e58SBosko Milekic * \____________(VM)_________________/ 8256a4e45aSAndre Oppermann * 8356a4e45aSAndre Oppermann * 84fcf90618SGleb Smirnoff * Whenever an object is allocated with uma_zalloc() out of 8556a4e45aSAndre Oppermann * one of the Zones its _ctor_ function is executed. The same 86fcf90618SGleb Smirnoff * for any deallocation through uma_zfree() the _dtor_ function 8756a4e45aSAndre Oppermann * is executed. 8856a4e45aSAndre Oppermann * 8956a4e45aSAndre Oppermann * Caches are per-CPU and are filled from the Master Zone. 9056a4e45aSAndre Oppermann * 91fcf90618SGleb Smirnoff * Whenever an object is allocated from the underlying global 9256a4e45aSAndre Oppermann * memory pool it gets pre-initialized with the _zinit_ functions. 9356a4e45aSAndre Oppermann * When the Keg's are overfull objects get decomissioned with 9456a4e45aSAndre Oppermann * _zfini_ functions and free'd back to the global memory pool. 9556a4e45aSAndre Oppermann * 96099a0e58SBosko Milekic */ 97099a0e58SBosko Milekic 9856a4e45aSAndre Oppermann int nmbclusters; /* limits number of mbuf clusters */ 99ec63cb90SAndre Oppermann int nmbjumbop; /* limits number of page size jumbo clusters */ 10056a4e45aSAndre Oppermann int nmbjumbo9; /* limits number of 9k jumbo clusters */ 10156a4e45aSAndre Oppermann int nmbjumbo16; /* limits number of 16k jumbo clusters */ 102099a0e58SBosko Milekic struct mbstat mbstat; 103099a0e58SBosko Milekic 104099a0e58SBosko Milekic static void 105099a0e58SBosko Milekic tunable_mbinit(void *dummy) 106099a0e58SBosko Milekic { 107099a0e58SBosko Milekic 108099a0e58SBosko Milekic /* This has to be done before VM init. */ 109099a0e58SBosko Milekic nmbclusters = 1024 + maxusers * 64; 110099a0e58SBosko Milekic TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 111099a0e58SBosko Milekic } 112099a0e58SBosko Milekic SYSINIT(tunable_mbinit, SI_SUB_TUNABLES, SI_ORDER_ANY, tunable_mbinit, NULL); 113099a0e58SBosko Milekic 11456a4e45aSAndre Oppermann /* XXX: These should be tuneables. Can't change UMA limits on the fly. */ 1154f590175SPaul Saab static int 1164f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS) 1174f590175SPaul Saab { 1184f590175SPaul Saab int error, newnmbclusters; 1194f590175SPaul Saab 1204f590175SPaul Saab newnmbclusters = nmbclusters; 1214f590175SPaul Saab error = sysctl_handle_int(oidp, &newnmbclusters, sizeof(int), req); 1224f590175SPaul Saab if (error == 0 && req->newptr) { 1234f590175SPaul Saab if (newnmbclusters > nmbclusters) { 1244f590175SPaul Saab nmbclusters = newnmbclusters; 1254f590175SPaul Saab uma_zone_set_max(zone_clust, nmbclusters); 1264f590175SPaul Saab EVENTHANDLER_INVOKE(nmbclusters_change); 1274f590175SPaul Saab } else 1284f590175SPaul Saab error = EINVAL; 1294f590175SPaul Saab } 1304f590175SPaul Saab return (error); 1314f590175SPaul Saab } 1324f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW, 1334f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU", 134099a0e58SBosko Milekic "Maximum number of mbuf clusters allowed"); 135ec63cb90SAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbop, CTLFLAG_RW, &nmbjumbop, 0, 136ec63cb90SAndre Oppermann "Maximum number of mbuf page size jumbo clusters allowed"); 13756a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo9, CTLFLAG_RW, &nmbjumbo9, 0, 13856a4e45aSAndre Oppermann "Maximum number of mbuf 9k jumbo clusters allowed"); 13956a4e45aSAndre Oppermann SYSCTL_INT(_kern_ipc, OID_AUTO, nmbjumbo16, CTLFLAG_RW, &nmbjumbo16, 0, 14056a4e45aSAndre Oppermann "Maximum number of mbuf 16k jumbo clusters allowed"); 141099a0e58SBosko Milekic SYSCTL_STRUCT(_kern_ipc, OID_AUTO, mbstat, CTLFLAG_RD, &mbstat, mbstat, 142099a0e58SBosko Milekic "Mbuf general information and statistics"); 143099a0e58SBosko Milekic 144099a0e58SBosko Milekic /* 145099a0e58SBosko Milekic * Zones from which we allocate. 146099a0e58SBosko Milekic */ 147099a0e58SBosko Milekic uma_zone_t zone_mbuf; 148099a0e58SBosko Milekic uma_zone_t zone_clust; 149099a0e58SBosko Milekic uma_zone_t zone_pack; 150ec63cb90SAndre Oppermann uma_zone_t zone_jumbop; 15156a4e45aSAndre Oppermann uma_zone_t zone_jumbo9; 15256a4e45aSAndre Oppermann uma_zone_t zone_jumbo16; 15356a4e45aSAndre Oppermann uma_zone_t zone_ext_refcnt; 154099a0e58SBosko Milekic 155099a0e58SBosko Milekic /* 156099a0e58SBosko Milekic * Local prototypes. 157099a0e58SBosko Milekic */ 158b23f72e9SBrian Feldman static int mb_ctor_mbuf(void *, int, void *, int); 159b23f72e9SBrian Feldman static int mb_ctor_clust(void *, int, void *, int); 160b23f72e9SBrian Feldman static int mb_ctor_pack(void *, int, void *, int); 161099a0e58SBosko Milekic static void mb_dtor_mbuf(void *, int, void *); 16256a4e45aSAndre Oppermann static void mb_dtor_clust(void *, int, void *); 16356a4e45aSAndre Oppermann static void mb_dtor_pack(void *, int, void *); 16456a4e45aSAndre Oppermann static int mb_zinit_pack(void *, int, int); 16556a4e45aSAndre Oppermann static void mb_zfini_pack(void *, int); 166099a0e58SBosko Milekic 167099a0e58SBosko Milekic static void mb_reclaim(void *); 168099a0e58SBosko Milekic static void mbuf_init(void *); 169099a0e58SBosko Milekic 170a04946cfSBrian Somers /* Ensure that MSIZE doesn't break dtom() - it must be a power of 2 */ 171a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 172a04946cfSBrian Somers 173099a0e58SBosko Milekic /* 174099a0e58SBosko Milekic * Initialize FreeBSD Network buffer allocation. 175099a0e58SBosko Milekic */ 176099a0e58SBosko Milekic SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL) 177099a0e58SBosko Milekic static void 178099a0e58SBosko Milekic mbuf_init(void *dummy) 179099a0e58SBosko Milekic { 180099a0e58SBosko Milekic 181099a0e58SBosko Milekic /* 182099a0e58SBosko Milekic * Configure UMA zones for Mbufs, Clusters, and Packets. 183099a0e58SBosko Milekic */ 18456a4e45aSAndre Oppermann zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 18556a4e45aSAndre Oppermann mb_ctor_mbuf, mb_dtor_mbuf, 186121f0509SMike Silbersack #ifdef INVARIANTS 18756a4e45aSAndre Oppermann trash_init, trash_fini, 188121f0509SMike Silbersack #else 18956a4e45aSAndre Oppermann NULL, NULL, 190121f0509SMike Silbersack #endif 19156a4e45aSAndre Oppermann MSIZE - 1, UMA_ZONE_MAXBUCKET); 19256a4e45aSAndre Oppermann 19368352adfSRobert Watson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 19456a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 195121f0509SMike Silbersack #ifdef INVARIANTS 19656a4e45aSAndre Oppermann trash_init, trash_fini, 197121f0509SMike Silbersack #else 19856a4e45aSAndre Oppermann NULL, NULL, 199121f0509SMike Silbersack #endif 20056a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 201099a0e58SBosko Milekic if (nmbclusters > 0) 202099a0e58SBosko Milekic uma_zone_set_max(zone_clust, nmbclusters); 203099a0e58SBosko Milekic 20456a4e45aSAndre Oppermann zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 20556a4e45aSAndre Oppermann mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 20656a4e45aSAndre Oppermann 207fcf90618SGleb Smirnoff /* Make jumbo frame zone too. Page size, 9k and 16k. */ 208ec63cb90SAndre Oppermann zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 209d5269a63SAndre Oppermann mb_ctor_clust, mb_dtor_clust, 210d5269a63SAndre Oppermann #ifdef INVARIANTS 211d5269a63SAndre Oppermann trash_init, trash_fini, 212d5269a63SAndre Oppermann #else 213d5269a63SAndre Oppermann NULL, NULL, 214d5269a63SAndre Oppermann #endif 215d5269a63SAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 216ec63cb90SAndre Oppermann if (nmbjumbop > 0) 217ec63cb90SAndre Oppermann uma_zone_set_max(zone_jumbop, nmbjumbop); 218d5269a63SAndre Oppermann 21956a4e45aSAndre Oppermann zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 22056a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 22156a4e45aSAndre Oppermann #ifdef INVARIANTS 22256a4e45aSAndre Oppermann trash_init, trash_fini, 22356a4e45aSAndre Oppermann #else 22456a4e45aSAndre Oppermann NULL, NULL, 22556a4e45aSAndre Oppermann #endif 22656a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 22756a4e45aSAndre Oppermann if (nmbjumbo9 > 0) 22856a4e45aSAndre Oppermann uma_zone_set_max(zone_jumbo9, nmbjumbo9); 22956a4e45aSAndre Oppermann 23056a4e45aSAndre Oppermann zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 23156a4e45aSAndre Oppermann mb_ctor_clust, mb_dtor_clust, 23256a4e45aSAndre Oppermann #ifdef INVARIANTS 23356a4e45aSAndre Oppermann trash_init, trash_fini, 23456a4e45aSAndre Oppermann #else 23556a4e45aSAndre Oppermann NULL, NULL, 23656a4e45aSAndre Oppermann #endif 23756a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_REFCNT); 23856a4e45aSAndre Oppermann if (nmbjumbo16 > 0) 23956a4e45aSAndre Oppermann uma_zone_set_max(zone_jumbo16, nmbjumbo16); 24056a4e45aSAndre Oppermann 24156a4e45aSAndre Oppermann zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int), 24256a4e45aSAndre Oppermann NULL, NULL, 24356a4e45aSAndre Oppermann NULL, NULL, 24456a4e45aSAndre Oppermann UMA_ALIGN_PTR, UMA_ZONE_ZINIT); 24556a4e45aSAndre Oppermann 24656a4e45aSAndre Oppermann /* uma_prealloc() goes here... */ 247099a0e58SBosko Milekic 248099a0e58SBosko Milekic /* 249099a0e58SBosko Milekic * Hook event handler for low-memory situation, used to 250099a0e58SBosko Milekic * drain protocols and push data back to the caches (UMA 251099a0e58SBosko Milekic * later pushes it back to VM). 252099a0e58SBosko Milekic */ 253099a0e58SBosko Milekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 254099a0e58SBosko Milekic EVENTHANDLER_PRI_FIRST); 255099a0e58SBosko Milekic 256099a0e58SBosko Milekic /* 257099a0e58SBosko Milekic * [Re]set counters and local statistics knobs. 258099a0e58SBosko Milekic * XXX Some of these should go and be replaced, but UMA stat 259099a0e58SBosko Milekic * gathering needs to be revised. 260099a0e58SBosko Milekic */ 261099a0e58SBosko Milekic mbstat.m_mbufs = 0; 262099a0e58SBosko Milekic mbstat.m_mclusts = 0; 263099a0e58SBosko Milekic mbstat.m_drain = 0; 264099a0e58SBosko Milekic mbstat.m_msize = MSIZE; 265099a0e58SBosko Milekic mbstat.m_mclbytes = MCLBYTES; 266099a0e58SBosko Milekic mbstat.m_minclsize = MINCLSIZE; 267099a0e58SBosko Milekic mbstat.m_mlen = MLEN; 268099a0e58SBosko Milekic mbstat.m_mhlen = MHLEN; 269099a0e58SBosko Milekic mbstat.m_numtypes = MT_NTYPES; 270099a0e58SBosko Milekic 271099a0e58SBosko Milekic mbstat.m_mcfail = mbstat.m_mpfail = 0; 272099a0e58SBosko Milekic mbstat.sf_iocnt = 0; 273099a0e58SBosko Milekic mbstat.sf_allocwait = mbstat.sf_allocfail = 0; 274099a0e58SBosko Milekic } 275099a0e58SBosko Milekic 276099a0e58SBosko Milekic /* 277099a0e58SBosko Milekic * Constructor for Mbuf master zone. 278099a0e58SBosko Milekic * 279099a0e58SBosko Milekic * The 'arg' pointer points to a mb_args structure which 280099a0e58SBosko Milekic * contains call-specific information required to support the 28156a4e45aSAndre Oppermann * mbuf allocation API. See mbuf.h. 282099a0e58SBosko Milekic */ 283b23f72e9SBrian Feldman static int 284b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how) 285099a0e58SBosko Milekic { 286099a0e58SBosko Milekic struct mbuf *m; 287099a0e58SBosko Milekic struct mb_args *args; 288b23f72e9SBrian Feldman #ifdef MAC 289b23f72e9SBrian Feldman int error; 290b23f72e9SBrian Feldman #endif 291099a0e58SBosko Milekic int flags; 292099a0e58SBosko Milekic short type; 293099a0e58SBosko Milekic 294121f0509SMike Silbersack #ifdef INVARIANTS 295121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 296121f0509SMike Silbersack #endif 297099a0e58SBosko Milekic m = (struct mbuf *)mem; 298099a0e58SBosko Milekic args = (struct mb_args *)arg; 299099a0e58SBosko Milekic flags = args->flags; 300099a0e58SBosko Milekic type = args->type; 301099a0e58SBosko Milekic 30256a4e45aSAndre Oppermann /* 30356a4e45aSAndre Oppermann * The mbuf is initialized later. The caller has the 304fcf90618SGleb Smirnoff * responsibility to set up any MAC labels too. 30556a4e45aSAndre Oppermann */ 30656a4e45aSAndre Oppermann if (type == MT_NOINIT) 30756a4e45aSAndre Oppermann return (0); 30856a4e45aSAndre Oppermann 309099a0e58SBosko Milekic m->m_next = NULL; 310099a0e58SBosko Milekic m->m_nextpkt = NULL; 31156a4e45aSAndre Oppermann m->m_len = 0; 3126bc72ab9SBosko Milekic m->m_flags = flags; 31356a4e45aSAndre Oppermann m->m_type = type; 314099a0e58SBosko Milekic if (flags & M_PKTHDR) { 315099a0e58SBosko Milekic m->m_data = m->m_pktdat; 316099a0e58SBosko Milekic m->m_pkthdr.rcvif = NULL; 31756a4e45aSAndre Oppermann m->m_pkthdr.len = 0; 31856a4e45aSAndre Oppermann m->m_pkthdr.header = NULL; 319099a0e58SBosko Milekic m->m_pkthdr.csum_flags = 0; 32056a4e45aSAndre Oppermann m->m_pkthdr.csum_data = 0; 321a855e2b4SAndre Oppermann m->m_pkthdr.tso_segsz = 0; 322a855e2b4SAndre Oppermann m->m_pkthdr.ether_vtag = 0; 323099a0e58SBosko Milekic SLIST_INIT(&m->m_pkthdr.tags); 324099a0e58SBosko Milekic #ifdef MAC 325099a0e58SBosko Milekic /* If the label init fails, fail the alloc */ 326b23f72e9SBrian Feldman error = mac_init_mbuf(m, how); 327b23f72e9SBrian Feldman if (error) 328b23f72e9SBrian Feldman return (error); 329099a0e58SBosko Milekic #endif 3306bc72ab9SBosko Milekic } else 331099a0e58SBosko Milekic m->m_data = m->m_dat; 332b23f72e9SBrian Feldman return (0); 333099a0e58SBosko Milekic } 334099a0e58SBosko Milekic 335099a0e58SBosko Milekic /* 33656a4e45aSAndre Oppermann * The Mbuf master zone destructor. 337099a0e58SBosko Milekic */ 338099a0e58SBosko Milekic static void 339099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg) 340099a0e58SBosko Milekic { 341099a0e58SBosko Milekic struct mbuf *m; 342099a0e58SBosko Milekic 343099a0e58SBosko Milekic m = (struct mbuf *)mem; 344099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 345099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 34656a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == 0, ("%s: M_EXT set", __func__)); 347121f0509SMike Silbersack #ifdef INVARIANTS 348121f0509SMike Silbersack trash_dtor(mem, size, arg); 349121f0509SMike Silbersack #endif 350099a0e58SBosko Milekic } 351099a0e58SBosko Milekic 35256a4e45aSAndre Oppermann /* 35356a4e45aSAndre Oppermann * The Mbuf Packet zone destructor. 35456a4e45aSAndre Oppermann */ 355099a0e58SBosko Milekic static void 356099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg) 357099a0e58SBosko Milekic { 358099a0e58SBosko Milekic struct mbuf *m; 359099a0e58SBosko Milekic 360099a0e58SBosko Milekic m = (struct mbuf *)mem; 361099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 362099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 36356a4e45aSAndre Oppermann 36456a4e45aSAndre Oppermann /* Make sure we've got a clean cluster back. */ 36556a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 36656a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 36756a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 36856a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_args == NULL, ("%s: ext_args != NULL", __func__)); 36956a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 37049d46b61SGleb Smirnoff KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); 37156a4e45aSAndre Oppermann KASSERT(*m->m_ext.ref_cnt == 1, ("%s: ref_cnt != 1", __func__)); 372121f0509SMike Silbersack #ifdef INVARIANTS 373121f0509SMike Silbersack trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 374121f0509SMike Silbersack #endif 3756c125b8dSMohan Srinivasan /* 3766c125b8dSMohan Srinivasan * If there are processes blocked on zone_clust, waiting for pages to be freed up, 3776c125b8dSMohan Srinivasan * cause them to be woken up by draining the packet zone. We are exposed to a race here 3786c125b8dSMohan Srinivasan * (in the check for the UMA_ZFLAG_FULL) where we might miss the flag set, but that is 3796c125b8dSMohan Srinivasan * deliberate. We don't want to acquire the zone lock for every mbuf free. 3806c125b8dSMohan Srinivasan */ 3816c125b8dSMohan Srinivasan if (uma_zone_exhausted_nolock(zone_clust)) 3826c125b8dSMohan Srinivasan zone_drain(zone_pack); 383099a0e58SBosko Milekic } 384099a0e58SBosko Milekic 385099a0e58SBosko Milekic /* 386ec63cb90SAndre Oppermann * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. 387099a0e58SBosko Milekic * 388099a0e58SBosko Milekic * Here the 'arg' pointer points to the Mbuf which we 38956a4e45aSAndre Oppermann * are configuring cluster storage for. If 'arg' is 39056a4e45aSAndre Oppermann * empty we allocate just the cluster without setting 39156a4e45aSAndre Oppermann * the mbuf to it. See mbuf.h. 392099a0e58SBosko Milekic */ 393b23f72e9SBrian Feldman static int 394b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how) 395099a0e58SBosko Milekic { 396099a0e58SBosko Milekic struct mbuf *m; 39756a4e45aSAndre Oppermann u_int *refcnt; 39856a4e45aSAndre Oppermann int type = 0; 399099a0e58SBosko Milekic 400121f0509SMike Silbersack #ifdef INVARIANTS 401121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 402121f0509SMike Silbersack #endif 403099a0e58SBosko Milekic m = (struct mbuf *)arg; 40456a4e45aSAndre Oppermann if (m != NULL) { 40556a4e45aSAndre Oppermann switch (size) { 40656a4e45aSAndre Oppermann case MCLBYTES: 40756a4e45aSAndre Oppermann type = EXT_CLUSTER; 40856a4e45aSAndre Oppermann break; 409ec63cb90SAndre Oppermann #if MJUMPAGESIZE != MCLBYTES 410ec63cb90SAndre Oppermann case MJUMPAGESIZE: 411ec63cb90SAndre Oppermann type = EXT_JUMBOP; 412d5269a63SAndre Oppermann break; 41336ae3fd3SAndre Oppermann #endif 41456a4e45aSAndre Oppermann case MJUM9BYTES: 41556a4e45aSAndre Oppermann type = EXT_JUMBO9; 41656a4e45aSAndre Oppermann break; 41756a4e45aSAndre Oppermann case MJUM16BYTES: 41856a4e45aSAndre Oppermann type = EXT_JUMBO16; 41956a4e45aSAndre Oppermann break; 42056a4e45aSAndre Oppermann default: 42156a4e45aSAndre Oppermann panic("unknown cluster size"); 42256a4e45aSAndre Oppermann break; 42356a4e45aSAndre Oppermann } 424099a0e58SBosko Milekic m->m_ext.ext_buf = (caddr_t)mem; 425099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 426099a0e58SBosko Milekic m->m_flags |= M_EXT; 427099a0e58SBosko Milekic m->m_ext.ext_free = NULL; 428099a0e58SBosko Milekic m->m_ext.ext_args = NULL; 42956a4e45aSAndre Oppermann m->m_ext.ext_size = size; 43056a4e45aSAndre Oppermann m->m_ext.ext_type = type; 43156a4e45aSAndre Oppermann m->m_ext.ref_cnt = uma_find_refcnt(zone_clust, mem); 43256a4e45aSAndre Oppermann *m->m_ext.ref_cnt = 1; 43356a4e45aSAndre Oppermann } else { 43456a4e45aSAndre Oppermann refcnt = uma_find_refcnt(zone_clust, mem); 43556a4e45aSAndre Oppermann *refcnt = 1; 43656a4e45aSAndre Oppermann } 437b23f72e9SBrian Feldman return (0); 438099a0e58SBosko Milekic } 439099a0e58SBosko Milekic 44056a4e45aSAndre Oppermann /* 44156a4e45aSAndre Oppermann * The Mbuf Cluster zone destructor. 44256a4e45aSAndre Oppermann */ 443099a0e58SBosko Milekic static void 444099a0e58SBosko Milekic mb_dtor_clust(void *mem, int size, void *arg) 445099a0e58SBosko Milekic { 44656a4e45aSAndre Oppermann 447a5f77087SAndre Oppermann KASSERT(*(uma_find_refcnt(zone_clust, mem)) <= 1, 448a5f77087SAndre Oppermann ("%s: refcnt incorrect %u", __func__, 449a5f77087SAndre Oppermann *(uma_find_refcnt(zone_clust, mem))) ); 450121f0509SMike Silbersack #ifdef INVARIANTS 451121f0509SMike Silbersack trash_dtor(mem, size, arg); 452121f0509SMike Silbersack #endif 453099a0e58SBosko Milekic } 454099a0e58SBosko Milekic 455099a0e58SBosko Milekic /* 456099a0e58SBosko Milekic * The Packet secondary zone's init routine, executed on the 45756a4e45aSAndre Oppermann * object's transition from mbuf keg slab to zone cache. 458099a0e58SBosko Milekic */ 459b23f72e9SBrian Feldman static int 46056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how) 461099a0e58SBosko Milekic { 462099a0e58SBosko Milekic struct mbuf *m; 463099a0e58SBosko Milekic 46456a4e45aSAndre Oppermann m = (struct mbuf *)mem; /* m is virgin. */ 465a7bd90efSAndre Oppermann if (uma_zalloc_arg(zone_clust, m, how) == NULL || 466a7bd90efSAndre Oppermann m->m_ext.ext_buf == NULL) 467b23f72e9SBrian Feldman return (ENOMEM); 468cd5bb63bSAndre Oppermann m->m_ext.ext_type = EXT_PACKET; /* Override. */ 469121f0509SMike Silbersack #ifdef INVARIANTS 470121f0509SMike Silbersack trash_init(m->m_ext.ext_buf, MCLBYTES, how); 471121f0509SMike Silbersack #endif 472b23f72e9SBrian Feldman return (0); 473099a0e58SBosko Milekic } 474099a0e58SBosko Milekic 475099a0e58SBosko Milekic /* 476099a0e58SBosko Milekic * The Packet secondary zone's fini routine, executed on the 477099a0e58SBosko Milekic * object's transition from zone cache to keg slab. 478099a0e58SBosko Milekic */ 479099a0e58SBosko Milekic static void 48056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size) 481099a0e58SBosko Milekic { 482099a0e58SBosko Milekic struct mbuf *m; 483099a0e58SBosko Milekic 484099a0e58SBosko Milekic m = (struct mbuf *)mem; 485121f0509SMike Silbersack #ifdef INVARIANTS 486121f0509SMike Silbersack trash_fini(m->m_ext.ext_buf, MCLBYTES); 487121f0509SMike Silbersack #endif 488099a0e58SBosko Milekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 489a7b844d2SMike Silbersack #ifdef INVARIANTS 490a7b844d2SMike Silbersack trash_dtor(mem, size, NULL); 491a7b844d2SMike Silbersack #endif 492099a0e58SBosko Milekic } 493099a0e58SBosko Milekic 494099a0e58SBosko Milekic /* 495099a0e58SBosko Milekic * The "packet" keg constructor. 496099a0e58SBosko Milekic */ 497b23f72e9SBrian Feldman static int 498b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how) 499099a0e58SBosko Milekic { 500099a0e58SBosko Milekic struct mbuf *m; 501099a0e58SBosko Milekic struct mb_args *args; 502b23f72e9SBrian Feldman #ifdef MAC 503b23f72e9SBrian Feldman int error; 504b23f72e9SBrian Feldman #endif 505b23f72e9SBrian Feldman int flags; 506099a0e58SBosko Milekic short type; 507099a0e58SBosko Milekic 508099a0e58SBosko Milekic m = (struct mbuf *)mem; 509099a0e58SBosko Milekic args = (struct mb_args *)arg; 510099a0e58SBosko Milekic flags = args->flags; 511099a0e58SBosko Milekic type = args->type; 512099a0e58SBosko Milekic 513121f0509SMike Silbersack #ifdef INVARIANTS 514121f0509SMike Silbersack trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 515121f0509SMike Silbersack #endif 516099a0e58SBosko Milekic m->m_next = NULL; 5176bc72ab9SBosko Milekic m->m_nextpkt = NULL; 518099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 51956a4e45aSAndre Oppermann m->m_len = 0; 52056a4e45aSAndre Oppermann m->m_flags = (flags | M_EXT); 52156a4e45aSAndre Oppermann m->m_type = type; 522099a0e58SBosko Milekic 523099a0e58SBosko Milekic if (flags & M_PKTHDR) { 524099a0e58SBosko Milekic m->m_pkthdr.rcvif = NULL; 52556a4e45aSAndre Oppermann m->m_pkthdr.len = 0; 52656a4e45aSAndre Oppermann m->m_pkthdr.header = NULL; 527099a0e58SBosko Milekic m->m_pkthdr.csum_flags = 0; 52856a4e45aSAndre Oppermann m->m_pkthdr.csum_data = 0; 529a855e2b4SAndre Oppermann m->m_pkthdr.tso_segsz = 0; 530a855e2b4SAndre Oppermann m->m_pkthdr.ether_vtag = 0; 531099a0e58SBosko Milekic SLIST_INIT(&m->m_pkthdr.tags); 532099a0e58SBosko Milekic #ifdef MAC 533099a0e58SBosko Milekic /* If the label init fails, fail the alloc */ 534b23f72e9SBrian Feldman error = mac_init_mbuf(m, how); 535b23f72e9SBrian Feldman if (error) 536b23f72e9SBrian Feldman return (error); 537099a0e58SBosko Milekic #endif 538099a0e58SBosko Milekic } 53956a4e45aSAndre Oppermann /* m_ext is already initialized. */ 54056a4e45aSAndre Oppermann 541b23f72e9SBrian Feldman return (0); 542099a0e58SBosko Milekic } 543099a0e58SBosko Milekic 544099a0e58SBosko Milekic /* 545099a0e58SBosko Milekic * This is the protocol drain routine. 546099a0e58SBosko Milekic * 547099a0e58SBosko Milekic * No locks should be held when this is called. The drain routines have to 548099a0e58SBosko Milekic * presently acquire some locks which raises the possibility of lock order 549099a0e58SBosko Milekic * reversal. 550099a0e58SBosko Milekic */ 551099a0e58SBosko Milekic static void 552099a0e58SBosko Milekic mb_reclaim(void *junk) 553099a0e58SBosko Milekic { 554099a0e58SBosko Milekic struct domain *dp; 555099a0e58SBosko Milekic struct protosw *pr; 556099a0e58SBosko Milekic 557099a0e58SBosko Milekic WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, 558099a0e58SBosko Milekic "mb_reclaim()"); 559099a0e58SBosko Milekic 560099a0e58SBosko Milekic for (dp = domains; dp != NULL; dp = dp->dom_next) 561099a0e58SBosko Milekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 562099a0e58SBosko Milekic if (pr->pr_drain != NULL) 563099a0e58SBosko Milekic (*pr->pr_drain)(); 564099a0e58SBosko Milekic } 565