1099a0e58SBosko Milekic /*- 28076cb52SBosko Milekic * Copyright (c) 2004, 2005, 38076cb52SBosko Milekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 4099a0e58SBosko Milekic * 5099a0e58SBosko Milekic * Redistribution and use in source and binary forms, with or without 6099a0e58SBosko Milekic * modification, are permitted provided that the following conditions 7099a0e58SBosko Milekic * are met: 8099a0e58SBosko Milekic * 1. Redistributions of source code must retain the above copyright 9099a0e58SBosko Milekic * notice unmodified, this list of conditions and the following 10099a0e58SBosko Milekic * disclaimer. 11099a0e58SBosko Milekic * 2. Redistributions in binary form must reproduce the above copyright 12099a0e58SBosko Milekic * notice, this list of conditions and the following disclaimer in the 13099a0e58SBosko Milekic * documentation and/or other materials provided with the distribution. 14099a0e58SBosko Milekic * 15099a0e58SBosko Milekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16099a0e58SBosko Milekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17099a0e58SBosko Milekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18099a0e58SBosko Milekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19099a0e58SBosko Milekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20099a0e58SBosko Milekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21099a0e58SBosko Milekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22099a0e58SBosko Milekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23099a0e58SBosko Milekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24099a0e58SBosko Milekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25099a0e58SBosko Milekic * SUCH DAMAGE. 26099a0e58SBosko Milekic */ 27099a0e58SBosko Milekic 28099a0e58SBosko Milekic #include <sys/cdefs.h> 29099a0e58SBosko Milekic __FBSDID("$FreeBSD$"); 30099a0e58SBosko Milekic 31099a0e58SBosko Milekic #include "opt_param.h" 32099a0e58SBosko Milekic 33099a0e58SBosko Milekic #include <sys/param.h> 34099a0e58SBosko Milekic #include <sys/malloc.h> 3554503a13SJonathan T. Looney #include <sys/types.h> 36099a0e58SBosko Milekic #include <sys/systm.h> 37099a0e58SBosko Milekic #include <sys/mbuf.h> 38099a0e58SBosko Milekic #include <sys/domain.h> 39099a0e58SBosko Milekic #include <sys/eventhandler.h> 40099a0e58SBosko Milekic #include <sys/kernel.h> 4154503a13SJonathan T. Looney #include <sys/lock.h> 4254503a13SJonathan T. Looney #include <sys/mutex.h> 43099a0e58SBosko Milekic #include <sys/protosw.h> 44099a0e58SBosko Milekic #include <sys/smp.h> 45099a0e58SBosko Milekic #include <sys/sysctl.h> 46099a0e58SBosko Milekic 47099a0e58SBosko Milekic #include <vm/vm.h> 48c45c0034SAlan Cox #include <vm/vm_extern.h> 49c45c0034SAlan Cox #include <vm/vm_kern.h> 50099a0e58SBosko Milekic #include <vm/vm_page.h> 5137140716SAndre Oppermann #include <vm/vm_map.h> 52099a0e58SBosko Milekic #include <vm/uma.h> 53121f0509SMike Silbersack #include <vm/uma_dbg.h> 54099a0e58SBosko Milekic 55099a0e58SBosko Milekic /* 56099a0e58SBosko Milekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 57099a0e58SBosko Milekic * Zones. 58099a0e58SBosko Milekic * 59099a0e58SBosko Milekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 60099a0e58SBosko Milekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 61099a0e58SBosko Milekic * administrator so desires. 62099a0e58SBosko Milekic * 63099a0e58SBosko Milekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 64099a0e58SBosko Milekic * Zone. 65099a0e58SBosko Milekic * 66099a0e58SBosko Milekic * Additionally, FreeBSD provides a Packet Zone, which it 67099a0e58SBosko Milekic * configures as a Secondary Zone to the Mbuf Master Zone, 68099a0e58SBosko Milekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 69099a0e58SBosko Milekic * 70099a0e58SBosko Milekic * Thus common-case allocations and locking are simplified: 71099a0e58SBosko Milekic * 72099a0e58SBosko Milekic * m_clget() m_getcl() 73099a0e58SBosko Milekic * | | 74099a0e58SBosko Milekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 75099a0e58SBosko Milekic * | | [ Packet ] | 76099a0e58SBosko Milekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 77099a0e58SBosko Milekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 78099a0e58SBosko Milekic * | \________ | 79099a0e58SBosko Milekic * [ Cluster Keg ] \ / 80099a0e58SBosko Milekic * | [ Mbuf Keg ] 81099a0e58SBosko Milekic * [ Cluster Slabs ] | 82099a0e58SBosko Milekic * | [ Mbuf Slabs ] 83099a0e58SBosko Milekic * \____________(VM)_________________/ 8456a4e45aSAndre Oppermann * 8556a4e45aSAndre Oppermann * 86fcf90618SGleb Smirnoff * Whenever an object is allocated with uma_zalloc() out of 8756a4e45aSAndre Oppermann * one of the Zones its _ctor_ function is executed. The same 88fcf90618SGleb Smirnoff * for any deallocation through uma_zfree() the _dtor_ function 8956a4e45aSAndre Oppermann * is executed. 9056a4e45aSAndre Oppermann * 9156a4e45aSAndre Oppermann * Caches are per-CPU and are filled from the Master Zone. 9256a4e45aSAndre Oppermann * 93fcf90618SGleb Smirnoff * Whenever an object is allocated from the underlying global 9456a4e45aSAndre Oppermann * memory pool it gets pre-initialized with the _zinit_ functions. 9556a4e45aSAndre Oppermann * When the Keg's are overfull objects get decomissioned with 9656a4e45aSAndre Oppermann * _zfini_ functions and free'd back to the global memory pool. 9756a4e45aSAndre Oppermann * 98099a0e58SBosko Milekic */ 99099a0e58SBosko Milekic 100ead46972SAndre Oppermann int nmbufs; /* limits number of mbufs */ 10156a4e45aSAndre Oppermann int nmbclusters; /* limits number of mbuf clusters */ 102ec63cb90SAndre Oppermann int nmbjumbop; /* limits number of page size jumbo clusters */ 10356a4e45aSAndre Oppermann int nmbjumbo9; /* limits number of 9k jumbo clusters */ 10456a4e45aSAndre Oppermann int nmbjumbo16; /* limits number of 16k jumbo clusters */ 105099a0e58SBosko Milekic 106e0c00addSAndre Oppermann static quad_t maxmbufmem; /* overall real memory limit for all mbufs */ 107e0c00addSAndre Oppermann 108af3b2549SHans Petter Selasky SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0, 109b6f49c23SHiren Panchasara "Maximum real memory allocatable to various mbuf types"); 110e0c00addSAndre Oppermann 11162938659SBjoern A. Zeeb /* 11237140716SAndre Oppermann * tunable_mbinit() has to be run before any mbuf allocations are done. 11362938659SBjoern A. Zeeb */ 114099a0e58SBosko Milekic static void 115099a0e58SBosko Milekic tunable_mbinit(void *dummy) 116099a0e58SBosko Milekic { 117e0c00addSAndre Oppermann quad_t realmem; 11837140716SAndre Oppermann 11937140716SAndre Oppermann /* 12037140716SAndre Oppermann * The default limit for all mbuf related memory is 1/2 of all 12137140716SAndre Oppermann * available kernel memory (physical or kmem). 12237140716SAndre Oppermann * At most it can be 3/4 of available kernel memory. 12337140716SAndre Oppermann */ 1245df87b21SJeff Roberson realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size); 12537140716SAndre Oppermann maxmbufmem = realmem / 2; 126e0c00addSAndre Oppermann TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem); 12737140716SAndre Oppermann if (maxmbufmem > realmem / 4 * 3) 12837140716SAndre Oppermann maxmbufmem = realmem / 4 * 3; 129099a0e58SBosko Milekic 130812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 131416a434cSAndre Oppermann if (nmbclusters == 0) 132416a434cSAndre Oppermann nmbclusters = maxmbufmem / MCLBYTES / 4; 133812302c3SNavdeep Parhar 134812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop); 135812302c3SNavdeep Parhar if (nmbjumbop == 0) 136416a434cSAndre Oppermann nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4; 137812302c3SNavdeep Parhar 138812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9); 139812302c3SNavdeep Parhar if (nmbjumbo9 == 0) 140416a434cSAndre Oppermann nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6; 141812302c3SNavdeep Parhar 142812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16); 143812302c3SNavdeep Parhar if (nmbjumbo16 == 0) 144416a434cSAndre Oppermann nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6; 145416a434cSAndre Oppermann 146416a434cSAndre Oppermann /* 147416a434cSAndre Oppermann * We need at least as many mbufs as we have clusters of 148416a434cSAndre Oppermann * the various types added together. 149416a434cSAndre Oppermann */ 150416a434cSAndre Oppermann TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs); 151416a434cSAndre Oppermann if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) 152416a434cSAndre Oppermann nmbufs = lmax(maxmbufmem / MSIZE / 5, 153416a434cSAndre Oppermann nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16); 154099a0e58SBosko Milekic } 15537140716SAndre Oppermann SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL); 156099a0e58SBosko Milekic 1574f590175SPaul Saab static int 1584f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS) 1594f590175SPaul Saab { 1604f590175SPaul Saab int error, newnmbclusters; 1614f590175SPaul Saab 1624f590175SPaul Saab newnmbclusters = nmbclusters; 163041b706bSDavid Malone error = sysctl_handle_int(oidp, &newnmbclusters, 0, req); 164d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbclusters != nmbclusters) { 165ead46972SAndre Oppermann if (newnmbclusters > nmbclusters && 166ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 1674f590175SPaul Saab nmbclusters = newnmbclusters; 168bc4a1b8cSAndre Oppermann nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 1694f590175SPaul Saab EVENTHANDLER_INVOKE(nmbclusters_change); 1704f590175SPaul Saab } else 1714f590175SPaul Saab error = EINVAL; 1724f590175SPaul Saab } 1734f590175SPaul Saab return (error); 1744f590175SPaul Saab } 1754f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW, 1764f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU", 177099a0e58SBosko Milekic "Maximum number of mbuf clusters allowed"); 178cf70a46bSRandall Stewart 179cf70a46bSRandall Stewart static int 180cf70a46bSRandall Stewart sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS) 181cf70a46bSRandall Stewart { 182cf70a46bSRandall Stewart int error, newnmbjumbop; 183cf70a46bSRandall Stewart 184cf70a46bSRandall Stewart newnmbjumbop = nmbjumbop; 185cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req); 186d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) { 187ead46972SAndre Oppermann if (newnmbjumbop > nmbjumbop && 188ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 189cf70a46bSRandall Stewart nmbjumbop = newnmbjumbop; 190bc4a1b8cSAndre Oppermann nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 191cf70a46bSRandall Stewart } else 192cf70a46bSRandall Stewart error = EINVAL; 193cf70a46bSRandall Stewart } 194cf70a46bSRandall Stewart return (error); 195cf70a46bSRandall Stewart } 196cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW, 197cf70a46bSRandall Stewart &nmbjumbop, 0, sysctl_nmbjumbop, "IU", 198ec63cb90SAndre Oppermann "Maximum number of mbuf page size jumbo clusters allowed"); 199cf70a46bSRandall Stewart 200cf70a46bSRandall Stewart static int 201cf70a46bSRandall Stewart sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS) 202cf70a46bSRandall Stewart { 203cf70a46bSRandall Stewart int error, newnmbjumbo9; 204cf70a46bSRandall Stewart 205cf70a46bSRandall Stewart newnmbjumbo9 = nmbjumbo9; 206cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req); 207d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) { 208ead46972SAndre Oppermann if (newnmbjumbo9 > nmbjumbo9 && 209ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 210cf70a46bSRandall Stewart nmbjumbo9 = newnmbjumbo9; 211bc4a1b8cSAndre Oppermann nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 212cf70a46bSRandall Stewart } else 213cf70a46bSRandall Stewart error = EINVAL; 214cf70a46bSRandall Stewart } 215cf70a46bSRandall Stewart return (error); 216cf70a46bSRandall Stewart } 217cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW, 218cf70a46bSRandall Stewart &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU", 21956a4e45aSAndre Oppermann "Maximum number of mbuf 9k jumbo clusters allowed"); 220cf70a46bSRandall Stewart 221cf70a46bSRandall Stewart static int 222cf70a46bSRandall Stewart sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS) 223cf70a46bSRandall Stewart { 224cf70a46bSRandall Stewart int error, newnmbjumbo16; 225cf70a46bSRandall Stewart 226cf70a46bSRandall Stewart newnmbjumbo16 = nmbjumbo16; 227cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req); 228d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) { 229ead46972SAndre Oppermann if (newnmbjumbo16 > nmbjumbo16 && 230ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 231cf70a46bSRandall Stewart nmbjumbo16 = newnmbjumbo16; 232bc4a1b8cSAndre Oppermann nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 233cf70a46bSRandall Stewart } else 234cf70a46bSRandall Stewart error = EINVAL; 235cf70a46bSRandall Stewart } 236cf70a46bSRandall Stewart return (error); 237cf70a46bSRandall Stewart } 238cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW, 239cf70a46bSRandall Stewart &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU", 24056a4e45aSAndre Oppermann "Maximum number of mbuf 16k jumbo clusters allowed"); 241cf70a46bSRandall Stewart 242ead46972SAndre Oppermann static int 243ead46972SAndre Oppermann sysctl_nmbufs(SYSCTL_HANDLER_ARGS) 244ead46972SAndre Oppermann { 245ead46972SAndre Oppermann int error, newnmbufs; 246ead46972SAndre Oppermann 247ead46972SAndre Oppermann newnmbufs = nmbufs; 248ead46972SAndre Oppermann error = sysctl_handle_int(oidp, &newnmbufs, 0, req); 249d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbufs != nmbufs) { 250ead46972SAndre Oppermann if (newnmbufs > nmbufs) { 251ead46972SAndre Oppermann nmbufs = newnmbufs; 252bc4a1b8cSAndre Oppermann nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 253ead46972SAndre Oppermann EVENTHANDLER_INVOKE(nmbufs_change); 254ead46972SAndre Oppermann } else 255ead46972SAndre Oppermann error = EINVAL; 256ead46972SAndre Oppermann } 257ead46972SAndre Oppermann return (error); 258ead46972SAndre Oppermann } 259e0c00addSAndre Oppermann SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW, 260ead46972SAndre Oppermann &nmbufs, 0, sysctl_nmbufs, "IU", 261ead46972SAndre Oppermann "Maximum number of mbufs allowed"); 262cf70a46bSRandall Stewart 263099a0e58SBosko Milekic /* 264099a0e58SBosko Milekic * Zones from which we allocate. 265099a0e58SBosko Milekic */ 266099a0e58SBosko Milekic uma_zone_t zone_mbuf; 267099a0e58SBosko Milekic uma_zone_t zone_clust; 268099a0e58SBosko Milekic uma_zone_t zone_pack; 269ec63cb90SAndre Oppermann uma_zone_t zone_jumbop; 27056a4e45aSAndre Oppermann uma_zone_t zone_jumbo9; 27156a4e45aSAndre Oppermann uma_zone_t zone_jumbo16; 272099a0e58SBosko Milekic 273099a0e58SBosko Milekic /* 274099a0e58SBosko Milekic * Local prototypes. 275099a0e58SBosko Milekic */ 276b23f72e9SBrian Feldman static int mb_ctor_mbuf(void *, int, void *, int); 277b23f72e9SBrian Feldman static int mb_ctor_clust(void *, int, void *, int); 278b23f72e9SBrian Feldman static int mb_ctor_pack(void *, int, void *, int); 279099a0e58SBosko Milekic static void mb_dtor_mbuf(void *, int, void *); 28056a4e45aSAndre Oppermann static void mb_dtor_pack(void *, int, void *); 28156a4e45aSAndre Oppermann static int mb_zinit_pack(void *, int, int); 28256a4e45aSAndre Oppermann static void mb_zfini_pack(void *, int); 283e60b2fcbSGleb Smirnoff static void mb_reclaim(uma_zone_t, int); 284f2c2231eSRyan Stone static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int); 285099a0e58SBosko Milekic 28637140716SAndre Oppermann /* Ensure that MSIZE is a power of 2. */ 287a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 288a04946cfSBrian Somers 289099a0e58SBosko Milekic /* 290099a0e58SBosko Milekic * Initialize FreeBSD Network buffer allocation. 291099a0e58SBosko Milekic */ 292099a0e58SBosko Milekic static void 293099a0e58SBosko Milekic mbuf_init(void *dummy) 294099a0e58SBosko Milekic { 295099a0e58SBosko Milekic 296099a0e58SBosko Milekic /* 297099a0e58SBosko Milekic * Configure UMA zones for Mbufs, Clusters, and Packets. 298099a0e58SBosko Milekic */ 29956a4e45aSAndre Oppermann zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 30056a4e45aSAndre Oppermann mb_ctor_mbuf, mb_dtor_mbuf, 301121f0509SMike Silbersack #ifdef INVARIANTS 30256a4e45aSAndre Oppermann trash_init, trash_fini, 303121f0509SMike Silbersack #else 30456a4e45aSAndre Oppermann NULL, NULL, 305121f0509SMike Silbersack #endif 30656a4e45aSAndre Oppermann MSIZE - 1, UMA_ZONE_MAXBUCKET); 30745fe0bf7SPawel Jakub Dawidek if (nmbufs > 0) 30845fe0bf7SPawel Jakub Dawidek nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 3096e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); 310e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_mbuf, mb_reclaim); 31156a4e45aSAndre Oppermann 31268352adfSRobert Watson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 31356a5f52eSGleb Smirnoff mb_ctor_clust, 314121f0509SMike Silbersack #ifdef INVARIANTS 31556a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 316121f0509SMike Silbersack #else 31756a5f52eSGleb Smirnoff NULL, NULL, NULL, 318121f0509SMike Silbersack #endif 31956a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 32045fe0bf7SPawel Jakub Dawidek if (nmbclusters > 0) 32145fe0bf7SPawel Jakub Dawidek nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 3226e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); 323e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_clust, mb_reclaim); 324099a0e58SBosko Milekic 32556a4e45aSAndre Oppermann zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 32656a4e45aSAndre Oppermann mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 32756a4e45aSAndre Oppermann 328fcf90618SGleb Smirnoff /* Make jumbo frame zone too. Page size, 9k and 16k. */ 329ec63cb90SAndre Oppermann zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 33056a5f52eSGleb Smirnoff mb_ctor_clust, 331d5269a63SAndre Oppermann #ifdef INVARIANTS 33256a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 333d5269a63SAndre Oppermann #else 33456a5f52eSGleb Smirnoff NULL, NULL, NULL, 335d5269a63SAndre Oppermann #endif 33656a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 33745fe0bf7SPawel Jakub Dawidek if (nmbjumbop > 0) 33845fe0bf7SPawel Jakub Dawidek nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 3396e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); 340e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbop, mb_reclaim); 341d5269a63SAndre Oppermann 34256a4e45aSAndre Oppermann zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 34356a5f52eSGleb Smirnoff mb_ctor_clust, 34456a4e45aSAndre Oppermann #ifdef INVARIANTS 34556a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 34656a4e45aSAndre Oppermann #else 34756a5f52eSGleb Smirnoff NULL, NULL, NULL, 34856a4e45aSAndre Oppermann #endif 34956a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 350ba63339aSAlan Cox uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); 35145fe0bf7SPawel Jakub Dawidek if (nmbjumbo9 > 0) 35245fe0bf7SPawel Jakub Dawidek nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 3536e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); 354e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbo9, mb_reclaim); 35556a4e45aSAndre Oppermann 35656a4e45aSAndre Oppermann zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 35756a5f52eSGleb Smirnoff mb_ctor_clust, 35856a4e45aSAndre Oppermann #ifdef INVARIANTS 35956a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 36056a4e45aSAndre Oppermann #else 36156a5f52eSGleb Smirnoff NULL, NULL, NULL, 36256a4e45aSAndre Oppermann #endif 36356a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 364ba63339aSAlan Cox uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); 36545fe0bf7SPawel Jakub Dawidek if (nmbjumbo16 > 0) 36645fe0bf7SPawel Jakub Dawidek nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 3676e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); 368e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbo16, mb_reclaim); 36956a4e45aSAndre Oppermann 370099a0e58SBosko Milekic /* 371099a0e58SBosko Milekic * Hook event handler for low-memory situation, used to 372099a0e58SBosko Milekic * drain protocols and push data back to the caches (UMA 373099a0e58SBosko Milekic * later pushes it back to VM). 374099a0e58SBosko Milekic */ 375099a0e58SBosko Milekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 376099a0e58SBosko Milekic EVENTHANDLER_PRI_FIRST); 377099a0e58SBosko Milekic } 37837140716SAndre Oppermann SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL); 379099a0e58SBosko Milekic 380099a0e58SBosko Milekic /* 381ba63339aSAlan Cox * UMA backend page allocator for the jumbo frame zones. 382ba63339aSAlan Cox * 383ba63339aSAlan Cox * Allocates kernel virtual memory that is backed by contiguous physical 384ba63339aSAlan Cox * pages. 385ba63339aSAlan Cox */ 386ba63339aSAlan Cox static void * 387f2c2231eSRyan Stone mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait) 388ba63339aSAlan Cox { 389ba63339aSAlan Cox 3907630c265SAlan Cox /* Inform UMA that this allocator uses kernel_map/object. */ 3917630c265SAlan Cox *flags = UMA_SLAB_KERNEL; 3925df87b21SJeff Roberson return ((void *)kmem_alloc_contig(kernel_arena, bytes, wait, 3933153e878SAlan Cox (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); 394ba63339aSAlan Cox } 395ba63339aSAlan Cox 396ba63339aSAlan Cox /* 397099a0e58SBosko Milekic * Constructor for Mbuf master zone. 398099a0e58SBosko Milekic * 399099a0e58SBosko Milekic * The 'arg' pointer points to a mb_args structure which 400099a0e58SBosko Milekic * contains call-specific information required to support the 40156a4e45aSAndre Oppermann * mbuf allocation API. See mbuf.h. 402099a0e58SBosko Milekic */ 403b23f72e9SBrian Feldman static int 404b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how) 405099a0e58SBosko Milekic { 406099a0e58SBosko Milekic struct mbuf *m; 407099a0e58SBosko Milekic struct mb_args *args; 408b23f72e9SBrian Feldman int error; 409099a0e58SBosko Milekic int flags; 410099a0e58SBosko Milekic short type; 411099a0e58SBosko Milekic 412121f0509SMike Silbersack #ifdef INVARIANTS 413121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 414121f0509SMike Silbersack #endif 415099a0e58SBosko Milekic args = (struct mb_args *)arg; 416099a0e58SBosko Milekic type = args->type; 417099a0e58SBosko Milekic 41856a4e45aSAndre Oppermann /* 41956a4e45aSAndre Oppermann * The mbuf is initialized later. The caller has the 420fcf90618SGleb Smirnoff * responsibility to set up any MAC labels too. 42156a4e45aSAndre Oppermann */ 42256a4e45aSAndre Oppermann if (type == MT_NOINIT) 42356a4e45aSAndre Oppermann return (0); 42456a4e45aSAndre Oppermann 425afb295ccSAndre Oppermann m = (struct mbuf *)mem; 426afb295ccSAndre Oppermann flags = args->flags; 427afb295ccSAndre Oppermann 428b4b12e52SGleb Smirnoff error = m_init(m, how, type, flags); 429afb295ccSAndre Oppermann 430b23f72e9SBrian Feldman return (error); 431099a0e58SBosko Milekic } 432099a0e58SBosko Milekic 433099a0e58SBosko Milekic /* 43456a4e45aSAndre Oppermann * The Mbuf master zone destructor. 435099a0e58SBosko Milekic */ 436099a0e58SBosko Milekic static void 437099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg) 438099a0e58SBosko Milekic { 439099a0e58SBosko Milekic struct mbuf *m; 440629b9e08SKip Macy unsigned long flags; 441099a0e58SBosko Milekic 442099a0e58SBosko Milekic m = (struct mbuf *)mem; 443629b9e08SKip Macy flags = (unsigned long)arg; 444629b9e08SKip Macy 445a9fa76f2SNavdeep Parhar KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__)); 446ce6169e7SAndre Oppermann if ((m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags)) 447099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 448121f0509SMike Silbersack #ifdef INVARIANTS 449121f0509SMike Silbersack trash_dtor(mem, size, arg); 450121f0509SMike Silbersack #endif 451099a0e58SBosko Milekic } 452099a0e58SBosko Milekic 45356a4e45aSAndre Oppermann /* 45456a4e45aSAndre Oppermann * The Mbuf Packet zone destructor. 45556a4e45aSAndre Oppermann */ 456099a0e58SBosko Milekic static void 457099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg) 458099a0e58SBosko Milekic { 459099a0e58SBosko Milekic struct mbuf *m; 460099a0e58SBosko Milekic 461099a0e58SBosko Milekic m = (struct mbuf *)mem; 462099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 463099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 46456a4e45aSAndre Oppermann 46556a4e45aSAndre Oppermann /* Make sure we've got a clean cluster back. */ 46656a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 46756a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 46856a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 469cf827063SPoul-Henning Kamp KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__)); 470cf827063SPoul-Henning Kamp KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__)); 47156a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 47249d46b61SGleb Smirnoff KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); 473121f0509SMike Silbersack #ifdef INVARIANTS 474121f0509SMike Silbersack trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 475121f0509SMike Silbersack #endif 4766c125b8dSMohan Srinivasan /* 477ef44c8d2SDavid E. O'Brien * If there are processes blocked on zone_clust, waiting for pages 478ef44c8d2SDavid E. O'Brien * to be freed up, * cause them to be woken up by draining the 479ef44c8d2SDavid E. O'Brien * packet zone. We are exposed to a race here * (in the check for 480ef44c8d2SDavid E. O'Brien * the UMA_ZFLAG_FULL) where we might miss the flag set, but that 481ef44c8d2SDavid E. O'Brien * is deliberate. We don't want to acquire the zone lock for every 482ef44c8d2SDavid E. O'Brien * mbuf free. 4836c125b8dSMohan Srinivasan */ 4846c125b8dSMohan Srinivasan if (uma_zone_exhausted_nolock(zone_clust)) 4856c125b8dSMohan Srinivasan zone_drain(zone_pack); 486099a0e58SBosko Milekic } 487099a0e58SBosko Milekic 488099a0e58SBosko Milekic /* 489ec63cb90SAndre Oppermann * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. 490099a0e58SBosko Milekic * 491099a0e58SBosko Milekic * Here the 'arg' pointer points to the Mbuf which we 49256a4e45aSAndre Oppermann * are configuring cluster storage for. If 'arg' is 49356a4e45aSAndre Oppermann * empty we allocate just the cluster without setting 49456a4e45aSAndre Oppermann * the mbuf to it. See mbuf.h. 495099a0e58SBosko Milekic */ 496b23f72e9SBrian Feldman static int 497b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how) 498099a0e58SBosko Milekic { 499099a0e58SBosko Milekic struct mbuf *m; 500099a0e58SBosko Milekic 501121f0509SMike Silbersack #ifdef INVARIANTS 502121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 503121f0509SMike Silbersack #endif 5040f4d9d04SKip Macy m = (struct mbuf *)arg; 5050f4d9d04SKip Macy if (m != NULL) { 506099a0e58SBosko Milekic m->m_ext.ext_buf = (caddr_t)mem; 507099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 508099a0e58SBosko Milekic m->m_flags |= M_EXT; 509099a0e58SBosko Milekic m->m_ext.ext_free = NULL; 510cf827063SPoul-Henning Kamp m->m_ext.ext_arg1 = NULL; 511cf827063SPoul-Henning Kamp m->m_ext.ext_arg2 = NULL; 51256a4e45aSAndre Oppermann m->m_ext.ext_size = size; 51356a5f52eSGleb Smirnoff m->m_ext.ext_type = m_gettype(size); 51456a5f52eSGleb Smirnoff m->m_ext.ext_flags = EXT_FLAG_EMBREF; 51556a5f52eSGleb Smirnoff m->m_ext.ext_count = 1; 51656a4e45aSAndre Oppermann } 5170f4d9d04SKip Macy 518b23f72e9SBrian Feldman return (0); 519099a0e58SBosko Milekic } 520099a0e58SBosko Milekic 52156a4e45aSAndre Oppermann /* 522099a0e58SBosko Milekic * The Packet secondary zone's init routine, executed on the 52356a4e45aSAndre Oppermann * object's transition from mbuf keg slab to zone cache. 524099a0e58SBosko Milekic */ 525b23f72e9SBrian Feldman static int 52656a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how) 527099a0e58SBosko Milekic { 528099a0e58SBosko Milekic struct mbuf *m; 529099a0e58SBosko Milekic 53056a4e45aSAndre Oppermann m = (struct mbuf *)mem; /* m is virgin. */ 531a7bd90efSAndre Oppermann if (uma_zalloc_arg(zone_clust, m, how) == NULL || 532a7bd90efSAndre Oppermann m->m_ext.ext_buf == NULL) 533b23f72e9SBrian Feldman return (ENOMEM); 534cd5bb63bSAndre Oppermann m->m_ext.ext_type = EXT_PACKET; /* Override. */ 535121f0509SMike Silbersack #ifdef INVARIANTS 536121f0509SMike Silbersack trash_init(m->m_ext.ext_buf, MCLBYTES, how); 537121f0509SMike Silbersack #endif 538b23f72e9SBrian Feldman return (0); 539099a0e58SBosko Milekic } 540099a0e58SBosko Milekic 541099a0e58SBosko Milekic /* 542099a0e58SBosko Milekic * The Packet secondary zone's fini routine, executed on the 543099a0e58SBosko Milekic * object's transition from zone cache to keg slab. 544099a0e58SBosko Milekic */ 545099a0e58SBosko Milekic static void 54656a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size) 547099a0e58SBosko Milekic { 548099a0e58SBosko Milekic struct mbuf *m; 549099a0e58SBosko Milekic 550099a0e58SBosko Milekic m = (struct mbuf *)mem; 551121f0509SMike Silbersack #ifdef INVARIANTS 552121f0509SMike Silbersack trash_fini(m->m_ext.ext_buf, MCLBYTES); 553121f0509SMike Silbersack #endif 554099a0e58SBosko Milekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 555a7b844d2SMike Silbersack #ifdef INVARIANTS 556a7b844d2SMike Silbersack trash_dtor(mem, size, NULL); 557a7b844d2SMike Silbersack #endif 558099a0e58SBosko Milekic } 559099a0e58SBosko Milekic 560099a0e58SBosko Milekic /* 561099a0e58SBosko Milekic * The "packet" keg constructor. 562099a0e58SBosko Milekic */ 563b23f72e9SBrian Feldman static int 564b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how) 565099a0e58SBosko Milekic { 566099a0e58SBosko Milekic struct mbuf *m; 567099a0e58SBosko Milekic struct mb_args *args; 568ce28636bSAndre Oppermann int error, flags; 569099a0e58SBosko Milekic short type; 570099a0e58SBosko Milekic 571099a0e58SBosko Milekic m = (struct mbuf *)mem; 572099a0e58SBosko Milekic args = (struct mb_args *)arg; 573099a0e58SBosko Milekic flags = args->flags; 574099a0e58SBosko Milekic type = args->type; 575099a0e58SBosko Milekic 576121f0509SMike Silbersack #ifdef INVARIANTS 577121f0509SMike Silbersack trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 578121f0509SMike Silbersack #endif 579099a0e58SBosko Milekic 580b4b12e52SGleb Smirnoff error = m_init(m, how, type, flags); 581afb295ccSAndre Oppermann 58256a4e45aSAndre Oppermann /* m_ext is already initialized. */ 583afb295ccSAndre Oppermann m->m_data = m->m_ext.ext_buf; 584afb295ccSAndre Oppermann m->m_flags = (flags | M_EXT); 58556a4e45aSAndre Oppermann 586afb295ccSAndre Oppermann return (error); 587099a0e58SBosko Milekic } 588099a0e58SBosko Milekic 589099a0e58SBosko Milekic /* 590e60b2fcbSGleb Smirnoff * This is the protocol drain routine. Called by UMA whenever any of the 591e60b2fcbSGleb Smirnoff * mbuf zones is closed to its limit. 592099a0e58SBosko Milekic * 593099a0e58SBosko Milekic * No locks should be held when this is called. The drain routines have to 594099a0e58SBosko Milekic * presently acquire some locks which raises the possibility of lock order 595099a0e58SBosko Milekic * reversal. 596099a0e58SBosko Milekic */ 597099a0e58SBosko Milekic static void 598e60b2fcbSGleb Smirnoff mb_reclaim(uma_zone_t zone __unused, int pending __unused) 599099a0e58SBosko Milekic { 600099a0e58SBosko Milekic struct domain *dp; 601099a0e58SBosko Milekic struct protosw *pr; 602099a0e58SBosko Milekic 603e60b2fcbSGleb Smirnoff WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__); 604099a0e58SBosko Milekic 605099a0e58SBosko Milekic for (dp = domains; dp != NULL; dp = dp->dom_next) 606099a0e58SBosko Milekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 607099a0e58SBosko Milekic if (pr->pr_drain != NULL) 608099a0e58SBosko Milekic (*pr->pr_drain)(); 609099a0e58SBosko Milekic } 6105e4bc63bSGleb Smirnoff 6115e4bc63bSGleb Smirnoff /* 6125e4bc63bSGleb Smirnoff * Clean up after mbufs with M_EXT storage attached to them if the 6135e4bc63bSGleb Smirnoff * reference count hits 1. 6145e4bc63bSGleb Smirnoff */ 6155e4bc63bSGleb Smirnoff void 6165e4bc63bSGleb Smirnoff mb_free_ext(struct mbuf *m) 6175e4bc63bSGleb Smirnoff { 61856a5f52eSGleb Smirnoff volatile u_int *refcnt; 61956a5f52eSGleb Smirnoff struct mbuf *mref; 6205e4bc63bSGleb Smirnoff int freembuf; 6215e4bc63bSGleb Smirnoff 6225e4bc63bSGleb Smirnoff KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); 6235e4bc63bSGleb Smirnoff 62456a5f52eSGleb Smirnoff /* See if this is the mbuf that holds the embedded refcount. */ 62556a5f52eSGleb Smirnoff if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { 62656a5f52eSGleb Smirnoff refcnt = &m->m_ext.ext_count; 62756a5f52eSGleb Smirnoff mref = m; 62856a5f52eSGleb Smirnoff } else { 62956a5f52eSGleb Smirnoff KASSERT(m->m_ext.ext_cnt != NULL, 63056a5f52eSGleb Smirnoff ("%s: no refcounting pointer on %p", __func__, m)); 63156a5f52eSGleb Smirnoff refcnt = m->m_ext.ext_cnt; 63256a5f52eSGleb Smirnoff mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 63356a5f52eSGleb Smirnoff } 63456a5f52eSGleb Smirnoff 6355e4bc63bSGleb Smirnoff /* 63656a5f52eSGleb Smirnoff * Check if the header is embedded in the cluster. It is 63756a5f52eSGleb Smirnoff * important that we can't touch any of the mbuf fields 63856a5f52eSGleb Smirnoff * after we have freed the external storage, since mbuf 63956a5f52eSGleb Smirnoff * could have been embedded in it. 6405e4bc63bSGleb Smirnoff */ 6415e4bc63bSGleb Smirnoff freembuf = (m->m_flags & M_NOFREE) ? 0 : 1; 6425e4bc63bSGleb Smirnoff 64356a5f52eSGleb Smirnoff /* Free attached storage if this mbuf is the only reference to it. */ 64456a5f52eSGleb Smirnoff if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) { 6455e4bc63bSGleb Smirnoff switch (m->m_ext.ext_type) { 64656a5f52eSGleb Smirnoff case EXT_PACKET: 64756a5f52eSGleb Smirnoff /* The packet zone is special. */ 64856a5f52eSGleb Smirnoff if (*refcnt == 0) 64956a5f52eSGleb Smirnoff *refcnt = 1; 65056a5f52eSGleb Smirnoff uma_zfree(zone_pack, mref); 6515e4bc63bSGleb Smirnoff break; 6525e4bc63bSGleb Smirnoff case EXT_CLUSTER: 6535e4bc63bSGleb Smirnoff uma_zfree(zone_clust, m->m_ext.ext_buf); 65456a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 6555e4bc63bSGleb Smirnoff break; 6565e4bc63bSGleb Smirnoff case EXT_JUMBOP: 6575e4bc63bSGleb Smirnoff uma_zfree(zone_jumbop, m->m_ext.ext_buf); 65856a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 6595e4bc63bSGleb Smirnoff break; 6605e4bc63bSGleb Smirnoff case EXT_JUMBO9: 6615e4bc63bSGleb Smirnoff uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 66256a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 6635e4bc63bSGleb Smirnoff break; 6645e4bc63bSGleb Smirnoff case EXT_JUMBO16: 6655e4bc63bSGleb Smirnoff uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 66656a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 66756a5f52eSGleb Smirnoff break; 66856a5f52eSGleb Smirnoff case EXT_SFBUF: 66956a5f52eSGleb Smirnoff sf_ext_free(m->m_ext.ext_arg1, m->m_ext.ext_arg2); 67056a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 67156a5f52eSGleb Smirnoff break; 67256a5f52eSGleb Smirnoff case EXT_SFBUF_NOCACHE: 67356a5f52eSGleb Smirnoff sf_ext_free_nocache(m->m_ext.ext_arg1, 67456a5f52eSGleb Smirnoff m->m_ext.ext_arg2); 67556a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 6765e4bc63bSGleb Smirnoff break; 6775e4bc63bSGleb Smirnoff case EXT_NET_DRV: 6785e4bc63bSGleb Smirnoff case EXT_MOD_TYPE: 6795e4bc63bSGleb Smirnoff case EXT_DISPOSABLE: 680*0ea37a86SGleb Smirnoff KASSERT(m->m_ext.ext_free != NULL, 681*0ea37a86SGleb Smirnoff ("%s: ext_free not set", __func__)); 682*0ea37a86SGleb Smirnoff (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, 683*0ea37a86SGleb Smirnoff m->m_ext.ext_arg2); 68456a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 685*0ea37a86SGleb Smirnoff break; 6865e4bc63bSGleb Smirnoff case EXT_EXTREF: 6875e4bc63bSGleb Smirnoff KASSERT(m->m_ext.ext_free != NULL, 6885e4bc63bSGleb Smirnoff ("%s: ext_free not set", __func__)); 6895e4bc63bSGleb Smirnoff (*(m->m_ext.ext_free))(m, m->m_ext.ext_arg1, 6905e4bc63bSGleb Smirnoff m->m_ext.ext_arg2); 6915e4bc63bSGleb Smirnoff break; 6925e4bc63bSGleb Smirnoff default: 6935e4bc63bSGleb Smirnoff KASSERT(m->m_ext.ext_type == 0, 6945e4bc63bSGleb Smirnoff ("%s: unknown ext_type", __func__)); 6955e4bc63bSGleb Smirnoff } 6965e4bc63bSGleb Smirnoff } 6975e4bc63bSGleb Smirnoff 69856a5f52eSGleb Smirnoff if (freembuf && m != mref) 6995e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 7005e4bc63bSGleb Smirnoff } 7015e4bc63bSGleb Smirnoff 7025e4bc63bSGleb Smirnoff /* 7035e4bc63bSGleb Smirnoff * Official mbuf(9) allocation KPI for stack and drivers: 7045e4bc63bSGleb Smirnoff * 7055e4bc63bSGleb Smirnoff * m_get() - a single mbuf without any attachments, sys/mbuf.h. 7065e4bc63bSGleb Smirnoff * m_gethdr() - a single mbuf initialized as M_PKTHDR, sys/mbuf.h. 7075e4bc63bSGleb Smirnoff * m_getcl() - an mbuf + 2k cluster, sys/mbuf.h. 7085e4bc63bSGleb Smirnoff * m_clget() - attach cluster to already allocated mbuf. 7095e4bc63bSGleb Smirnoff * m_cljget() - attach jumbo cluster to already allocated mbuf. 7105e4bc63bSGleb Smirnoff * m_get2() - allocate minimum mbuf that would fit size argument. 7115e4bc63bSGleb Smirnoff * m_getm2() - allocate a chain of mbufs/clusters. 7125e4bc63bSGleb Smirnoff * m_extadd() - attach external cluster to mbuf. 7135e4bc63bSGleb Smirnoff * 7145e4bc63bSGleb Smirnoff * m_free() - free single mbuf with its tags and ext, sys/mbuf.h. 7155e4bc63bSGleb Smirnoff * m_freem() - free chain of mbufs. 7165e4bc63bSGleb Smirnoff */ 7175e4bc63bSGleb Smirnoff 7185e4bc63bSGleb Smirnoff int 7195e4bc63bSGleb Smirnoff m_clget(struct mbuf *m, int how) 7205e4bc63bSGleb Smirnoff { 7215e4bc63bSGleb Smirnoff 7225e4bc63bSGleb Smirnoff KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", 7235e4bc63bSGleb Smirnoff __func__, m)); 7245e4bc63bSGleb Smirnoff m->m_ext.ext_buf = (char *)NULL; 7255e4bc63bSGleb Smirnoff uma_zalloc_arg(zone_clust, m, how); 7265e4bc63bSGleb Smirnoff /* 7275e4bc63bSGleb Smirnoff * On a cluster allocation failure, drain the packet zone and retry, 7285e4bc63bSGleb Smirnoff * we might be able to loosen a few clusters up on the drain. 7295e4bc63bSGleb Smirnoff */ 7305e4bc63bSGleb Smirnoff if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { 7315e4bc63bSGleb Smirnoff zone_drain(zone_pack); 7325e4bc63bSGleb Smirnoff uma_zalloc_arg(zone_clust, m, how); 7335e4bc63bSGleb Smirnoff } 7345e4bc63bSGleb Smirnoff return (m->m_flags & M_EXT); 7355e4bc63bSGleb Smirnoff } 7365e4bc63bSGleb Smirnoff 7375e4bc63bSGleb Smirnoff /* 7385e4bc63bSGleb Smirnoff * m_cljget() is different from m_clget() as it can allocate clusters without 7395e4bc63bSGleb Smirnoff * attaching them to an mbuf. In that case the return value is the pointer 7405e4bc63bSGleb Smirnoff * to the cluster of the requested size. If an mbuf was specified, it gets 7415e4bc63bSGleb Smirnoff * the cluster attached to it and the return value can be safely ignored. 7425e4bc63bSGleb Smirnoff * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. 7435e4bc63bSGleb Smirnoff */ 7445e4bc63bSGleb Smirnoff void * 7455e4bc63bSGleb Smirnoff m_cljget(struct mbuf *m, int how, int size) 7465e4bc63bSGleb Smirnoff { 7475e4bc63bSGleb Smirnoff uma_zone_t zone; 7485e4bc63bSGleb Smirnoff 7495e4bc63bSGleb Smirnoff if (m != NULL) { 7505e4bc63bSGleb Smirnoff KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", 7515e4bc63bSGleb Smirnoff __func__, m)); 7525e4bc63bSGleb Smirnoff m->m_ext.ext_buf = NULL; 7535e4bc63bSGleb Smirnoff } 7545e4bc63bSGleb Smirnoff 7555e4bc63bSGleb Smirnoff zone = m_getzone(size); 7565e4bc63bSGleb Smirnoff return (uma_zalloc_arg(zone, m, how)); 7575e4bc63bSGleb Smirnoff } 7585e4bc63bSGleb Smirnoff 7595e4bc63bSGleb Smirnoff /* 7605e4bc63bSGleb Smirnoff * m_get2() allocates minimum mbuf that would fit "size" argument. 7615e4bc63bSGleb Smirnoff */ 7625e4bc63bSGleb Smirnoff struct mbuf * 7635e4bc63bSGleb Smirnoff m_get2(int size, int how, short type, int flags) 7645e4bc63bSGleb Smirnoff { 7655e4bc63bSGleb Smirnoff struct mb_args args; 7665e4bc63bSGleb Smirnoff struct mbuf *m, *n; 7675e4bc63bSGleb Smirnoff 7685e4bc63bSGleb Smirnoff args.flags = flags; 7695e4bc63bSGleb Smirnoff args.type = type; 7705e4bc63bSGleb Smirnoff 7715e4bc63bSGleb Smirnoff if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) 7725e4bc63bSGleb Smirnoff return (uma_zalloc_arg(zone_mbuf, &args, how)); 7735e4bc63bSGleb Smirnoff if (size <= MCLBYTES) 7745e4bc63bSGleb Smirnoff return (uma_zalloc_arg(zone_pack, &args, how)); 7755e4bc63bSGleb Smirnoff 7765e4bc63bSGleb Smirnoff if (size > MJUMPAGESIZE) 7775e4bc63bSGleb Smirnoff return (NULL); 7785e4bc63bSGleb Smirnoff 7795e4bc63bSGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 7805e4bc63bSGleb Smirnoff if (m == NULL) 7815e4bc63bSGleb Smirnoff return (NULL); 7825e4bc63bSGleb Smirnoff 7835e4bc63bSGleb Smirnoff n = uma_zalloc_arg(zone_jumbop, m, how); 7845e4bc63bSGleb Smirnoff if (n == NULL) { 7855e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 7865e4bc63bSGleb Smirnoff return (NULL); 7875e4bc63bSGleb Smirnoff } 7885e4bc63bSGleb Smirnoff 7895e4bc63bSGleb Smirnoff return (m); 7905e4bc63bSGleb Smirnoff } 7915e4bc63bSGleb Smirnoff 7925e4bc63bSGleb Smirnoff /* 7935e4bc63bSGleb Smirnoff * m_getjcl() returns an mbuf with a cluster of the specified size attached. 7945e4bc63bSGleb Smirnoff * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. 7955e4bc63bSGleb Smirnoff */ 7965e4bc63bSGleb Smirnoff struct mbuf * 7975e4bc63bSGleb Smirnoff m_getjcl(int how, short type, int flags, int size) 7985e4bc63bSGleb Smirnoff { 7995e4bc63bSGleb Smirnoff struct mb_args args; 8005e4bc63bSGleb Smirnoff struct mbuf *m, *n; 8015e4bc63bSGleb Smirnoff uma_zone_t zone; 8025e4bc63bSGleb Smirnoff 8035e4bc63bSGleb Smirnoff if (size == MCLBYTES) 8045e4bc63bSGleb Smirnoff return m_getcl(how, type, flags); 8055e4bc63bSGleb Smirnoff 8065e4bc63bSGleb Smirnoff args.flags = flags; 8075e4bc63bSGleb Smirnoff args.type = type; 8085e4bc63bSGleb Smirnoff 8095e4bc63bSGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 8105e4bc63bSGleb Smirnoff if (m == NULL) 8115e4bc63bSGleb Smirnoff return (NULL); 8125e4bc63bSGleb Smirnoff 8135e4bc63bSGleb Smirnoff zone = m_getzone(size); 8145e4bc63bSGleb Smirnoff n = uma_zalloc_arg(zone, m, how); 8155e4bc63bSGleb Smirnoff if (n == NULL) { 8165e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 8175e4bc63bSGleb Smirnoff return (NULL); 8185e4bc63bSGleb Smirnoff } 8195e4bc63bSGleb Smirnoff return (m); 8205e4bc63bSGleb Smirnoff } 8215e4bc63bSGleb Smirnoff 8225e4bc63bSGleb Smirnoff /* 8235e4bc63bSGleb Smirnoff * Allocate a given length worth of mbufs and/or clusters (whatever fits 8245e4bc63bSGleb Smirnoff * best) and return a pointer to the top of the allocated chain. If an 8255e4bc63bSGleb Smirnoff * existing mbuf chain is provided, then we will append the new chain 8265e4bc63bSGleb Smirnoff * to the existing one but still return the top of the newly allocated 8275e4bc63bSGleb Smirnoff * chain. 8285e4bc63bSGleb Smirnoff */ 8295e4bc63bSGleb Smirnoff struct mbuf * 8305e4bc63bSGleb Smirnoff m_getm2(struct mbuf *m, int len, int how, short type, int flags) 8315e4bc63bSGleb Smirnoff { 8325e4bc63bSGleb Smirnoff struct mbuf *mb, *nm = NULL, *mtail = NULL; 8335e4bc63bSGleb Smirnoff 8345e4bc63bSGleb Smirnoff KASSERT(len >= 0, ("%s: len is < 0", __func__)); 8355e4bc63bSGleb Smirnoff 8365e4bc63bSGleb Smirnoff /* Validate flags. */ 8375e4bc63bSGleb Smirnoff flags &= (M_PKTHDR | M_EOR); 8385e4bc63bSGleb Smirnoff 8395e4bc63bSGleb Smirnoff /* Packet header mbuf must be first in chain. */ 8405e4bc63bSGleb Smirnoff if ((flags & M_PKTHDR) && m != NULL) 8415e4bc63bSGleb Smirnoff flags &= ~M_PKTHDR; 8425e4bc63bSGleb Smirnoff 8435e4bc63bSGleb Smirnoff /* Loop and append maximum sized mbufs to the chain tail. */ 8445e4bc63bSGleb Smirnoff while (len > 0) { 8455e4bc63bSGleb Smirnoff if (len > MCLBYTES) 8465e4bc63bSGleb Smirnoff mb = m_getjcl(how, type, (flags & M_PKTHDR), 8475e4bc63bSGleb Smirnoff MJUMPAGESIZE); 8485e4bc63bSGleb Smirnoff else if (len >= MINCLSIZE) 8495e4bc63bSGleb Smirnoff mb = m_getcl(how, type, (flags & M_PKTHDR)); 8505e4bc63bSGleb Smirnoff else if (flags & M_PKTHDR) 8515e4bc63bSGleb Smirnoff mb = m_gethdr(how, type); 8525e4bc63bSGleb Smirnoff else 8535e4bc63bSGleb Smirnoff mb = m_get(how, type); 8545e4bc63bSGleb Smirnoff 8555e4bc63bSGleb Smirnoff /* Fail the whole operation if one mbuf can't be allocated. */ 8565e4bc63bSGleb Smirnoff if (mb == NULL) { 8575e4bc63bSGleb Smirnoff if (nm != NULL) 8585e4bc63bSGleb Smirnoff m_freem(nm); 8595e4bc63bSGleb Smirnoff return (NULL); 8605e4bc63bSGleb Smirnoff } 8615e4bc63bSGleb Smirnoff 8625e4bc63bSGleb Smirnoff /* Book keeping. */ 8635e4bc63bSGleb Smirnoff len -= M_SIZE(mb); 8645e4bc63bSGleb Smirnoff if (mtail != NULL) 8655e4bc63bSGleb Smirnoff mtail->m_next = mb; 8665e4bc63bSGleb Smirnoff else 8675e4bc63bSGleb Smirnoff nm = mb; 8685e4bc63bSGleb Smirnoff mtail = mb; 8695e4bc63bSGleb Smirnoff flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ 8705e4bc63bSGleb Smirnoff } 8715e4bc63bSGleb Smirnoff if (flags & M_EOR) 8725e4bc63bSGleb Smirnoff mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ 8735e4bc63bSGleb Smirnoff 8745e4bc63bSGleb Smirnoff /* If mbuf was supplied, append new chain to the end of it. */ 8755e4bc63bSGleb Smirnoff if (m != NULL) { 8765e4bc63bSGleb Smirnoff for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) 8775e4bc63bSGleb Smirnoff ; 8785e4bc63bSGleb Smirnoff mtail->m_next = nm; 8795e4bc63bSGleb Smirnoff mtail->m_flags &= ~M_EOR; 8805e4bc63bSGleb Smirnoff } else 8815e4bc63bSGleb Smirnoff m = nm; 8825e4bc63bSGleb Smirnoff 8835e4bc63bSGleb Smirnoff return (m); 8845e4bc63bSGleb Smirnoff } 8855e4bc63bSGleb Smirnoff 8865e4bc63bSGleb Smirnoff /*- 8875e4bc63bSGleb Smirnoff * Configure a provided mbuf to refer to the provided external storage 88856a5f52eSGleb Smirnoff * buffer and setup a reference count for said buffer. 8895e4bc63bSGleb Smirnoff * 8905e4bc63bSGleb Smirnoff * Arguments: 8915e4bc63bSGleb Smirnoff * mb The existing mbuf to which to attach the provided buffer. 8925e4bc63bSGleb Smirnoff * buf The address of the provided external storage buffer. 8935e4bc63bSGleb Smirnoff * size The size of the provided buffer. 8945e4bc63bSGleb Smirnoff * freef A pointer to a routine that is responsible for freeing the 8955e4bc63bSGleb Smirnoff * provided external storage buffer. 8965e4bc63bSGleb Smirnoff * args A pointer to an argument structure (of any type) to be passed 8975e4bc63bSGleb Smirnoff * to the provided freef routine (may be NULL). 8985e4bc63bSGleb Smirnoff * flags Any other flags to be passed to the provided mbuf. 8995e4bc63bSGleb Smirnoff * type The type that the external storage buffer should be 9005e4bc63bSGleb Smirnoff * labeled with. 9015e4bc63bSGleb Smirnoff * 9025e4bc63bSGleb Smirnoff * Returns: 9035e4bc63bSGleb Smirnoff * Nothing. 9045e4bc63bSGleb Smirnoff */ 90556a5f52eSGleb Smirnoff void 9065e4bc63bSGleb Smirnoff m_extadd(struct mbuf *mb, caddr_t buf, u_int size, 9075e4bc63bSGleb Smirnoff void (*freef)(struct mbuf *, void *, void *), void *arg1, void *arg2, 90856a5f52eSGleb Smirnoff int flags, int type) 9095e4bc63bSGleb Smirnoff { 91056a5f52eSGleb Smirnoff 9115e4bc63bSGleb Smirnoff KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); 9125e4bc63bSGleb Smirnoff 9135e4bc63bSGleb Smirnoff mb->m_flags |= (M_EXT | flags); 9145e4bc63bSGleb Smirnoff mb->m_ext.ext_buf = buf; 9155e4bc63bSGleb Smirnoff mb->m_data = mb->m_ext.ext_buf; 9165e4bc63bSGleb Smirnoff mb->m_ext.ext_size = size; 9175e4bc63bSGleb Smirnoff mb->m_ext.ext_free = freef; 9185e4bc63bSGleb Smirnoff mb->m_ext.ext_arg1 = arg1; 9195e4bc63bSGleb Smirnoff mb->m_ext.ext_arg2 = arg2; 9205e4bc63bSGleb Smirnoff mb->m_ext.ext_type = type; 9215e4bc63bSGleb Smirnoff 92256a5f52eSGleb Smirnoff if (type != EXT_EXTREF) { 92356a5f52eSGleb Smirnoff mb->m_ext.ext_count = 1; 92456a5f52eSGleb Smirnoff mb->m_ext.ext_flags = EXT_FLAG_EMBREF; 92556a5f52eSGleb Smirnoff } else 92656a5f52eSGleb Smirnoff mb->m_ext.ext_flags = 0; 9275e4bc63bSGleb Smirnoff } 9285e4bc63bSGleb Smirnoff 9295e4bc63bSGleb Smirnoff /* 9305e4bc63bSGleb Smirnoff * Free an entire chain of mbufs and associated external buffers, if 9315e4bc63bSGleb Smirnoff * applicable. 9325e4bc63bSGleb Smirnoff */ 9335e4bc63bSGleb Smirnoff void 9345e4bc63bSGleb Smirnoff m_freem(struct mbuf *mb) 9355e4bc63bSGleb Smirnoff { 9365e4bc63bSGleb Smirnoff 9375e4bc63bSGleb Smirnoff while (mb != NULL) 9385e4bc63bSGleb Smirnoff mb = m_free(mb); 9395e4bc63bSGleb Smirnoff } 940