1099a0e58SBosko Milekic /*- 28a36da99SPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 38a36da99SPedro F. Giffuni * 48076cb52SBosko Milekic * Copyright (c) 2004, 2005, 58076cb52SBosko Milekic * Bosko Milekic <bmilekic@FreeBSD.org>. All rights reserved. 6099a0e58SBosko Milekic * 7099a0e58SBosko Milekic * Redistribution and use in source and binary forms, with or without 8099a0e58SBosko Milekic * modification, are permitted provided that the following conditions 9099a0e58SBosko Milekic * are met: 10099a0e58SBosko Milekic * 1. Redistributions of source code must retain the above copyright 11099a0e58SBosko Milekic * notice unmodified, this list of conditions and the following 12099a0e58SBosko Milekic * disclaimer. 13099a0e58SBosko Milekic * 2. Redistributions in binary form must reproduce the above copyright 14099a0e58SBosko Milekic * notice, this list of conditions and the following disclaimer in the 15099a0e58SBosko Milekic * documentation and/or other materials provided with the distribution. 16099a0e58SBosko Milekic * 17099a0e58SBosko Milekic * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18099a0e58SBosko Milekic * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19099a0e58SBosko Milekic * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20099a0e58SBosko Milekic * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21099a0e58SBosko Milekic * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22099a0e58SBosko Milekic * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23099a0e58SBosko Milekic * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24099a0e58SBosko Milekic * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25099a0e58SBosko Milekic * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26099a0e58SBosko Milekic * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27099a0e58SBosko Milekic * SUCH DAMAGE. 28099a0e58SBosko Milekic */ 29099a0e58SBosko Milekic 30099a0e58SBosko Milekic #include <sys/cdefs.h> 31099a0e58SBosko Milekic __FBSDID("$FreeBSD$"); 32099a0e58SBosko Milekic 33099a0e58SBosko Milekic #include "opt_param.h" 34099a0e58SBosko Milekic 35099a0e58SBosko Milekic #include <sys/param.h> 363937ee75SConrad Meyer #include <sys/conf.h> 379978bd99SMark Johnston #include <sys/domainset.h> 38099a0e58SBosko Milekic #include <sys/malloc.h> 39099a0e58SBosko Milekic #include <sys/systm.h> 40099a0e58SBosko Milekic #include <sys/mbuf.h> 41099a0e58SBosko Milekic #include <sys/domain.h> 42099a0e58SBosko Milekic #include <sys/eventhandler.h> 43099a0e58SBosko Milekic #include <sys/kernel.h> 445475ca5aSMark Johnston #include <sys/limits.h> 4554503a13SJonathan T. Looney #include <sys/lock.h> 4654503a13SJonathan T. Looney #include <sys/mutex.h> 47099a0e58SBosko Milekic #include <sys/protosw.h> 48099a0e58SBosko Milekic #include <sys/smp.h> 49*fb3bc596SJohn Baldwin #include <sys/socket.h> 50099a0e58SBosko Milekic #include <sys/sysctl.h> 51099a0e58SBosko Milekic 52*fb3bc596SJohn Baldwin #include <net/if.h> 53*fb3bc596SJohn Baldwin #include <net/if_var.h> 54*fb3bc596SJohn Baldwin 55099a0e58SBosko Milekic #include <vm/vm.h> 56c45c0034SAlan Cox #include <vm/vm_extern.h> 57c45c0034SAlan Cox #include <vm/vm_kern.h> 58099a0e58SBosko Milekic #include <vm/vm_page.h> 5937140716SAndre Oppermann #include <vm/vm_map.h> 60099a0e58SBosko Milekic #include <vm/uma.h> 61121f0509SMike Silbersack #include <vm/uma_dbg.h> 62099a0e58SBosko Milekic 63099a0e58SBosko Milekic /* 64099a0e58SBosko Milekic * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA 65099a0e58SBosko Milekic * Zones. 66099a0e58SBosko Milekic * 67099a0e58SBosko Milekic * Mbuf Clusters (2K, contiguous) are allocated from the Cluster 68099a0e58SBosko Milekic * Zone. The Zone can be capped at kern.ipc.nmbclusters, if the 69099a0e58SBosko Milekic * administrator so desires. 70099a0e58SBosko Milekic * 71099a0e58SBosko Milekic * Mbufs are allocated from a UMA Master Zone called the Mbuf 72099a0e58SBosko Milekic * Zone. 73099a0e58SBosko Milekic * 74099a0e58SBosko Milekic * Additionally, FreeBSD provides a Packet Zone, which it 75099a0e58SBosko Milekic * configures as a Secondary Zone to the Mbuf Master Zone, 76099a0e58SBosko Milekic * thus sharing backend Slab kegs with the Mbuf Master Zone. 77099a0e58SBosko Milekic * 78099a0e58SBosko Milekic * Thus common-case allocations and locking are simplified: 79099a0e58SBosko Milekic * 80099a0e58SBosko Milekic * m_clget() m_getcl() 81099a0e58SBosko Milekic * | | 82099a0e58SBosko Milekic * | .------------>[(Packet Cache)] m_get(), m_gethdr() 83099a0e58SBosko Milekic * | | [ Packet ] | 84099a0e58SBosko Milekic * [(Cluster Cache)] [ Secondary ] [ (Mbuf Cache) ] 85099a0e58SBosko Milekic * [ Cluster Zone ] [ Zone ] [ Mbuf Master Zone ] 86099a0e58SBosko Milekic * | \________ | 87099a0e58SBosko Milekic * [ Cluster Keg ] \ / 88099a0e58SBosko Milekic * | [ Mbuf Keg ] 89099a0e58SBosko Milekic * [ Cluster Slabs ] | 90099a0e58SBosko Milekic * | [ Mbuf Slabs ] 91099a0e58SBosko Milekic * \____________(VM)_________________/ 9256a4e45aSAndre Oppermann * 9356a4e45aSAndre Oppermann * 94fcf90618SGleb Smirnoff * Whenever an object is allocated with uma_zalloc() out of 9556a4e45aSAndre Oppermann * one of the Zones its _ctor_ function is executed. The same 96fcf90618SGleb Smirnoff * for any deallocation through uma_zfree() the _dtor_ function 9756a4e45aSAndre Oppermann * is executed. 9856a4e45aSAndre Oppermann * 9956a4e45aSAndre Oppermann * Caches are per-CPU and are filled from the Master Zone. 10056a4e45aSAndre Oppermann * 101fcf90618SGleb Smirnoff * Whenever an object is allocated from the underlying global 10256a4e45aSAndre Oppermann * memory pool it gets pre-initialized with the _zinit_ functions. 103e3043798SPedro F. Giffuni * When the Keg's are overfull objects get decommissioned with 10456a4e45aSAndre Oppermann * _zfini_ functions and free'd back to the global memory pool. 10556a4e45aSAndre Oppermann * 106099a0e58SBosko Milekic */ 107099a0e58SBosko Milekic 108ead46972SAndre Oppermann int nmbufs; /* limits number of mbufs */ 10956a4e45aSAndre Oppermann int nmbclusters; /* limits number of mbuf clusters */ 110ec63cb90SAndre Oppermann int nmbjumbop; /* limits number of page size jumbo clusters */ 11156a4e45aSAndre Oppermann int nmbjumbo9; /* limits number of 9k jumbo clusters */ 11256a4e45aSAndre Oppermann int nmbjumbo16; /* limits number of 16k jumbo clusters */ 113099a0e58SBosko Milekic 114e0c00addSAndre Oppermann static quad_t maxmbufmem; /* overall real memory limit for all mbufs */ 115e0c00addSAndre Oppermann 116af3b2549SHans Petter Selasky SYSCTL_QUAD(_kern_ipc, OID_AUTO, maxmbufmem, CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &maxmbufmem, 0, 117b6f49c23SHiren Panchasara "Maximum real memory allocatable to various mbuf types"); 118e0c00addSAndre Oppermann 119*fb3bc596SJohn Baldwin static counter_u64_t snd_tag_count; 120*fb3bc596SJohn Baldwin SYSCTL_COUNTER_U64(_kern_ipc, OID_AUTO, num_snd_tags, CTLFLAG_RW, 121*fb3bc596SJohn Baldwin &snd_tag_count, "# of active mbuf send tags"); 122*fb3bc596SJohn Baldwin 12362938659SBjoern A. Zeeb /* 12437140716SAndre Oppermann * tunable_mbinit() has to be run before any mbuf allocations are done. 12562938659SBjoern A. Zeeb */ 126099a0e58SBosko Milekic static void 127099a0e58SBosko Milekic tunable_mbinit(void *dummy) 128099a0e58SBosko Milekic { 129e0c00addSAndre Oppermann quad_t realmem; 13037140716SAndre Oppermann 13137140716SAndre Oppermann /* 13237140716SAndre Oppermann * The default limit for all mbuf related memory is 1/2 of all 13337140716SAndre Oppermann * available kernel memory (physical or kmem). 13437140716SAndre Oppermann * At most it can be 3/4 of available kernel memory. 13537140716SAndre Oppermann */ 1365df87b21SJeff Roberson realmem = qmin((quad_t)physmem * PAGE_SIZE, vm_kmem_size); 13737140716SAndre Oppermann maxmbufmem = realmem / 2; 138e0c00addSAndre Oppermann TUNABLE_QUAD_FETCH("kern.ipc.maxmbufmem", &maxmbufmem); 13937140716SAndre Oppermann if (maxmbufmem > realmem / 4 * 3) 14037140716SAndre Oppermann maxmbufmem = realmem / 4 * 3; 141099a0e58SBosko Milekic 142812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbclusters", &nmbclusters); 143416a434cSAndre Oppermann if (nmbclusters == 0) 144416a434cSAndre Oppermann nmbclusters = maxmbufmem / MCLBYTES / 4; 145812302c3SNavdeep Parhar 146812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbop", &nmbjumbop); 147812302c3SNavdeep Parhar if (nmbjumbop == 0) 148416a434cSAndre Oppermann nmbjumbop = maxmbufmem / MJUMPAGESIZE / 4; 149812302c3SNavdeep Parhar 150812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbo9", &nmbjumbo9); 151812302c3SNavdeep Parhar if (nmbjumbo9 == 0) 152416a434cSAndre Oppermann nmbjumbo9 = maxmbufmem / MJUM9BYTES / 6; 153812302c3SNavdeep Parhar 154812302c3SNavdeep Parhar TUNABLE_INT_FETCH("kern.ipc.nmbjumbo16", &nmbjumbo16); 155812302c3SNavdeep Parhar if (nmbjumbo16 == 0) 156416a434cSAndre Oppermann nmbjumbo16 = maxmbufmem / MJUM16BYTES / 6; 157416a434cSAndre Oppermann 158416a434cSAndre Oppermann /* 159416a434cSAndre Oppermann * We need at least as many mbufs as we have clusters of 160416a434cSAndre Oppermann * the various types added together. 161416a434cSAndre Oppermann */ 162416a434cSAndre Oppermann TUNABLE_INT_FETCH("kern.ipc.nmbufs", &nmbufs); 163416a434cSAndre Oppermann if (nmbufs < nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) 164416a434cSAndre Oppermann nmbufs = lmax(maxmbufmem / MSIZE / 5, 165416a434cSAndre Oppermann nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16); 166099a0e58SBosko Milekic } 16737140716SAndre Oppermann SYSINIT(tunable_mbinit, SI_SUB_KMEM, SI_ORDER_MIDDLE, tunable_mbinit, NULL); 168099a0e58SBosko Milekic 1694f590175SPaul Saab static int 1704f590175SPaul Saab sysctl_nmbclusters(SYSCTL_HANDLER_ARGS) 1714f590175SPaul Saab { 1724f590175SPaul Saab int error, newnmbclusters; 1734f590175SPaul Saab 1744f590175SPaul Saab newnmbclusters = nmbclusters; 175041b706bSDavid Malone error = sysctl_handle_int(oidp, &newnmbclusters, 0, req); 176d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbclusters != nmbclusters) { 177ead46972SAndre Oppermann if (newnmbclusters > nmbclusters && 178ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 1794f590175SPaul Saab nmbclusters = newnmbclusters; 180bc4a1b8cSAndre Oppermann nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 1814f590175SPaul Saab EVENTHANDLER_INVOKE(nmbclusters_change); 1824f590175SPaul Saab } else 1834f590175SPaul Saab error = EINVAL; 1844f590175SPaul Saab } 1854f590175SPaul Saab return (error); 1864f590175SPaul Saab } 1874f590175SPaul Saab SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbclusters, CTLTYPE_INT|CTLFLAG_RW, 1884f590175SPaul Saab &nmbclusters, 0, sysctl_nmbclusters, "IU", 189099a0e58SBosko Milekic "Maximum number of mbuf clusters allowed"); 190cf70a46bSRandall Stewart 191cf70a46bSRandall Stewart static int 192cf70a46bSRandall Stewart sysctl_nmbjumbop(SYSCTL_HANDLER_ARGS) 193cf70a46bSRandall Stewart { 194cf70a46bSRandall Stewart int error, newnmbjumbop; 195cf70a46bSRandall Stewart 196cf70a46bSRandall Stewart newnmbjumbop = nmbjumbop; 197cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbop, 0, req); 198d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbop != nmbjumbop) { 199ead46972SAndre Oppermann if (newnmbjumbop > nmbjumbop && 200ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 201cf70a46bSRandall Stewart nmbjumbop = newnmbjumbop; 202bc4a1b8cSAndre Oppermann nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 203cf70a46bSRandall Stewart } else 204cf70a46bSRandall Stewart error = EINVAL; 205cf70a46bSRandall Stewart } 206cf70a46bSRandall Stewart return (error); 207cf70a46bSRandall Stewart } 208cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbop, CTLTYPE_INT|CTLFLAG_RW, 209cf70a46bSRandall Stewart &nmbjumbop, 0, sysctl_nmbjumbop, "IU", 210ec63cb90SAndre Oppermann "Maximum number of mbuf page size jumbo clusters allowed"); 211cf70a46bSRandall Stewart 212cf70a46bSRandall Stewart static int 213cf70a46bSRandall Stewart sysctl_nmbjumbo9(SYSCTL_HANDLER_ARGS) 214cf70a46bSRandall Stewart { 215cf70a46bSRandall Stewart int error, newnmbjumbo9; 216cf70a46bSRandall Stewart 217cf70a46bSRandall Stewart newnmbjumbo9 = nmbjumbo9; 218cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbo9, 0, req); 219d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbo9 != nmbjumbo9) { 220ead46972SAndre Oppermann if (newnmbjumbo9 > nmbjumbo9 && 221ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 222cf70a46bSRandall Stewart nmbjumbo9 = newnmbjumbo9; 223bc4a1b8cSAndre Oppermann nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 224cf70a46bSRandall Stewart } else 225cf70a46bSRandall Stewart error = EINVAL; 226cf70a46bSRandall Stewart } 227cf70a46bSRandall Stewart return (error); 228cf70a46bSRandall Stewart } 229cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo9, CTLTYPE_INT|CTLFLAG_RW, 230cf70a46bSRandall Stewart &nmbjumbo9, 0, sysctl_nmbjumbo9, "IU", 23156a4e45aSAndre Oppermann "Maximum number of mbuf 9k jumbo clusters allowed"); 232cf70a46bSRandall Stewart 233cf70a46bSRandall Stewart static int 234cf70a46bSRandall Stewart sysctl_nmbjumbo16(SYSCTL_HANDLER_ARGS) 235cf70a46bSRandall Stewart { 236cf70a46bSRandall Stewart int error, newnmbjumbo16; 237cf70a46bSRandall Stewart 238cf70a46bSRandall Stewart newnmbjumbo16 = nmbjumbo16; 239cf70a46bSRandall Stewart error = sysctl_handle_int(oidp, &newnmbjumbo16, 0, req); 240d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbjumbo16 != nmbjumbo16) { 241ead46972SAndre Oppermann if (newnmbjumbo16 > nmbjumbo16 && 242ead46972SAndre Oppermann nmbufs >= nmbclusters + nmbjumbop + nmbjumbo9 + nmbjumbo16) { 243cf70a46bSRandall Stewart nmbjumbo16 = newnmbjumbo16; 244bc4a1b8cSAndre Oppermann nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 245cf70a46bSRandall Stewart } else 246cf70a46bSRandall Stewart error = EINVAL; 247cf70a46bSRandall Stewart } 248cf70a46bSRandall Stewart return (error); 249cf70a46bSRandall Stewart } 250cf70a46bSRandall Stewart SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbjumbo16, CTLTYPE_INT|CTLFLAG_RW, 251cf70a46bSRandall Stewart &nmbjumbo16, 0, sysctl_nmbjumbo16, "IU", 25256a4e45aSAndre Oppermann "Maximum number of mbuf 16k jumbo clusters allowed"); 253cf70a46bSRandall Stewart 254ead46972SAndre Oppermann static int 255ead46972SAndre Oppermann sysctl_nmbufs(SYSCTL_HANDLER_ARGS) 256ead46972SAndre Oppermann { 257ead46972SAndre Oppermann int error, newnmbufs; 258ead46972SAndre Oppermann 259ead46972SAndre Oppermann newnmbufs = nmbufs; 260ead46972SAndre Oppermann error = sysctl_handle_int(oidp, &newnmbufs, 0, req); 261d251e700SJohn Baldwin if (error == 0 && req->newptr && newnmbufs != nmbufs) { 262ead46972SAndre Oppermann if (newnmbufs > nmbufs) { 263ead46972SAndre Oppermann nmbufs = newnmbufs; 264bc4a1b8cSAndre Oppermann nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 265ead46972SAndre Oppermann EVENTHANDLER_INVOKE(nmbufs_change); 266ead46972SAndre Oppermann } else 267ead46972SAndre Oppermann error = EINVAL; 268ead46972SAndre Oppermann } 269ead46972SAndre Oppermann return (error); 270ead46972SAndre Oppermann } 271e0c00addSAndre Oppermann SYSCTL_PROC(_kern_ipc, OID_AUTO, nmbufs, CTLTYPE_INT|CTLFLAG_RW, 272ead46972SAndre Oppermann &nmbufs, 0, sysctl_nmbufs, "IU", 273ead46972SAndre Oppermann "Maximum number of mbufs allowed"); 274cf70a46bSRandall Stewart 275099a0e58SBosko Milekic /* 276099a0e58SBosko Milekic * Zones from which we allocate. 277099a0e58SBosko Milekic */ 278099a0e58SBosko Milekic uma_zone_t zone_mbuf; 279099a0e58SBosko Milekic uma_zone_t zone_clust; 280099a0e58SBosko Milekic uma_zone_t zone_pack; 281ec63cb90SAndre Oppermann uma_zone_t zone_jumbop; 28256a4e45aSAndre Oppermann uma_zone_t zone_jumbo9; 28356a4e45aSAndre Oppermann uma_zone_t zone_jumbo16; 284099a0e58SBosko Milekic 285099a0e58SBosko Milekic /* 286099a0e58SBosko Milekic * Local prototypes. 287099a0e58SBosko Milekic */ 288b23f72e9SBrian Feldman static int mb_ctor_mbuf(void *, int, void *, int); 289b23f72e9SBrian Feldman static int mb_ctor_clust(void *, int, void *, int); 290b23f72e9SBrian Feldman static int mb_ctor_pack(void *, int, void *, int); 291099a0e58SBosko Milekic static void mb_dtor_mbuf(void *, int, void *); 29256a4e45aSAndre Oppermann static void mb_dtor_pack(void *, int, void *); 29356a4e45aSAndre Oppermann static int mb_zinit_pack(void *, int, int); 29456a4e45aSAndre Oppermann static void mb_zfini_pack(void *, int); 295e60b2fcbSGleb Smirnoff static void mb_reclaim(uma_zone_t, int); 296ab3185d1SJeff Roberson static void *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 297099a0e58SBosko Milekic 29837140716SAndre Oppermann /* Ensure that MSIZE is a power of 2. */ 299a04946cfSBrian Somers CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE); 300a04946cfSBrian Somers 301099a0e58SBosko Milekic /* 302099a0e58SBosko Milekic * Initialize FreeBSD Network buffer allocation. 303099a0e58SBosko Milekic */ 304099a0e58SBosko Milekic static void 305099a0e58SBosko Milekic mbuf_init(void *dummy) 306099a0e58SBosko Milekic { 307099a0e58SBosko Milekic 308099a0e58SBosko Milekic /* 309099a0e58SBosko Milekic * Configure UMA zones for Mbufs, Clusters, and Packets. 310099a0e58SBosko Milekic */ 31156a4e45aSAndre Oppermann zone_mbuf = uma_zcreate(MBUF_MEM_NAME, MSIZE, 31256a4e45aSAndre Oppermann mb_ctor_mbuf, mb_dtor_mbuf, 313121f0509SMike Silbersack #ifdef INVARIANTS 31456a4e45aSAndre Oppermann trash_init, trash_fini, 315121f0509SMike Silbersack #else 31656a4e45aSAndre Oppermann NULL, NULL, 317121f0509SMike Silbersack #endif 31856a4e45aSAndre Oppermann MSIZE - 1, UMA_ZONE_MAXBUCKET); 31945fe0bf7SPawel Jakub Dawidek if (nmbufs > 0) 32045fe0bf7SPawel Jakub Dawidek nmbufs = uma_zone_set_max(zone_mbuf, nmbufs); 3216e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached"); 322e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_mbuf, mb_reclaim); 32356a4e45aSAndre Oppermann 32468352adfSRobert Watson zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES, 32556a5f52eSGleb Smirnoff mb_ctor_clust, 326121f0509SMike Silbersack #ifdef INVARIANTS 32756a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 328121f0509SMike Silbersack #else 32956a5f52eSGleb Smirnoff NULL, NULL, NULL, 330121f0509SMike Silbersack #endif 33156a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 33245fe0bf7SPawel Jakub Dawidek if (nmbclusters > 0) 33345fe0bf7SPawel Jakub Dawidek nmbclusters = uma_zone_set_max(zone_clust, nmbclusters); 3346e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached"); 335e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_clust, mb_reclaim); 336099a0e58SBosko Milekic 33756a4e45aSAndre Oppermann zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack, 33856a4e45aSAndre Oppermann mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf); 33956a4e45aSAndre Oppermann 340fcf90618SGleb Smirnoff /* Make jumbo frame zone too. Page size, 9k and 16k. */ 341ec63cb90SAndre Oppermann zone_jumbop = uma_zcreate(MBUF_JUMBOP_MEM_NAME, MJUMPAGESIZE, 34256a5f52eSGleb Smirnoff mb_ctor_clust, 343d5269a63SAndre Oppermann #ifdef INVARIANTS 34456a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 345d5269a63SAndre Oppermann #else 34656a5f52eSGleb Smirnoff NULL, NULL, NULL, 347d5269a63SAndre Oppermann #endif 34856a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 34945fe0bf7SPawel Jakub Dawidek if (nmbjumbop > 0) 35045fe0bf7SPawel Jakub Dawidek nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop); 3516e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached"); 352e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbop, mb_reclaim); 353d5269a63SAndre Oppermann 35456a4e45aSAndre Oppermann zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES, 35556a5f52eSGleb Smirnoff mb_ctor_clust, 35656a4e45aSAndre Oppermann #ifdef INVARIANTS 35756a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 35856a4e45aSAndre Oppermann #else 35956a5f52eSGleb Smirnoff NULL, NULL, NULL, 36056a4e45aSAndre Oppermann #endif 36156a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 362ba63339aSAlan Cox uma_zone_set_allocf(zone_jumbo9, mbuf_jumbo_alloc); 36345fe0bf7SPawel Jakub Dawidek if (nmbjumbo9 > 0) 36445fe0bf7SPawel Jakub Dawidek nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9); 3656e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached"); 366e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbo9, mb_reclaim); 36756a4e45aSAndre Oppermann 36856a4e45aSAndre Oppermann zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES, 36956a5f52eSGleb Smirnoff mb_ctor_clust, 37056a4e45aSAndre Oppermann #ifdef INVARIANTS 37156a5f52eSGleb Smirnoff trash_dtor, trash_init, trash_fini, 37256a4e45aSAndre Oppermann #else 37356a5f52eSGleb Smirnoff NULL, NULL, NULL, 37456a4e45aSAndre Oppermann #endif 37556a5f52eSGleb Smirnoff UMA_ALIGN_PTR, 0); 376ba63339aSAlan Cox uma_zone_set_allocf(zone_jumbo16, mbuf_jumbo_alloc); 37745fe0bf7SPawel Jakub Dawidek if (nmbjumbo16 > 0) 37845fe0bf7SPawel Jakub Dawidek nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16); 3796e0b6746SPawel Jakub Dawidek uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached"); 380e60b2fcbSGleb Smirnoff uma_zone_set_maxaction(zone_jumbo16, mb_reclaim); 38156a4e45aSAndre Oppermann 382099a0e58SBosko Milekic /* 383099a0e58SBosko Milekic * Hook event handler for low-memory situation, used to 384099a0e58SBosko Milekic * drain protocols and push data back to the caches (UMA 385099a0e58SBosko Milekic * later pushes it back to VM). 386099a0e58SBosko Milekic */ 387099a0e58SBosko Milekic EVENTHANDLER_REGISTER(vm_lowmem, mb_reclaim, NULL, 388099a0e58SBosko Milekic EVENTHANDLER_PRI_FIRST); 389*fb3bc596SJohn Baldwin 390*fb3bc596SJohn Baldwin snd_tag_count = counter_u64_alloc(M_WAITOK); 391099a0e58SBosko Milekic } 39237140716SAndre Oppermann SYSINIT(mbuf, SI_SUB_MBUF, SI_ORDER_FIRST, mbuf_init, NULL); 393099a0e58SBosko Milekic 3945475ca5aSMark Johnston #ifdef NETDUMP 3955475ca5aSMark Johnston /* 3965475ca5aSMark Johnston * netdump makes use of a pre-allocated pool of mbufs and clusters. When 3975475ca5aSMark Johnston * netdump is configured, we initialize a set of UMA cache zones which return 3985475ca5aSMark Johnston * items from this pool. At panic-time, the regular UMA zone pointers are 3995475ca5aSMark Johnston * overwritten with those of the cache zones so that drivers may allocate and 4005475ca5aSMark Johnston * free mbufs and clusters without attempting to allocate physical memory. 4015475ca5aSMark Johnston * 4025475ca5aSMark Johnston * We keep mbufs and clusters in a pair of mbuf queues. In particular, for 4035475ca5aSMark Johnston * the purpose of caching clusters, we treat them as mbufs. 4045475ca5aSMark Johnston */ 4055475ca5aSMark Johnston static struct mbufq nd_mbufq = 4065475ca5aSMark Johnston { STAILQ_HEAD_INITIALIZER(nd_mbufq.mq_head), 0, INT_MAX }; 4075475ca5aSMark Johnston static struct mbufq nd_clustq = 4085475ca5aSMark Johnston { STAILQ_HEAD_INITIALIZER(nd_clustq.mq_head), 0, INT_MAX }; 4095475ca5aSMark Johnston 4105475ca5aSMark Johnston static int nd_clsize; 4115475ca5aSMark Johnston static uma_zone_t nd_zone_mbuf; 4125475ca5aSMark Johnston static uma_zone_t nd_zone_clust; 4135475ca5aSMark Johnston static uma_zone_t nd_zone_pack; 4145475ca5aSMark Johnston 4155475ca5aSMark Johnston static int 4165475ca5aSMark Johnston nd_buf_import(void *arg, void **store, int count, int domain __unused, 4175475ca5aSMark Johnston int flags) 4185475ca5aSMark Johnston { 4195475ca5aSMark Johnston struct mbufq *q; 4205475ca5aSMark Johnston struct mbuf *m; 4215475ca5aSMark Johnston int i; 4225475ca5aSMark Johnston 4235475ca5aSMark Johnston q = arg; 4245475ca5aSMark Johnston 4255475ca5aSMark Johnston for (i = 0; i < count; i++) { 4265475ca5aSMark Johnston m = mbufq_dequeue(q); 4275475ca5aSMark Johnston if (m == NULL) 4285475ca5aSMark Johnston break; 4295475ca5aSMark Johnston trash_init(m, q == &nd_mbufq ? MSIZE : nd_clsize, flags); 4305475ca5aSMark Johnston store[i] = m; 4315475ca5aSMark Johnston } 4320d1467b1SConrad Meyer KASSERT((flags & M_WAITOK) == 0 || i == count, 4330d1467b1SConrad Meyer ("%s: ran out of pre-allocated mbufs", __func__)); 4345475ca5aSMark Johnston return (i); 4355475ca5aSMark Johnston } 4365475ca5aSMark Johnston 4375475ca5aSMark Johnston static void 4385475ca5aSMark Johnston nd_buf_release(void *arg, void **store, int count) 4395475ca5aSMark Johnston { 4405475ca5aSMark Johnston struct mbufq *q; 4415475ca5aSMark Johnston struct mbuf *m; 4425475ca5aSMark Johnston int i; 4435475ca5aSMark Johnston 4445475ca5aSMark Johnston q = arg; 4455475ca5aSMark Johnston 4465475ca5aSMark Johnston for (i = 0; i < count; i++) { 4475475ca5aSMark Johnston m = store[i]; 4485475ca5aSMark Johnston (void)mbufq_enqueue(q, m); 4495475ca5aSMark Johnston } 4505475ca5aSMark Johnston } 4515475ca5aSMark Johnston 4525475ca5aSMark Johnston static int 4535475ca5aSMark Johnston nd_pack_import(void *arg __unused, void **store, int count, int domain __unused, 4545475ca5aSMark Johnston int flags __unused) 4555475ca5aSMark Johnston { 4565475ca5aSMark Johnston struct mbuf *m; 4575475ca5aSMark Johnston void *clust; 4585475ca5aSMark Johnston int i; 4595475ca5aSMark Johnston 4605475ca5aSMark Johnston for (i = 0; i < count; i++) { 4615475ca5aSMark Johnston m = m_get(MT_DATA, M_NOWAIT); 4625475ca5aSMark Johnston if (m == NULL) 4635475ca5aSMark Johnston break; 4645475ca5aSMark Johnston clust = uma_zalloc(nd_zone_clust, M_NOWAIT); 4655475ca5aSMark Johnston if (clust == NULL) { 4665475ca5aSMark Johnston m_free(m); 4675475ca5aSMark Johnston break; 4685475ca5aSMark Johnston } 4695475ca5aSMark Johnston mb_ctor_clust(clust, nd_clsize, m, 0); 4705475ca5aSMark Johnston store[i] = m; 4715475ca5aSMark Johnston } 4720d1467b1SConrad Meyer KASSERT((flags & M_WAITOK) == 0 || i == count, 4730d1467b1SConrad Meyer ("%s: ran out of pre-allocated mbufs", __func__)); 4745475ca5aSMark Johnston return (i); 4755475ca5aSMark Johnston } 4765475ca5aSMark Johnston 4775475ca5aSMark Johnston static void 4785475ca5aSMark Johnston nd_pack_release(void *arg __unused, void **store, int count) 4795475ca5aSMark Johnston { 4805475ca5aSMark Johnston struct mbuf *m; 4815475ca5aSMark Johnston void *clust; 4825475ca5aSMark Johnston int i; 4835475ca5aSMark Johnston 4845475ca5aSMark Johnston for (i = 0; i < count; i++) { 4855475ca5aSMark Johnston m = store[i]; 4865475ca5aSMark Johnston clust = m->m_ext.ext_buf; 4875475ca5aSMark Johnston uma_zfree(nd_zone_clust, clust); 4885475ca5aSMark Johnston uma_zfree(nd_zone_mbuf, m); 4895475ca5aSMark Johnston } 4905475ca5aSMark Johnston } 4915475ca5aSMark Johnston 4925475ca5aSMark Johnston /* 4935475ca5aSMark Johnston * Free the pre-allocated mbufs and clusters reserved for netdump, and destroy 4945475ca5aSMark Johnston * the corresponding UMA cache zones. 4955475ca5aSMark Johnston */ 4965475ca5aSMark Johnston void 4975475ca5aSMark Johnston netdump_mbuf_drain(void) 4985475ca5aSMark Johnston { 4995475ca5aSMark Johnston struct mbuf *m; 5005475ca5aSMark Johnston void *item; 5015475ca5aSMark Johnston 5025475ca5aSMark Johnston if (nd_zone_mbuf != NULL) { 5035475ca5aSMark Johnston uma_zdestroy(nd_zone_mbuf); 5045475ca5aSMark Johnston nd_zone_mbuf = NULL; 5055475ca5aSMark Johnston } 5065475ca5aSMark Johnston if (nd_zone_clust != NULL) { 5075475ca5aSMark Johnston uma_zdestroy(nd_zone_clust); 5085475ca5aSMark Johnston nd_zone_clust = NULL; 5095475ca5aSMark Johnston } 5105475ca5aSMark Johnston if (nd_zone_pack != NULL) { 5115475ca5aSMark Johnston uma_zdestroy(nd_zone_pack); 5125475ca5aSMark Johnston nd_zone_pack = NULL; 5135475ca5aSMark Johnston } 5145475ca5aSMark Johnston 5155475ca5aSMark Johnston while ((m = mbufq_dequeue(&nd_mbufq)) != NULL) 5165475ca5aSMark Johnston m_free(m); 5175475ca5aSMark Johnston while ((item = mbufq_dequeue(&nd_clustq)) != NULL) 5185475ca5aSMark Johnston uma_zfree(m_getzone(nd_clsize), item); 5195475ca5aSMark Johnston } 5205475ca5aSMark Johnston 5215475ca5aSMark Johnston /* 5225475ca5aSMark Johnston * Callback invoked immediately prior to starting a netdump. 5235475ca5aSMark Johnston */ 5245475ca5aSMark Johnston void 5255475ca5aSMark Johnston netdump_mbuf_dump(void) 5265475ca5aSMark Johnston { 5275475ca5aSMark Johnston 5285475ca5aSMark Johnston /* 5295475ca5aSMark Johnston * All cluster zones return buffers of the size requested by the 5305475ca5aSMark Johnston * drivers. It's up to the driver to reinitialize the zones if the 5315475ca5aSMark Johnston * MTU of a netdump-enabled interface changes. 5325475ca5aSMark Johnston */ 5335475ca5aSMark Johnston printf("netdump: overwriting mbuf zone pointers\n"); 5345475ca5aSMark Johnston zone_mbuf = nd_zone_mbuf; 5355475ca5aSMark Johnston zone_clust = nd_zone_clust; 5365475ca5aSMark Johnston zone_pack = nd_zone_pack; 5375475ca5aSMark Johnston zone_jumbop = nd_zone_clust; 5385475ca5aSMark Johnston zone_jumbo9 = nd_zone_clust; 5395475ca5aSMark Johnston zone_jumbo16 = nd_zone_clust; 5405475ca5aSMark Johnston } 5415475ca5aSMark Johnston 5425475ca5aSMark Johnston /* 5435475ca5aSMark Johnston * Reinitialize the netdump mbuf+cluster pool and cache zones. 5445475ca5aSMark Johnston */ 5455475ca5aSMark Johnston void 5465475ca5aSMark Johnston netdump_mbuf_reinit(int nmbuf, int nclust, int clsize) 5475475ca5aSMark Johnston { 5485475ca5aSMark Johnston struct mbuf *m; 5495475ca5aSMark Johnston void *item; 5505475ca5aSMark Johnston 5515475ca5aSMark Johnston netdump_mbuf_drain(); 5525475ca5aSMark Johnston 5535475ca5aSMark Johnston nd_clsize = clsize; 5545475ca5aSMark Johnston 5555475ca5aSMark Johnston nd_zone_mbuf = uma_zcache_create("netdump_" MBUF_MEM_NAME, 5565475ca5aSMark Johnston MSIZE, mb_ctor_mbuf, mb_dtor_mbuf, 5575475ca5aSMark Johnston #ifdef INVARIANTS 5585475ca5aSMark Johnston trash_init, trash_fini, 5595475ca5aSMark Johnston #else 5605475ca5aSMark Johnston NULL, NULL, 5615475ca5aSMark Johnston #endif 5625475ca5aSMark Johnston nd_buf_import, nd_buf_release, 5635475ca5aSMark Johnston &nd_mbufq, UMA_ZONE_NOBUCKET); 5645475ca5aSMark Johnston 5655475ca5aSMark Johnston nd_zone_clust = uma_zcache_create("netdump_" MBUF_CLUSTER_MEM_NAME, 5665475ca5aSMark Johnston clsize, mb_ctor_clust, 5675475ca5aSMark Johnston #ifdef INVARIANTS 5685475ca5aSMark Johnston trash_dtor, trash_init, trash_fini, 5695475ca5aSMark Johnston #else 5705475ca5aSMark Johnston NULL, NULL, NULL, 5715475ca5aSMark Johnston #endif 5725475ca5aSMark Johnston nd_buf_import, nd_buf_release, 5735475ca5aSMark Johnston &nd_clustq, UMA_ZONE_NOBUCKET); 5745475ca5aSMark Johnston 5755475ca5aSMark Johnston nd_zone_pack = uma_zcache_create("netdump_" MBUF_PACKET_MEM_NAME, 5765475ca5aSMark Johnston MCLBYTES, mb_ctor_pack, mb_dtor_pack, NULL, NULL, 5775475ca5aSMark Johnston nd_pack_import, nd_pack_release, 5785475ca5aSMark Johnston NULL, UMA_ZONE_NOBUCKET); 5795475ca5aSMark Johnston 5805475ca5aSMark Johnston while (nmbuf-- > 0) { 5815475ca5aSMark Johnston m = m_get(MT_DATA, M_WAITOK); 5825475ca5aSMark Johnston uma_zfree(nd_zone_mbuf, m); 5835475ca5aSMark Johnston } 5845475ca5aSMark Johnston while (nclust-- > 0) { 5855475ca5aSMark Johnston item = uma_zalloc(m_getzone(nd_clsize), M_WAITOK); 5865475ca5aSMark Johnston uma_zfree(nd_zone_clust, item); 5875475ca5aSMark Johnston } 5885475ca5aSMark Johnston } 5895475ca5aSMark Johnston #endif /* NETDUMP */ 5905475ca5aSMark Johnston 591099a0e58SBosko Milekic /* 592ba63339aSAlan Cox * UMA backend page allocator for the jumbo frame zones. 593ba63339aSAlan Cox * 594ba63339aSAlan Cox * Allocates kernel virtual memory that is backed by contiguous physical 595ba63339aSAlan Cox * pages. 596ba63339aSAlan Cox */ 597ba63339aSAlan Cox static void * 598ab3185d1SJeff Roberson mbuf_jumbo_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, 599ab3185d1SJeff Roberson int wait) 600ba63339aSAlan Cox { 601ba63339aSAlan Cox 6027630c265SAlan Cox /* Inform UMA that this allocator uses kernel_map/object. */ 6037630c265SAlan Cox *flags = UMA_SLAB_KERNEL; 6049978bd99SMark Johnston return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain), 6059978bd99SMark Johnston bytes, wait, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, 6069978bd99SMark Johnston VM_MEMATTR_DEFAULT)); 607ba63339aSAlan Cox } 608ba63339aSAlan Cox 609ba63339aSAlan Cox /* 610099a0e58SBosko Milekic * Constructor for Mbuf master zone. 611099a0e58SBosko Milekic * 612099a0e58SBosko Milekic * The 'arg' pointer points to a mb_args structure which 613099a0e58SBosko Milekic * contains call-specific information required to support the 61456a4e45aSAndre Oppermann * mbuf allocation API. See mbuf.h. 615099a0e58SBosko Milekic */ 616b23f72e9SBrian Feldman static int 617b23f72e9SBrian Feldman mb_ctor_mbuf(void *mem, int size, void *arg, int how) 618099a0e58SBosko Milekic { 619099a0e58SBosko Milekic struct mbuf *m; 620099a0e58SBosko Milekic struct mb_args *args; 621b23f72e9SBrian Feldman int error; 622099a0e58SBosko Milekic int flags; 623099a0e58SBosko Milekic short type; 624099a0e58SBosko Milekic 625121f0509SMike Silbersack #ifdef INVARIANTS 626121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 627121f0509SMike Silbersack #endif 628099a0e58SBosko Milekic args = (struct mb_args *)arg; 629099a0e58SBosko Milekic type = args->type; 630099a0e58SBosko Milekic 63156a4e45aSAndre Oppermann /* 63256a4e45aSAndre Oppermann * The mbuf is initialized later. The caller has the 633fcf90618SGleb Smirnoff * responsibility to set up any MAC labels too. 63456a4e45aSAndre Oppermann */ 63556a4e45aSAndre Oppermann if (type == MT_NOINIT) 63656a4e45aSAndre Oppermann return (0); 63756a4e45aSAndre Oppermann 638afb295ccSAndre Oppermann m = (struct mbuf *)mem; 639afb295ccSAndre Oppermann flags = args->flags; 640fddd4f62SNavdeep Parhar MPASS((flags & M_NOFREE) == 0); 641afb295ccSAndre Oppermann 642b4b12e52SGleb Smirnoff error = m_init(m, how, type, flags); 643afb295ccSAndre Oppermann 644b23f72e9SBrian Feldman return (error); 645099a0e58SBosko Milekic } 646099a0e58SBosko Milekic 647099a0e58SBosko Milekic /* 64856a4e45aSAndre Oppermann * The Mbuf master zone destructor. 649099a0e58SBosko Milekic */ 650099a0e58SBosko Milekic static void 651099a0e58SBosko Milekic mb_dtor_mbuf(void *mem, int size, void *arg) 652099a0e58SBosko Milekic { 653099a0e58SBosko Milekic struct mbuf *m; 654629b9e08SKip Macy unsigned long flags; 655099a0e58SBosko Milekic 656099a0e58SBosko Milekic m = (struct mbuf *)mem; 657629b9e08SKip Macy flags = (unsigned long)arg; 658629b9e08SKip Macy 659a9fa76f2SNavdeep Parhar KASSERT((m->m_flags & M_NOFREE) == 0, ("%s: M_NOFREE set", __func__)); 6604c7070dbSScott Long if (!(flags & MB_DTOR_SKIP) && (m->m_flags & M_PKTHDR) && !SLIST_EMPTY(&m->m_pkthdr.tags)) 661099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 662121f0509SMike Silbersack #ifdef INVARIANTS 663121f0509SMike Silbersack trash_dtor(mem, size, arg); 664121f0509SMike Silbersack #endif 665099a0e58SBosko Milekic } 666099a0e58SBosko Milekic 66756a4e45aSAndre Oppermann /* 66856a4e45aSAndre Oppermann * The Mbuf Packet zone destructor. 66956a4e45aSAndre Oppermann */ 670099a0e58SBosko Milekic static void 671099a0e58SBosko Milekic mb_dtor_pack(void *mem, int size, void *arg) 672099a0e58SBosko Milekic { 673099a0e58SBosko Milekic struct mbuf *m; 674099a0e58SBosko Milekic 675099a0e58SBosko Milekic m = (struct mbuf *)mem; 676099a0e58SBosko Milekic if ((m->m_flags & M_PKTHDR) != 0) 677099a0e58SBosko Milekic m_tag_delete_chain(m, NULL); 67856a4e45aSAndre Oppermann 67956a4e45aSAndre Oppermann /* Make sure we've got a clean cluster back. */ 68056a4e45aSAndre Oppermann KASSERT((m->m_flags & M_EXT) == M_EXT, ("%s: M_EXT not set", __func__)); 68156a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_buf != NULL, ("%s: ext_buf == NULL", __func__)); 68256a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_free == NULL, ("%s: ext_free != NULL", __func__)); 683cf827063SPoul-Henning Kamp KASSERT(m->m_ext.ext_arg1 == NULL, ("%s: ext_arg1 != NULL", __func__)); 684cf827063SPoul-Henning Kamp KASSERT(m->m_ext.ext_arg2 == NULL, ("%s: ext_arg2 != NULL", __func__)); 68556a4e45aSAndre Oppermann KASSERT(m->m_ext.ext_size == MCLBYTES, ("%s: ext_size != MCLBYTES", __func__)); 68649d46b61SGleb Smirnoff KASSERT(m->m_ext.ext_type == EXT_PACKET, ("%s: ext_type != EXT_PACKET", __func__)); 687121f0509SMike Silbersack #ifdef INVARIANTS 688121f0509SMike Silbersack trash_dtor(m->m_ext.ext_buf, MCLBYTES, arg); 689121f0509SMike Silbersack #endif 6906c125b8dSMohan Srinivasan /* 691ef44c8d2SDavid E. O'Brien * If there are processes blocked on zone_clust, waiting for pages 692ef44c8d2SDavid E. O'Brien * to be freed up, * cause them to be woken up by draining the 693ef44c8d2SDavid E. O'Brien * packet zone. We are exposed to a race here * (in the check for 694ef44c8d2SDavid E. O'Brien * the UMA_ZFLAG_FULL) where we might miss the flag set, but that 695ef44c8d2SDavid E. O'Brien * is deliberate. We don't want to acquire the zone lock for every 696ef44c8d2SDavid E. O'Brien * mbuf free. 6976c125b8dSMohan Srinivasan */ 6986c125b8dSMohan Srinivasan if (uma_zone_exhausted_nolock(zone_clust)) 6996c125b8dSMohan Srinivasan zone_drain(zone_pack); 700099a0e58SBosko Milekic } 701099a0e58SBosko Milekic 702099a0e58SBosko Milekic /* 703ec63cb90SAndre Oppermann * The Cluster and Jumbo[PAGESIZE|9|16] zone constructor. 704099a0e58SBosko Milekic * 705099a0e58SBosko Milekic * Here the 'arg' pointer points to the Mbuf which we 70656a4e45aSAndre Oppermann * are configuring cluster storage for. If 'arg' is 70756a4e45aSAndre Oppermann * empty we allocate just the cluster without setting 70856a4e45aSAndre Oppermann * the mbuf to it. See mbuf.h. 709099a0e58SBosko Milekic */ 710b23f72e9SBrian Feldman static int 711b23f72e9SBrian Feldman mb_ctor_clust(void *mem, int size, void *arg, int how) 712099a0e58SBosko Milekic { 713099a0e58SBosko Milekic struct mbuf *m; 714099a0e58SBosko Milekic 715121f0509SMike Silbersack #ifdef INVARIANTS 716121f0509SMike Silbersack trash_ctor(mem, size, arg, how); 717121f0509SMike Silbersack #endif 7180f4d9d04SKip Macy m = (struct mbuf *)arg; 7190f4d9d04SKip Macy if (m != NULL) { 720e8fd18f3SGleb Smirnoff m->m_ext.ext_buf = (char *)mem; 721099a0e58SBosko Milekic m->m_data = m->m_ext.ext_buf; 722099a0e58SBosko Milekic m->m_flags |= M_EXT; 723099a0e58SBosko Milekic m->m_ext.ext_free = NULL; 724cf827063SPoul-Henning Kamp m->m_ext.ext_arg1 = NULL; 725cf827063SPoul-Henning Kamp m->m_ext.ext_arg2 = NULL; 72656a4e45aSAndre Oppermann m->m_ext.ext_size = size; 72756a5f52eSGleb Smirnoff m->m_ext.ext_type = m_gettype(size); 72856a5f52eSGleb Smirnoff m->m_ext.ext_flags = EXT_FLAG_EMBREF; 72956a5f52eSGleb Smirnoff m->m_ext.ext_count = 1; 73056a4e45aSAndre Oppermann } 7310f4d9d04SKip Macy 732b23f72e9SBrian Feldman return (0); 733099a0e58SBosko Milekic } 734099a0e58SBosko Milekic 73556a4e45aSAndre Oppermann /* 736099a0e58SBosko Milekic * The Packet secondary zone's init routine, executed on the 73756a4e45aSAndre Oppermann * object's transition from mbuf keg slab to zone cache. 738099a0e58SBosko Milekic */ 739b23f72e9SBrian Feldman static int 74056a4e45aSAndre Oppermann mb_zinit_pack(void *mem, int size, int how) 741099a0e58SBosko Milekic { 742099a0e58SBosko Milekic struct mbuf *m; 743099a0e58SBosko Milekic 74456a4e45aSAndre Oppermann m = (struct mbuf *)mem; /* m is virgin. */ 745a7bd90efSAndre Oppermann if (uma_zalloc_arg(zone_clust, m, how) == NULL || 746a7bd90efSAndre Oppermann m->m_ext.ext_buf == NULL) 747b23f72e9SBrian Feldman return (ENOMEM); 748cd5bb63bSAndre Oppermann m->m_ext.ext_type = EXT_PACKET; /* Override. */ 749121f0509SMike Silbersack #ifdef INVARIANTS 750121f0509SMike Silbersack trash_init(m->m_ext.ext_buf, MCLBYTES, how); 751121f0509SMike Silbersack #endif 752b23f72e9SBrian Feldman return (0); 753099a0e58SBosko Milekic } 754099a0e58SBosko Milekic 755099a0e58SBosko Milekic /* 756099a0e58SBosko Milekic * The Packet secondary zone's fini routine, executed on the 757099a0e58SBosko Milekic * object's transition from zone cache to keg slab. 758099a0e58SBosko Milekic */ 759099a0e58SBosko Milekic static void 76056a4e45aSAndre Oppermann mb_zfini_pack(void *mem, int size) 761099a0e58SBosko Milekic { 762099a0e58SBosko Milekic struct mbuf *m; 763099a0e58SBosko Milekic 764099a0e58SBosko Milekic m = (struct mbuf *)mem; 765121f0509SMike Silbersack #ifdef INVARIANTS 766121f0509SMike Silbersack trash_fini(m->m_ext.ext_buf, MCLBYTES); 767121f0509SMike Silbersack #endif 768099a0e58SBosko Milekic uma_zfree_arg(zone_clust, m->m_ext.ext_buf, NULL); 769a7b844d2SMike Silbersack #ifdef INVARIANTS 770a7b844d2SMike Silbersack trash_dtor(mem, size, NULL); 771a7b844d2SMike Silbersack #endif 772099a0e58SBosko Milekic } 773099a0e58SBosko Milekic 774099a0e58SBosko Milekic /* 775099a0e58SBosko Milekic * The "packet" keg constructor. 776099a0e58SBosko Milekic */ 777b23f72e9SBrian Feldman static int 778b23f72e9SBrian Feldman mb_ctor_pack(void *mem, int size, void *arg, int how) 779099a0e58SBosko Milekic { 780099a0e58SBosko Milekic struct mbuf *m; 781099a0e58SBosko Milekic struct mb_args *args; 782ce28636bSAndre Oppermann int error, flags; 783099a0e58SBosko Milekic short type; 784099a0e58SBosko Milekic 785099a0e58SBosko Milekic m = (struct mbuf *)mem; 786099a0e58SBosko Milekic args = (struct mb_args *)arg; 787099a0e58SBosko Milekic flags = args->flags; 788099a0e58SBosko Milekic type = args->type; 789fddd4f62SNavdeep Parhar MPASS((flags & M_NOFREE) == 0); 790099a0e58SBosko Milekic 791121f0509SMike Silbersack #ifdef INVARIANTS 792121f0509SMike Silbersack trash_ctor(m->m_ext.ext_buf, MCLBYTES, arg, how); 793121f0509SMike Silbersack #endif 794099a0e58SBosko Milekic 795b4b12e52SGleb Smirnoff error = m_init(m, how, type, flags); 796afb295ccSAndre Oppermann 79756a4e45aSAndre Oppermann /* m_ext is already initialized. */ 798afb295ccSAndre Oppermann m->m_data = m->m_ext.ext_buf; 799afb295ccSAndre Oppermann m->m_flags = (flags | M_EXT); 80056a4e45aSAndre Oppermann 801afb295ccSAndre Oppermann return (error); 802099a0e58SBosko Milekic } 803099a0e58SBosko Milekic 804099a0e58SBosko Milekic /* 805e60b2fcbSGleb Smirnoff * This is the protocol drain routine. Called by UMA whenever any of the 806e60b2fcbSGleb Smirnoff * mbuf zones is closed to its limit. 807099a0e58SBosko Milekic * 808099a0e58SBosko Milekic * No locks should be held when this is called. The drain routines have to 809099a0e58SBosko Milekic * presently acquire some locks which raises the possibility of lock order 810099a0e58SBosko Milekic * reversal. 811099a0e58SBosko Milekic */ 812099a0e58SBosko Milekic static void 813e60b2fcbSGleb Smirnoff mb_reclaim(uma_zone_t zone __unused, int pending __unused) 814099a0e58SBosko Milekic { 815099a0e58SBosko Milekic struct domain *dp; 816099a0e58SBosko Milekic struct protosw *pr; 817099a0e58SBosko Milekic 818e60b2fcbSGleb Smirnoff WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__); 819099a0e58SBosko Milekic 820099a0e58SBosko Milekic for (dp = domains; dp != NULL; dp = dp->dom_next) 821099a0e58SBosko Milekic for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++) 822099a0e58SBosko Milekic if (pr->pr_drain != NULL) 823099a0e58SBosko Milekic (*pr->pr_drain)(); 824099a0e58SBosko Milekic } 8255e4bc63bSGleb Smirnoff 8265e4bc63bSGleb Smirnoff /* 8275e4bc63bSGleb Smirnoff * Clean up after mbufs with M_EXT storage attached to them if the 8285e4bc63bSGleb Smirnoff * reference count hits 1. 8295e4bc63bSGleb Smirnoff */ 8305e4bc63bSGleb Smirnoff void 8315e4bc63bSGleb Smirnoff mb_free_ext(struct mbuf *m) 8325e4bc63bSGleb Smirnoff { 83356a5f52eSGleb Smirnoff volatile u_int *refcnt; 83456a5f52eSGleb Smirnoff struct mbuf *mref; 8355e4bc63bSGleb Smirnoff int freembuf; 8365e4bc63bSGleb Smirnoff 8375e4bc63bSGleb Smirnoff KASSERT(m->m_flags & M_EXT, ("%s: M_EXT not set on %p", __func__, m)); 8385e4bc63bSGleb Smirnoff 83956a5f52eSGleb Smirnoff /* See if this is the mbuf that holds the embedded refcount. */ 84056a5f52eSGleb Smirnoff if (m->m_ext.ext_flags & EXT_FLAG_EMBREF) { 84156a5f52eSGleb Smirnoff refcnt = &m->m_ext.ext_count; 84256a5f52eSGleb Smirnoff mref = m; 84356a5f52eSGleb Smirnoff } else { 84456a5f52eSGleb Smirnoff KASSERT(m->m_ext.ext_cnt != NULL, 84556a5f52eSGleb Smirnoff ("%s: no refcounting pointer on %p", __func__, m)); 84656a5f52eSGleb Smirnoff refcnt = m->m_ext.ext_cnt; 84756a5f52eSGleb Smirnoff mref = __containerof(refcnt, struct mbuf, m_ext.ext_count); 84856a5f52eSGleb Smirnoff } 84956a5f52eSGleb Smirnoff 8505e4bc63bSGleb Smirnoff /* 85156a5f52eSGleb Smirnoff * Check if the header is embedded in the cluster. It is 85256a5f52eSGleb Smirnoff * important that we can't touch any of the mbuf fields 85356a5f52eSGleb Smirnoff * after we have freed the external storage, since mbuf 85417cd649fSGleb Smirnoff * could have been embedded in it. For now, the mbufs 85517cd649fSGleb Smirnoff * embedded into the cluster are always of type EXT_EXTREF, 85617cd649fSGleb Smirnoff * and for this type we won't free the mref. 8575e4bc63bSGleb Smirnoff */ 85817cd649fSGleb Smirnoff if (m->m_flags & M_NOFREE) { 85917cd649fSGleb Smirnoff freembuf = 0; 860eec189c7SGleb Smirnoff KASSERT(m->m_ext.ext_type == EXT_EXTREF || 861eec189c7SGleb Smirnoff m->m_ext.ext_type == EXT_RXRING, 86217cd649fSGleb Smirnoff ("%s: no-free mbuf %p has wrong type", __func__, m)); 86317cd649fSGleb Smirnoff } else 86417cd649fSGleb Smirnoff freembuf = 1; 8655e4bc63bSGleb Smirnoff 86656a5f52eSGleb Smirnoff /* Free attached storage if this mbuf is the only reference to it. */ 86756a5f52eSGleb Smirnoff if (*refcnt == 1 || atomic_fetchadd_int(refcnt, -1) == 1) { 8685e4bc63bSGleb Smirnoff switch (m->m_ext.ext_type) { 86956a5f52eSGleb Smirnoff case EXT_PACKET: 87056a5f52eSGleb Smirnoff /* The packet zone is special. */ 87156a5f52eSGleb Smirnoff if (*refcnt == 0) 87256a5f52eSGleb Smirnoff *refcnt = 1; 87356a5f52eSGleb Smirnoff uma_zfree(zone_pack, mref); 8745e4bc63bSGleb Smirnoff break; 8755e4bc63bSGleb Smirnoff case EXT_CLUSTER: 8765e4bc63bSGleb Smirnoff uma_zfree(zone_clust, m->m_ext.ext_buf); 87756a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 8785e4bc63bSGleb Smirnoff break; 8795e4bc63bSGleb Smirnoff case EXT_JUMBOP: 8805e4bc63bSGleb Smirnoff uma_zfree(zone_jumbop, m->m_ext.ext_buf); 88156a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 8825e4bc63bSGleb Smirnoff break; 8835e4bc63bSGleb Smirnoff case EXT_JUMBO9: 8845e4bc63bSGleb Smirnoff uma_zfree(zone_jumbo9, m->m_ext.ext_buf); 88556a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 8865e4bc63bSGleb Smirnoff break; 8875e4bc63bSGleb Smirnoff case EXT_JUMBO16: 8885e4bc63bSGleb Smirnoff uma_zfree(zone_jumbo16, m->m_ext.ext_buf); 88956a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 89056a5f52eSGleb Smirnoff break; 89156a5f52eSGleb Smirnoff case EXT_SFBUF: 8925e4bc63bSGleb Smirnoff case EXT_NET_DRV: 8935e4bc63bSGleb Smirnoff case EXT_MOD_TYPE: 8945e4bc63bSGleb Smirnoff case EXT_DISPOSABLE: 89507e87a1dSGleb Smirnoff KASSERT(mref->m_ext.ext_free != NULL, 8960ea37a86SGleb Smirnoff ("%s: ext_free not set", __func__)); 89707e87a1dSGleb Smirnoff mref->m_ext.ext_free(mref); 89856a5f52eSGleb Smirnoff uma_zfree(zone_mbuf, mref); 8990ea37a86SGleb Smirnoff break; 9005e4bc63bSGleb Smirnoff case EXT_EXTREF: 9015e4bc63bSGleb Smirnoff KASSERT(m->m_ext.ext_free != NULL, 9025e4bc63bSGleb Smirnoff ("%s: ext_free not set", __func__)); 903e8fd18f3SGleb Smirnoff m->m_ext.ext_free(m); 9045e4bc63bSGleb Smirnoff break; 905eec189c7SGleb Smirnoff case EXT_RXRING: 906eec189c7SGleb Smirnoff KASSERT(m->m_ext.ext_free == NULL, 907eec189c7SGleb Smirnoff ("%s: ext_free is set", __func__)); 908eec189c7SGleb Smirnoff break; 9095e4bc63bSGleb Smirnoff default: 9105e4bc63bSGleb Smirnoff KASSERT(m->m_ext.ext_type == 0, 9115e4bc63bSGleb Smirnoff ("%s: unknown ext_type", __func__)); 9125e4bc63bSGleb Smirnoff } 9135e4bc63bSGleb Smirnoff } 9145e4bc63bSGleb Smirnoff 91556a5f52eSGleb Smirnoff if (freembuf && m != mref) 9165e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 9175e4bc63bSGleb Smirnoff } 9185e4bc63bSGleb Smirnoff 9195e4bc63bSGleb Smirnoff /* 9205e4bc63bSGleb Smirnoff * Official mbuf(9) allocation KPI for stack and drivers: 9215e4bc63bSGleb Smirnoff * 9225e4bc63bSGleb Smirnoff * m_get() - a single mbuf without any attachments, sys/mbuf.h. 9235e4bc63bSGleb Smirnoff * m_gethdr() - a single mbuf initialized as M_PKTHDR, sys/mbuf.h. 9245e4bc63bSGleb Smirnoff * m_getcl() - an mbuf + 2k cluster, sys/mbuf.h. 9255e4bc63bSGleb Smirnoff * m_clget() - attach cluster to already allocated mbuf. 9265e4bc63bSGleb Smirnoff * m_cljget() - attach jumbo cluster to already allocated mbuf. 9275e4bc63bSGleb Smirnoff * m_get2() - allocate minimum mbuf that would fit size argument. 9285e4bc63bSGleb Smirnoff * m_getm2() - allocate a chain of mbufs/clusters. 9295e4bc63bSGleb Smirnoff * m_extadd() - attach external cluster to mbuf. 9305e4bc63bSGleb Smirnoff * 9315e4bc63bSGleb Smirnoff * m_free() - free single mbuf with its tags and ext, sys/mbuf.h. 9325e4bc63bSGleb Smirnoff * m_freem() - free chain of mbufs. 9335e4bc63bSGleb Smirnoff */ 9345e4bc63bSGleb Smirnoff 9355e4bc63bSGleb Smirnoff int 9365e4bc63bSGleb Smirnoff m_clget(struct mbuf *m, int how) 9375e4bc63bSGleb Smirnoff { 9385e4bc63bSGleb Smirnoff 9395e4bc63bSGleb Smirnoff KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", 9405e4bc63bSGleb Smirnoff __func__, m)); 9415e4bc63bSGleb Smirnoff m->m_ext.ext_buf = (char *)NULL; 9425e4bc63bSGleb Smirnoff uma_zalloc_arg(zone_clust, m, how); 9435e4bc63bSGleb Smirnoff /* 9445e4bc63bSGleb Smirnoff * On a cluster allocation failure, drain the packet zone and retry, 9455e4bc63bSGleb Smirnoff * we might be able to loosen a few clusters up on the drain. 9465e4bc63bSGleb Smirnoff */ 9475e4bc63bSGleb Smirnoff if ((how & M_NOWAIT) && (m->m_ext.ext_buf == NULL)) { 9485e4bc63bSGleb Smirnoff zone_drain(zone_pack); 9495e4bc63bSGleb Smirnoff uma_zalloc_arg(zone_clust, m, how); 9505e4bc63bSGleb Smirnoff } 951480f4e94SGeorge V. Neville-Neil MBUF_PROBE2(m__clget, m, how); 9525e4bc63bSGleb Smirnoff return (m->m_flags & M_EXT); 9535e4bc63bSGleb Smirnoff } 9545e4bc63bSGleb Smirnoff 9555e4bc63bSGleb Smirnoff /* 9565e4bc63bSGleb Smirnoff * m_cljget() is different from m_clget() as it can allocate clusters without 9575e4bc63bSGleb Smirnoff * attaching them to an mbuf. In that case the return value is the pointer 9585e4bc63bSGleb Smirnoff * to the cluster of the requested size. If an mbuf was specified, it gets 9595e4bc63bSGleb Smirnoff * the cluster attached to it and the return value can be safely ignored. 9605e4bc63bSGleb Smirnoff * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. 9615e4bc63bSGleb Smirnoff */ 9625e4bc63bSGleb Smirnoff void * 9635e4bc63bSGleb Smirnoff m_cljget(struct mbuf *m, int how, int size) 9645e4bc63bSGleb Smirnoff { 9655e4bc63bSGleb Smirnoff uma_zone_t zone; 966480f4e94SGeorge V. Neville-Neil void *retval; 9675e4bc63bSGleb Smirnoff 9685e4bc63bSGleb Smirnoff if (m != NULL) { 9695e4bc63bSGleb Smirnoff KASSERT((m->m_flags & M_EXT) == 0, ("%s: mbuf %p has M_EXT", 9705e4bc63bSGleb Smirnoff __func__, m)); 9715e4bc63bSGleb Smirnoff m->m_ext.ext_buf = NULL; 9725e4bc63bSGleb Smirnoff } 9735e4bc63bSGleb Smirnoff 9745e4bc63bSGleb Smirnoff zone = m_getzone(size); 975480f4e94SGeorge V. Neville-Neil retval = uma_zalloc_arg(zone, m, how); 976480f4e94SGeorge V. Neville-Neil 977480f4e94SGeorge V. Neville-Neil MBUF_PROBE4(m__cljget, m, how, size, retval); 978480f4e94SGeorge V. Neville-Neil 979480f4e94SGeorge V. Neville-Neil return (retval); 9805e4bc63bSGleb Smirnoff } 9815e4bc63bSGleb Smirnoff 9825e4bc63bSGleb Smirnoff /* 9835e4bc63bSGleb Smirnoff * m_get2() allocates minimum mbuf that would fit "size" argument. 9845e4bc63bSGleb Smirnoff */ 9855e4bc63bSGleb Smirnoff struct mbuf * 9865e4bc63bSGleb Smirnoff m_get2(int size, int how, short type, int flags) 9875e4bc63bSGleb Smirnoff { 9885e4bc63bSGleb Smirnoff struct mb_args args; 9895e4bc63bSGleb Smirnoff struct mbuf *m, *n; 9905e4bc63bSGleb Smirnoff 9915e4bc63bSGleb Smirnoff args.flags = flags; 9925e4bc63bSGleb Smirnoff args.type = type; 9935e4bc63bSGleb Smirnoff 9945e4bc63bSGleb Smirnoff if (size <= MHLEN || (size <= MLEN && (flags & M_PKTHDR) == 0)) 9955e4bc63bSGleb Smirnoff return (uma_zalloc_arg(zone_mbuf, &args, how)); 9965e4bc63bSGleb Smirnoff if (size <= MCLBYTES) 9975e4bc63bSGleb Smirnoff return (uma_zalloc_arg(zone_pack, &args, how)); 9985e4bc63bSGleb Smirnoff 9995e4bc63bSGleb Smirnoff if (size > MJUMPAGESIZE) 10005e4bc63bSGleb Smirnoff return (NULL); 10015e4bc63bSGleb Smirnoff 10025e4bc63bSGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 10035e4bc63bSGleb Smirnoff if (m == NULL) 10045e4bc63bSGleb Smirnoff return (NULL); 10055e4bc63bSGleb Smirnoff 10065e4bc63bSGleb Smirnoff n = uma_zalloc_arg(zone_jumbop, m, how); 10075e4bc63bSGleb Smirnoff if (n == NULL) { 10085e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 10095e4bc63bSGleb Smirnoff return (NULL); 10105e4bc63bSGleb Smirnoff } 10115e4bc63bSGleb Smirnoff 10125e4bc63bSGleb Smirnoff return (m); 10135e4bc63bSGleb Smirnoff } 10145e4bc63bSGleb Smirnoff 10155e4bc63bSGleb Smirnoff /* 10165e4bc63bSGleb Smirnoff * m_getjcl() returns an mbuf with a cluster of the specified size attached. 10175e4bc63bSGleb Smirnoff * For size it takes MCLBYTES, MJUMPAGESIZE, MJUM9BYTES, MJUM16BYTES. 10185e4bc63bSGleb Smirnoff */ 10195e4bc63bSGleb Smirnoff struct mbuf * 10205e4bc63bSGleb Smirnoff m_getjcl(int how, short type, int flags, int size) 10215e4bc63bSGleb Smirnoff { 10225e4bc63bSGleb Smirnoff struct mb_args args; 10235e4bc63bSGleb Smirnoff struct mbuf *m, *n; 10245e4bc63bSGleb Smirnoff uma_zone_t zone; 10255e4bc63bSGleb Smirnoff 10265e4bc63bSGleb Smirnoff if (size == MCLBYTES) 10275e4bc63bSGleb Smirnoff return m_getcl(how, type, flags); 10285e4bc63bSGleb Smirnoff 10295e4bc63bSGleb Smirnoff args.flags = flags; 10305e4bc63bSGleb Smirnoff args.type = type; 10315e4bc63bSGleb Smirnoff 10325e4bc63bSGleb Smirnoff m = uma_zalloc_arg(zone_mbuf, &args, how); 10335e4bc63bSGleb Smirnoff if (m == NULL) 10345e4bc63bSGleb Smirnoff return (NULL); 10355e4bc63bSGleb Smirnoff 10365e4bc63bSGleb Smirnoff zone = m_getzone(size); 10375e4bc63bSGleb Smirnoff n = uma_zalloc_arg(zone, m, how); 10385e4bc63bSGleb Smirnoff if (n == NULL) { 10395e4bc63bSGleb Smirnoff uma_zfree(zone_mbuf, m); 10405e4bc63bSGleb Smirnoff return (NULL); 10415e4bc63bSGleb Smirnoff } 10425e4bc63bSGleb Smirnoff return (m); 10435e4bc63bSGleb Smirnoff } 10445e4bc63bSGleb Smirnoff 10455e4bc63bSGleb Smirnoff /* 10465e4bc63bSGleb Smirnoff * Allocate a given length worth of mbufs and/or clusters (whatever fits 10475e4bc63bSGleb Smirnoff * best) and return a pointer to the top of the allocated chain. If an 10485e4bc63bSGleb Smirnoff * existing mbuf chain is provided, then we will append the new chain 104958c43838SAndriy Voskoboinyk * to the existing one and return a pointer to the provided mbuf. 10505e4bc63bSGleb Smirnoff */ 10515e4bc63bSGleb Smirnoff struct mbuf * 10525e4bc63bSGleb Smirnoff m_getm2(struct mbuf *m, int len, int how, short type, int flags) 10535e4bc63bSGleb Smirnoff { 10545e4bc63bSGleb Smirnoff struct mbuf *mb, *nm = NULL, *mtail = NULL; 10555e4bc63bSGleb Smirnoff 10565e4bc63bSGleb Smirnoff KASSERT(len >= 0, ("%s: len is < 0", __func__)); 10575e4bc63bSGleb Smirnoff 10585e4bc63bSGleb Smirnoff /* Validate flags. */ 10595e4bc63bSGleb Smirnoff flags &= (M_PKTHDR | M_EOR); 10605e4bc63bSGleb Smirnoff 10615e4bc63bSGleb Smirnoff /* Packet header mbuf must be first in chain. */ 10625e4bc63bSGleb Smirnoff if ((flags & M_PKTHDR) && m != NULL) 10635e4bc63bSGleb Smirnoff flags &= ~M_PKTHDR; 10645e4bc63bSGleb Smirnoff 10655e4bc63bSGleb Smirnoff /* Loop and append maximum sized mbufs to the chain tail. */ 10665e4bc63bSGleb Smirnoff while (len > 0) { 10675e4bc63bSGleb Smirnoff if (len > MCLBYTES) 10685e4bc63bSGleb Smirnoff mb = m_getjcl(how, type, (flags & M_PKTHDR), 10695e4bc63bSGleb Smirnoff MJUMPAGESIZE); 10705e4bc63bSGleb Smirnoff else if (len >= MINCLSIZE) 10715e4bc63bSGleb Smirnoff mb = m_getcl(how, type, (flags & M_PKTHDR)); 10725e4bc63bSGleb Smirnoff else if (flags & M_PKTHDR) 10735e4bc63bSGleb Smirnoff mb = m_gethdr(how, type); 10745e4bc63bSGleb Smirnoff else 10755e4bc63bSGleb Smirnoff mb = m_get(how, type); 10765e4bc63bSGleb Smirnoff 10775e4bc63bSGleb Smirnoff /* Fail the whole operation if one mbuf can't be allocated. */ 10785e4bc63bSGleb Smirnoff if (mb == NULL) { 10795e4bc63bSGleb Smirnoff if (nm != NULL) 10805e4bc63bSGleb Smirnoff m_freem(nm); 10815e4bc63bSGleb Smirnoff return (NULL); 10825e4bc63bSGleb Smirnoff } 10835e4bc63bSGleb Smirnoff 10845e4bc63bSGleb Smirnoff /* Book keeping. */ 10855e4bc63bSGleb Smirnoff len -= M_SIZE(mb); 10865e4bc63bSGleb Smirnoff if (mtail != NULL) 10875e4bc63bSGleb Smirnoff mtail->m_next = mb; 10885e4bc63bSGleb Smirnoff else 10895e4bc63bSGleb Smirnoff nm = mb; 10905e4bc63bSGleb Smirnoff mtail = mb; 10915e4bc63bSGleb Smirnoff flags &= ~M_PKTHDR; /* Only valid on the first mbuf. */ 10925e4bc63bSGleb Smirnoff } 10935e4bc63bSGleb Smirnoff if (flags & M_EOR) 10945e4bc63bSGleb Smirnoff mtail->m_flags |= M_EOR; /* Only valid on the last mbuf. */ 10955e4bc63bSGleb Smirnoff 10965e4bc63bSGleb Smirnoff /* If mbuf was supplied, append new chain to the end of it. */ 10975e4bc63bSGleb Smirnoff if (m != NULL) { 10985e4bc63bSGleb Smirnoff for (mtail = m; mtail->m_next != NULL; mtail = mtail->m_next) 10995e4bc63bSGleb Smirnoff ; 11005e4bc63bSGleb Smirnoff mtail->m_next = nm; 11015e4bc63bSGleb Smirnoff mtail->m_flags &= ~M_EOR; 11025e4bc63bSGleb Smirnoff } else 11035e4bc63bSGleb Smirnoff m = nm; 11045e4bc63bSGleb Smirnoff 11055e4bc63bSGleb Smirnoff return (m); 11065e4bc63bSGleb Smirnoff } 11075e4bc63bSGleb Smirnoff 11085e4bc63bSGleb Smirnoff /*- 11095e4bc63bSGleb Smirnoff * Configure a provided mbuf to refer to the provided external storage 111056a5f52eSGleb Smirnoff * buffer and setup a reference count for said buffer. 11115e4bc63bSGleb Smirnoff * 11125e4bc63bSGleb Smirnoff * Arguments: 11135e4bc63bSGleb Smirnoff * mb The existing mbuf to which to attach the provided buffer. 11145e4bc63bSGleb Smirnoff * buf The address of the provided external storage buffer. 11155e4bc63bSGleb Smirnoff * size The size of the provided buffer. 11165e4bc63bSGleb Smirnoff * freef A pointer to a routine that is responsible for freeing the 11175e4bc63bSGleb Smirnoff * provided external storage buffer. 11185e4bc63bSGleb Smirnoff * args A pointer to an argument structure (of any type) to be passed 11195e4bc63bSGleb Smirnoff * to the provided freef routine (may be NULL). 11205e4bc63bSGleb Smirnoff * flags Any other flags to be passed to the provided mbuf. 11215e4bc63bSGleb Smirnoff * type The type that the external storage buffer should be 11225e4bc63bSGleb Smirnoff * labeled with. 11235e4bc63bSGleb Smirnoff * 11245e4bc63bSGleb Smirnoff * Returns: 11255e4bc63bSGleb Smirnoff * Nothing. 11265e4bc63bSGleb Smirnoff */ 112756a5f52eSGleb Smirnoff void 1128e8fd18f3SGleb Smirnoff m_extadd(struct mbuf *mb, char *buf, u_int size, m_ext_free_t freef, 1129e8fd18f3SGleb Smirnoff void *arg1, void *arg2, int flags, int type) 11305e4bc63bSGleb Smirnoff { 113156a5f52eSGleb Smirnoff 11325e4bc63bSGleb Smirnoff KASSERT(type != EXT_CLUSTER, ("%s: EXT_CLUSTER not allowed", __func__)); 11335e4bc63bSGleb Smirnoff 11345e4bc63bSGleb Smirnoff mb->m_flags |= (M_EXT | flags); 11355e4bc63bSGleb Smirnoff mb->m_ext.ext_buf = buf; 11365e4bc63bSGleb Smirnoff mb->m_data = mb->m_ext.ext_buf; 11375e4bc63bSGleb Smirnoff mb->m_ext.ext_size = size; 11385e4bc63bSGleb Smirnoff mb->m_ext.ext_free = freef; 11395e4bc63bSGleb Smirnoff mb->m_ext.ext_arg1 = arg1; 11405e4bc63bSGleb Smirnoff mb->m_ext.ext_arg2 = arg2; 11415e4bc63bSGleb Smirnoff mb->m_ext.ext_type = type; 11425e4bc63bSGleb Smirnoff 114356a5f52eSGleb Smirnoff if (type != EXT_EXTREF) { 114456a5f52eSGleb Smirnoff mb->m_ext.ext_count = 1; 114556a5f52eSGleb Smirnoff mb->m_ext.ext_flags = EXT_FLAG_EMBREF; 114656a5f52eSGleb Smirnoff } else 114756a5f52eSGleb Smirnoff mb->m_ext.ext_flags = 0; 11485e4bc63bSGleb Smirnoff } 11495e4bc63bSGleb Smirnoff 11505e4bc63bSGleb Smirnoff /* 11515e4bc63bSGleb Smirnoff * Free an entire chain of mbufs and associated external buffers, if 11525e4bc63bSGleb Smirnoff * applicable. 11535e4bc63bSGleb Smirnoff */ 11545e4bc63bSGleb Smirnoff void 11555e4bc63bSGleb Smirnoff m_freem(struct mbuf *mb) 11565e4bc63bSGleb Smirnoff { 11575e4bc63bSGleb Smirnoff 1158c8f59118SGleb Smirnoff MBUF_PROBE1(m__freem, mb); 11595e4bc63bSGleb Smirnoff while (mb != NULL) 11605e4bc63bSGleb Smirnoff mb = m_free(mb); 11615e4bc63bSGleb Smirnoff } 1162*fb3bc596SJohn Baldwin 1163*fb3bc596SJohn Baldwin void 1164*fb3bc596SJohn Baldwin m_snd_tag_init(struct m_snd_tag *mst, struct ifnet *ifp) 1165*fb3bc596SJohn Baldwin { 1166*fb3bc596SJohn Baldwin 1167*fb3bc596SJohn Baldwin if_ref(ifp); 1168*fb3bc596SJohn Baldwin mst->ifp = ifp; 1169*fb3bc596SJohn Baldwin refcount_init(&mst->refcount, 1); 1170*fb3bc596SJohn Baldwin counter_u64_add(snd_tag_count, 1); 1171*fb3bc596SJohn Baldwin } 1172*fb3bc596SJohn Baldwin 1173*fb3bc596SJohn Baldwin void 1174*fb3bc596SJohn Baldwin m_snd_tag_destroy(struct m_snd_tag *mst) 1175*fb3bc596SJohn Baldwin { 1176*fb3bc596SJohn Baldwin struct ifnet *ifp; 1177*fb3bc596SJohn Baldwin 1178*fb3bc596SJohn Baldwin ifp = mst->ifp; 1179*fb3bc596SJohn Baldwin ifp->if_snd_tag_free(mst); 1180*fb3bc596SJohn Baldwin if_rele(ifp); 1181*fb3bc596SJohn Baldwin counter_u64_add(snd_tag_count, -1); 1182*fb3bc596SJohn Baldwin } 1183