1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2002-2019 Jeffrey Roberson <jeff@FreeBSD.org> 5 * Copyright (c) 2004, 2005 Bosko Milekic <bmilekic@FreeBSD.org> 6 * Copyright (c) 2004-2006 Robert N. M. Watson 7 * All rights reserved. 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice unmodified, this list of conditions, and the following 14 * disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 24 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 28 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 29 */ 30 31 /* 32 * uma_core.c Implementation of the Universal Memory allocator 33 * 34 * This allocator is intended to replace the multitude of similar object caches 35 * in the standard FreeBSD kernel. The intent is to be flexible as well as 36 * efficient. A primary design goal is to return unused memory to the rest of 37 * the system. This will make the system as a whole more flexible due to the 38 * ability to move memory to subsystems which most need it instead of leaving 39 * pools of reserved memory unused. 40 * 41 * The basic ideas stem from similar slab/zone based allocators whose algorithms 42 * are well known. 43 * 44 */ 45 46 /* 47 * TODO: 48 * - Improve memory usage for large allocations 49 * - Investigate cache size adjustments 50 */ 51 52 #include <sys/cdefs.h> 53 #include "opt_ddb.h" 54 #include "opt_param.h" 55 #include "opt_vm.h" 56 57 #include <sys/param.h> 58 #include <sys/systm.h> 59 #include <sys/asan.h> 60 #include <sys/bitset.h> 61 #include <sys/domainset.h> 62 #include <sys/eventhandler.h> 63 #include <sys/kernel.h> 64 #include <sys/types.h> 65 #include <sys/limits.h> 66 #include <sys/queue.h> 67 #include <sys/malloc.h> 68 #include <sys/ktr.h> 69 #include <sys/lock.h> 70 #include <sys/msan.h> 71 #include <sys/mutex.h> 72 #include <sys/proc.h> 73 #include <sys/random.h> 74 #include <sys/rwlock.h> 75 #include <sys/sbuf.h> 76 #include <sys/sched.h> 77 #include <sys/sleepqueue.h> 78 #include <sys/smp.h> 79 #include <sys/smr.h> 80 #include <sys/sysctl.h> 81 #include <sys/taskqueue.h> 82 #include <sys/vmmeter.h> 83 84 #include <vm/vm.h> 85 #include <vm/vm_param.h> 86 #include <vm/vm_domainset.h> 87 #include <vm/vm_page.h> 88 #include <vm/vm_pageout.h> 89 #include <vm/vm_phys.h> 90 #include <vm/vm_pagequeue.h> 91 #include <vm/vm_map.h> 92 #include <vm/vm_kern.h> 93 #include <vm/vm_extern.h> 94 #include <vm/vm_dumpset.h> 95 #include <vm/uma.h> 96 #include <vm/uma_int.h> 97 #include <vm/uma_dbg.h> 98 99 #include <ddb/ddb.h> 100 101 #ifdef DEBUG_MEMGUARD 102 #include <vm/memguard.h> 103 #endif 104 105 #include <machine/md_var.h> 106 107 #ifdef INVARIANTS 108 #define UMA_ALWAYS_CTORDTOR 1 109 #else 110 #define UMA_ALWAYS_CTORDTOR 0 111 #endif 112 113 /* 114 * This is the zone and keg from which all zones are spawned. 115 */ 116 static uma_zone_t kegs; 117 static uma_zone_t zones; 118 119 /* 120 * On INVARIANTS builds, the slab contains a second bitset of the same size, 121 * "dbg_bits", which is laid out immediately after us_free. 122 */ 123 #ifdef INVARIANTS 124 #define SLAB_BITSETS 2 125 #else 126 #define SLAB_BITSETS 1 127 #endif 128 129 /* 130 * These are the two zones from which all offpage uma_slab_ts are allocated. 131 * 132 * One zone is for slab headers that can represent a larger number of items, 133 * making the slabs themselves more efficient, and the other zone is for 134 * headers that are smaller and represent fewer items, making the headers more 135 * efficient. 136 */ 137 #define SLABZONE_SIZE(setsize) \ 138 (sizeof(struct uma_hash_slab) + BITSET_SIZE(setsize) * SLAB_BITSETS) 139 #define SLABZONE0_SETSIZE (PAGE_SIZE / 16) 140 #define SLABZONE1_SETSIZE SLAB_MAX_SETSIZE 141 #define SLABZONE0_SIZE SLABZONE_SIZE(SLABZONE0_SETSIZE) 142 #define SLABZONE1_SIZE SLABZONE_SIZE(SLABZONE1_SETSIZE) 143 static uma_zone_t slabzones[2]; 144 145 /* 146 * The initial hash tables come out of this zone so they can be allocated 147 * prior to malloc coming up. 148 */ 149 static uma_zone_t hashzone; 150 151 /* The boot-time adjusted value for cache line alignment. */ 152 static unsigned int uma_cache_align_mask = 64 - 1; 153 154 static MALLOC_DEFINE(M_UMAHASH, "UMAHash", "UMA Hash Buckets"); 155 static MALLOC_DEFINE(M_UMA, "UMA", "UMA Misc"); 156 157 /* 158 * Are we allowed to allocate buckets? 159 */ 160 static int bucketdisable = 1; 161 162 /* Linked list of all kegs in the system */ 163 static LIST_HEAD(,uma_keg) uma_kegs = LIST_HEAD_INITIALIZER(uma_kegs); 164 165 /* Linked list of all cache-only zones in the system */ 166 static LIST_HEAD(,uma_zone) uma_cachezones = 167 LIST_HEAD_INITIALIZER(uma_cachezones); 168 169 /* 170 * Mutex for global lists: uma_kegs, uma_cachezones, and the per-keg list of 171 * zones. 172 */ 173 static struct rwlock_padalign __exclusive_cache_line uma_rwlock; 174 175 static struct sx uma_reclaim_lock; 176 177 /* 178 * First available virual address for boot time allocations. 179 */ 180 static vm_offset_t bootstart; 181 static vm_offset_t bootmem; 182 183 /* 184 * kmem soft limit, initialized by uma_set_limit(). Ensure that early 185 * allocations don't trigger a wakeup of the reclaim thread. 186 */ 187 unsigned long uma_kmem_limit = LONG_MAX; 188 SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_limit, CTLFLAG_RD, &uma_kmem_limit, 0, 189 "UMA kernel memory soft limit"); 190 unsigned long uma_kmem_total; 191 SYSCTL_ULONG(_vm, OID_AUTO, uma_kmem_total, CTLFLAG_RD, &uma_kmem_total, 0, 192 "UMA kernel memory usage"); 193 194 /* Is the VM done starting up? */ 195 static enum { 196 BOOT_COLD, 197 BOOT_KVA, 198 BOOT_PCPU, 199 BOOT_RUNNING, 200 BOOT_SHUTDOWN, 201 } booted = BOOT_COLD; 202 203 /* 204 * This is the handle used to schedule events that need to happen 205 * outside of the allocation fast path. 206 */ 207 static struct timeout_task uma_timeout_task; 208 #define UMA_TIMEOUT 20 /* Seconds for callout interval. */ 209 210 /* 211 * This structure is passed as the zone ctor arg so that I don't have to create 212 * a special allocation function just for zones. 213 */ 214 struct uma_zctor_args { 215 const char *name; 216 size_t size; 217 uma_ctor ctor; 218 uma_dtor dtor; 219 uma_init uminit; 220 uma_fini fini; 221 uma_import import; 222 uma_release release; 223 void *arg; 224 uma_keg_t keg; 225 int align; 226 uint32_t flags; 227 }; 228 229 struct uma_kctor_args { 230 uma_zone_t zone; 231 size_t size; 232 uma_init uminit; 233 uma_fini fini; 234 int align; 235 uint32_t flags; 236 }; 237 238 struct uma_bucket_zone { 239 uma_zone_t ubz_zone; 240 const char *ubz_name; 241 int ubz_entries; /* Number of items it can hold. */ 242 int ubz_maxsize; /* Maximum allocation size per-item. */ 243 }; 244 245 /* 246 * Compute the actual number of bucket entries to pack them in power 247 * of two sizes for more efficient space utilization. 248 */ 249 #define BUCKET_SIZE(n) \ 250 (((sizeof(void *) * (n)) - sizeof(struct uma_bucket)) / sizeof(void *)) 251 252 #define BUCKET_MAX BUCKET_SIZE(256) 253 254 struct uma_bucket_zone bucket_zones[] = { 255 /* Literal bucket sizes. */ 256 { NULL, "2 Bucket", 2, 4096 }, 257 { NULL, "4 Bucket", 4, 3072 }, 258 { NULL, "8 Bucket", 8, 2048 }, 259 { NULL, "16 Bucket", 16, 1024 }, 260 /* Rounded down power of 2 sizes for efficiency. */ 261 { NULL, "32 Bucket", BUCKET_SIZE(32), 512 }, 262 { NULL, "64 Bucket", BUCKET_SIZE(64), 256 }, 263 { NULL, "128 Bucket", BUCKET_SIZE(128), 128 }, 264 { NULL, "256 Bucket", BUCKET_SIZE(256), 64 }, 265 { NULL, NULL, 0} 266 }; 267 268 /* 269 * Flags and enumerations to be passed to internal functions. 270 */ 271 enum zfreeskip { 272 SKIP_NONE = 0, 273 SKIP_CNT = 0x00000001, 274 SKIP_DTOR = 0x00010000, 275 SKIP_FINI = 0x00020000, 276 }; 277 278 /* Prototypes.. */ 279 280 void uma_startup1(vm_offset_t); 281 void uma_startup2(void); 282 283 static void *noobj_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 284 static void *page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 285 static void *pcpu_page_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 286 static void *startup_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 287 static void *contig_alloc(uma_zone_t, vm_size_t, int, uint8_t *, int); 288 static void page_free(void *, vm_size_t, uint8_t); 289 static void pcpu_page_free(void *, vm_size_t, uint8_t); 290 static uma_slab_t keg_alloc_slab(uma_keg_t, uma_zone_t, int, int, int); 291 static void cache_drain(uma_zone_t); 292 static void bucket_drain(uma_zone_t, uma_bucket_t); 293 static void bucket_cache_reclaim(uma_zone_t zone, bool, int); 294 static bool bucket_cache_reclaim_domain(uma_zone_t, bool, bool, int); 295 static int keg_ctor(void *, int, void *, int); 296 static void keg_dtor(void *, int, void *); 297 static void keg_drain(uma_keg_t keg, int domain); 298 static int zone_ctor(void *, int, void *, int); 299 static void zone_dtor(void *, int, void *); 300 static inline void item_dtor(uma_zone_t zone, void *item, int size, 301 void *udata, enum zfreeskip skip); 302 static int zero_init(void *, int, int); 303 static void zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, 304 int itemdomain, bool ws); 305 static void zone_foreach(void (*zfunc)(uma_zone_t, void *), void *); 306 static void zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *), void *); 307 static void zone_timeout(uma_zone_t zone, void *); 308 static int hash_alloc(struct uma_hash *, u_int); 309 static int hash_expand(struct uma_hash *, struct uma_hash *); 310 static void hash_free(struct uma_hash *hash); 311 static void uma_timeout(void *, int); 312 static void uma_shutdown(void); 313 static void *zone_alloc_item(uma_zone_t, void *, int, int); 314 static void zone_free_item(uma_zone_t, void *, void *, enum zfreeskip); 315 static int zone_alloc_limit(uma_zone_t zone, int count, int flags); 316 static void zone_free_limit(uma_zone_t zone, int count); 317 static void bucket_enable(void); 318 static void bucket_init(void); 319 static uma_bucket_t bucket_alloc(uma_zone_t zone, void *, int); 320 static void bucket_free(uma_zone_t zone, uma_bucket_t, void *); 321 static void bucket_zone_drain(int domain); 322 static uma_bucket_t zone_alloc_bucket(uma_zone_t, void *, int, int); 323 static void *slab_alloc_item(uma_keg_t keg, uma_slab_t slab); 324 static void slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item); 325 static size_t slab_sizeof(int nitems); 326 static uma_keg_t uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, 327 uma_fini fini, int align, uint32_t flags); 328 static int zone_import(void *, void **, int, int, int); 329 static void zone_release(void *, void **, int); 330 static bool cache_alloc(uma_zone_t, uma_cache_t, void *, int); 331 static bool cache_free(uma_zone_t, uma_cache_t, void *, int); 332 333 static int sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS); 334 static int sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS); 335 static int sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS); 336 static int sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS); 337 static int sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS); 338 static int sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS); 339 static int sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS); 340 341 static uint64_t uma_zone_get_allocs(uma_zone_t zone); 342 343 static SYSCTL_NODE(_vm, OID_AUTO, debug, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 344 "Memory allocation debugging"); 345 346 #ifdef INVARIANTS 347 static uint64_t uma_keg_get_allocs(uma_keg_t zone); 348 static inline struct noslabbits *slab_dbg_bits(uma_slab_t slab, uma_keg_t keg); 349 350 static bool uma_dbg_kskip(uma_keg_t keg, void *mem); 351 static bool uma_dbg_zskip(uma_zone_t zone, void *mem); 352 static void uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item); 353 static void uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item); 354 355 static u_int dbg_divisor = 1; 356 SYSCTL_UINT(_vm_debug, OID_AUTO, divisor, 357 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &dbg_divisor, 0, 358 "Debug & thrash every this item in memory allocator"); 359 360 static counter_u64_t uma_dbg_cnt = EARLY_COUNTER; 361 static counter_u64_t uma_skip_cnt = EARLY_COUNTER; 362 SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, trashed, CTLFLAG_RD, 363 &uma_dbg_cnt, "memory items debugged"); 364 SYSCTL_COUNTER_U64(_vm_debug, OID_AUTO, skipped, CTLFLAG_RD, 365 &uma_skip_cnt, "memory items skipped, not debugged"); 366 #endif 367 368 SYSCTL_NODE(_vm, OID_AUTO, uma, CTLFLAG_RW | CTLFLAG_MPSAFE, 0, 369 "Universal Memory Allocator"); 370 371 SYSCTL_PROC(_vm, OID_AUTO, zone_count, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_INT, 372 0, 0, sysctl_vm_zone_count, "I", "Number of UMA zones"); 373 374 SYSCTL_PROC(_vm, OID_AUTO, zone_stats, CTLFLAG_RD|CTLFLAG_MPSAFE|CTLTYPE_STRUCT, 375 0, 0, sysctl_vm_zone_stats, "s,struct uma_type_header", "Zone Stats"); 376 377 static int zone_warnings = 1; 378 SYSCTL_INT(_vm, OID_AUTO, zone_warnings, CTLFLAG_RWTUN, &zone_warnings, 0, 379 "Warn when UMA zones becomes full"); 380 381 static int multipage_slabs = 1; 382 TUNABLE_INT("vm.debug.uma_multipage_slabs", &multipage_slabs); 383 SYSCTL_INT(_vm_debug, OID_AUTO, uma_multipage_slabs, 384 CTLFLAG_RDTUN | CTLFLAG_NOFETCH, &multipage_slabs, 0, 385 "UMA may choose larger slab sizes for better efficiency"); 386 387 /* 388 * Select the slab zone for an offpage slab with the given maximum item count. 389 */ 390 static inline uma_zone_t 391 slabzone(int ipers) 392 { 393 394 return (slabzones[ipers > SLABZONE0_SETSIZE]); 395 } 396 397 /* 398 * This routine checks to see whether or not it's safe to enable buckets. 399 */ 400 static void 401 bucket_enable(void) 402 { 403 404 KASSERT(booted >= BOOT_KVA, ("Bucket enable before init")); 405 bucketdisable = vm_page_count_min(); 406 } 407 408 /* 409 * Initialize bucket_zones, the array of zones of buckets of various sizes. 410 * 411 * For each zone, calculate the memory required for each bucket, consisting 412 * of the header and an array of pointers. 413 */ 414 static void 415 bucket_init(void) 416 { 417 struct uma_bucket_zone *ubz; 418 int size; 419 420 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) { 421 size = roundup(sizeof(struct uma_bucket), sizeof(void *)); 422 size += sizeof(void *) * ubz->ubz_entries; 423 ubz->ubz_zone = uma_zcreate(ubz->ubz_name, size, 424 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, 425 UMA_ZONE_MTXCLASS | UMA_ZFLAG_BUCKET | 426 UMA_ZONE_FIRSTTOUCH); 427 } 428 } 429 430 /* 431 * Given a desired number of entries for a bucket, return the zone from which 432 * to allocate the bucket. 433 */ 434 static struct uma_bucket_zone * 435 bucket_zone_lookup(int entries) 436 { 437 struct uma_bucket_zone *ubz; 438 439 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 440 if (ubz->ubz_entries >= entries) 441 return (ubz); 442 ubz--; 443 return (ubz); 444 } 445 446 static int 447 bucket_select(int size) 448 { 449 struct uma_bucket_zone *ubz; 450 451 ubz = &bucket_zones[0]; 452 if (size > ubz->ubz_maxsize) 453 return MAX((ubz->ubz_maxsize * ubz->ubz_entries) / size, 1); 454 455 for (; ubz->ubz_entries != 0; ubz++) 456 if (ubz->ubz_maxsize < size) 457 break; 458 ubz--; 459 return (ubz->ubz_entries); 460 } 461 462 static uma_bucket_t 463 bucket_alloc(uma_zone_t zone, void *udata, int flags) 464 { 465 struct uma_bucket_zone *ubz; 466 uma_bucket_t bucket; 467 468 /* 469 * Don't allocate buckets early in boot. 470 */ 471 if (__predict_false(booted < BOOT_KVA)) 472 return (NULL); 473 474 /* 475 * To limit bucket recursion we store the original zone flags 476 * in a cookie passed via zalloc_arg/zfree_arg. This allows the 477 * NOVM flag to persist even through deep recursions. We also 478 * store ZFLAG_BUCKET once we have recursed attempting to allocate 479 * a bucket for a bucket zone so we do not allow infinite bucket 480 * recursion. This cookie will even persist to frees of unused 481 * buckets via the allocation path or bucket allocations in the 482 * free path. 483 */ 484 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 485 udata = (void *)(uintptr_t)zone->uz_flags; 486 else { 487 if ((uintptr_t)udata & UMA_ZFLAG_BUCKET) 488 return (NULL); 489 udata = (void *)((uintptr_t)udata | UMA_ZFLAG_BUCKET); 490 } 491 if (((uintptr_t)udata & UMA_ZONE_VM) != 0) 492 flags |= M_NOVM; 493 ubz = bucket_zone_lookup(atomic_load_16(&zone->uz_bucket_size)); 494 if (ubz->ubz_zone == zone && (ubz + 1)->ubz_entries != 0) 495 ubz++; 496 bucket = uma_zalloc_arg(ubz->ubz_zone, udata, flags); 497 if (bucket) { 498 #ifdef INVARIANTS 499 bzero(bucket->ub_bucket, sizeof(void *) * ubz->ubz_entries); 500 #endif 501 bucket->ub_cnt = 0; 502 bucket->ub_entries = min(ubz->ubz_entries, 503 zone->uz_bucket_size_max); 504 bucket->ub_seq = SMR_SEQ_INVALID; 505 CTR3(KTR_UMA, "bucket_alloc: zone %s(%p) allocated bucket %p", 506 zone->uz_name, zone, bucket); 507 } 508 509 return (bucket); 510 } 511 512 static void 513 bucket_free(uma_zone_t zone, uma_bucket_t bucket, void *udata) 514 { 515 struct uma_bucket_zone *ubz; 516 517 if (bucket->ub_cnt != 0) 518 bucket_drain(zone, bucket); 519 520 KASSERT(bucket->ub_cnt == 0, 521 ("bucket_free: Freeing a non free bucket.")); 522 KASSERT(bucket->ub_seq == SMR_SEQ_INVALID, 523 ("bucket_free: Freeing an SMR bucket.")); 524 if ((zone->uz_flags & UMA_ZFLAG_BUCKET) == 0) 525 udata = (void *)(uintptr_t)zone->uz_flags; 526 ubz = bucket_zone_lookup(bucket->ub_entries); 527 uma_zfree_arg(ubz->ubz_zone, bucket, udata); 528 } 529 530 static void 531 bucket_zone_drain(int domain) 532 { 533 struct uma_bucket_zone *ubz; 534 535 for (ubz = &bucket_zones[0]; ubz->ubz_entries != 0; ubz++) 536 uma_zone_reclaim_domain(ubz->ubz_zone, UMA_RECLAIM_DRAIN, 537 domain); 538 } 539 540 #ifdef KASAN 541 _Static_assert(UMA_SMALLEST_UNIT % KASAN_SHADOW_SCALE == 0, 542 "Base UMA allocation size not a multiple of the KASAN scale factor"); 543 544 static void 545 kasan_mark_item_valid(uma_zone_t zone, void *item) 546 { 547 void *pcpu_item; 548 size_t sz, rsz; 549 int i; 550 551 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) 552 return; 553 554 sz = zone->uz_size; 555 rsz = roundup2(sz, KASAN_SHADOW_SCALE); 556 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { 557 kasan_mark(item, sz, rsz, KASAN_GENERIC_REDZONE); 558 } else { 559 pcpu_item = zpcpu_base_to_offset(item); 560 for (i = 0; i <= mp_maxid; i++) 561 kasan_mark(zpcpu_get_cpu(pcpu_item, i), sz, rsz, 562 KASAN_GENERIC_REDZONE); 563 } 564 } 565 566 static void 567 kasan_mark_item_invalid(uma_zone_t zone, void *item) 568 { 569 void *pcpu_item; 570 size_t sz; 571 int i; 572 573 if ((zone->uz_flags & UMA_ZONE_NOKASAN) != 0) 574 return; 575 576 sz = roundup2(zone->uz_size, KASAN_SHADOW_SCALE); 577 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { 578 kasan_mark(item, 0, sz, KASAN_UMA_FREED); 579 } else { 580 pcpu_item = zpcpu_base_to_offset(item); 581 for (i = 0; i <= mp_maxid; i++) 582 kasan_mark(zpcpu_get_cpu(pcpu_item, i), 0, sz, 583 KASAN_UMA_FREED); 584 } 585 } 586 587 static void 588 kasan_mark_slab_valid(uma_keg_t keg, void *mem) 589 { 590 size_t sz; 591 592 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { 593 sz = keg->uk_ppera * PAGE_SIZE; 594 kasan_mark(mem, sz, sz, 0); 595 } 596 } 597 598 static void 599 kasan_mark_slab_invalid(uma_keg_t keg, void *mem) 600 { 601 size_t sz; 602 603 if ((keg->uk_flags & UMA_ZONE_NOKASAN) == 0) { 604 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) 605 sz = keg->uk_ppera * PAGE_SIZE; 606 else 607 sz = keg->uk_pgoff; 608 kasan_mark(mem, 0, sz, KASAN_UMA_FREED); 609 } 610 } 611 #else /* !KASAN */ 612 static void 613 kasan_mark_item_valid(uma_zone_t zone __unused, void *item __unused) 614 { 615 } 616 617 static void 618 kasan_mark_item_invalid(uma_zone_t zone __unused, void *item __unused) 619 { 620 } 621 622 static void 623 kasan_mark_slab_valid(uma_keg_t keg __unused, void *mem __unused) 624 { 625 } 626 627 static void 628 kasan_mark_slab_invalid(uma_keg_t keg __unused, void *mem __unused) 629 { 630 } 631 #endif /* KASAN */ 632 633 #ifdef KMSAN 634 static inline void 635 kmsan_mark_item_uninitialized(uma_zone_t zone, void *item) 636 { 637 void *pcpu_item; 638 size_t sz; 639 int i; 640 641 if ((zone->uz_flags & 642 (UMA_ZFLAG_CACHE | UMA_ZONE_SECONDARY | UMA_ZONE_MALLOC)) != 0) { 643 /* 644 * Cache zones should not be instrumented by default, as UMA 645 * does not have enough information to do so correctly. 646 * Consumers can mark items themselves if it makes sense to do 647 * so. 648 * 649 * Items from secondary zones are initialized by the parent 650 * zone and thus cannot safely be marked by UMA. 651 * 652 * malloc zones are handled directly by malloc(9) and friends, 653 * since they can provide more precise origin tracking. 654 */ 655 return; 656 } 657 if (zone->uz_keg->uk_init != NULL) { 658 /* 659 * By definition, initialized items cannot be marked. The 660 * best we can do is mark items from these zones after they 661 * are freed to the keg. 662 */ 663 return; 664 } 665 666 sz = zone->uz_size; 667 if ((zone->uz_flags & UMA_ZONE_PCPU) == 0) { 668 kmsan_orig(item, sz, KMSAN_TYPE_UMA, KMSAN_RET_ADDR); 669 kmsan_mark(item, sz, KMSAN_STATE_UNINIT); 670 } else { 671 pcpu_item = zpcpu_base_to_offset(item); 672 for (i = 0; i <= mp_maxid; i++) { 673 kmsan_orig(zpcpu_get_cpu(pcpu_item, i), sz, 674 KMSAN_TYPE_UMA, KMSAN_RET_ADDR); 675 kmsan_mark(zpcpu_get_cpu(pcpu_item, i), sz, 676 KMSAN_STATE_INITED); 677 } 678 } 679 } 680 #else /* !KMSAN */ 681 static inline void 682 kmsan_mark_item_uninitialized(uma_zone_t zone __unused, void *item __unused) 683 { 684 } 685 #endif /* KMSAN */ 686 687 /* 688 * Acquire the domain lock and record contention. 689 */ 690 static uma_zone_domain_t 691 zone_domain_lock(uma_zone_t zone, int domain) 692 { 693 uma_zone_domain_t zdom; 694 bool lockfail; 695 696 zdom = ZDOM_GET(zone, domain); 697 lockfail = false; 698 if (ZDOM_OWNED(zdom)) 699 lockfail = true; 700 ZDOM_LOCK(zdom); 701 /* This is unsynchronized. The counter does not need to be precise. */ 702 if (lockfail && zone->uz_bucket_size < zone->uz_bucket_size_max) 703 zone->uz_bucket_size++; 704 return (zdom); 705 } 706 707 /* 708 * Search for the domain with the least cached items and return it if it 709 * is out of balance with the preferred domain. 710 */ 711 static __noinline int 712 zone_domain_lowest(uma_zone_t zone, int pref) 713 { 714 long least, nitems, prefitems; 715 int domain; 716 int i; 717 718 prefitems = least = LONG_MAX; 719 domain = 0; 720 for (i = 0; i < vm_ndomains; i++) { 721 nitems = ZDOM_GET(zone, i)->uzd_nitems; 722 if (nitems < least) { 723 domain = i; 724 least = nitems; 725 } 726 if (domain == pref) 727 prefitems = nitems; 728 } 729 if (prefitems < least * 2) 730 return (pref); 731 732 return (domain); 733 } 734 735 /* 736 * Search for the domain with the most cached items and return it or the 737 * preferred domain if it has enough to proceed. 738 */ 739 static __noinline int 740 zone_domain_highest(uma_zone_t zone, int pref) 741 { 742 long most, nitems; 743 int domain; 744 int i; 745 746 if (ZDOM_GET(zone, pref)->uzd_nitems > BUCKET_MAX) 747 return (pref); 748 749 most = 0; 750 domain = 0; 751 for (i = 0; i < vm_ndomains; i++) { 752 nitems = ZDOM_GET(zone, i)->uzd_nitems; 753 if (nitems > most) { 754 domain = i; 755 most = nitems; 756 } 757 } 758 759 return (domain); 760 } 761 762 /* 763 * Set the maximum imax value. 764 */ 765 static void 766 zone_domain_imax_set(uma_zone_domain_t zdom, int nitems) 767 { 768 long old; 769 770 old = zdom->uzd_imax; 771 do { 772 if (old >= nitems) 773 return; 774 } while (atomic_fcmpset_long(&zdom->uzd_imax, &old, nitems) == 0); 775 776 /* 777 * We are at new maximum, so do the last WSS update for the old 778 * bimin and prepare to measure next allocation batch. 779 */ 780 if (zdom->uzd_wss < old - zdom->uzd_bimin) 781 zdom->uzd_wss = old - zdom->uzd_bimin; 782 zdom->uzd_bimin = nitems; 783 } 784 785 /* 786 * Attempt to satisfy an allocation by retrieving a full bucket from one of the 787 * zone's caches. If a bucket is found the zone is not locked on return. 788 */ 789 static uma_bucket_t 790 zone_fetch_bucket(uma_zone_t zone, uma_zone_domain_t zdom, bool reclaim) 791 { 792 uma_bucket_t bucket; 793 long cnt; 794 int i; 795 bool dtor = false; 796 797 ZDOM_LOCK_ASSERT(zdom); 798 799 if ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) == NULL) 800 return (NULL); 801 802 /* SMR Buckets can not be re-used until readers expire. */ 803 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && 804 bucket->ub_seq != SMR_SEQ_INVALID) { 805 if (!smr_poll(zone->uz_smr, bucket->ub_seq, false)) 806 return (NULL); 807 bucket->ub_seq = SMR_SEQ_INVALID; 808 dtor = (zone->uz_dtor != NULL) || UMA_ALWAYS_CTORDTOR; 809 if (STAILQ_NEXT(bucket, ub_link) != NULL) 810 zdom->uzd_seq = STAILQ_NEXT(bucket, ub_link)->ub_seq; 811 } 812 STAILQ_REMOVE_HEAD(&zdom->uzd_buckets, ub_link); 813 814 KASSERT(zdom->uzd_nitems >= bucket->ub_cnt, 815 ("%s: item count underflow (%ld, %d)", 816 __func__, zdom->uzd_nitems, bucket->ub_cnt)); 817 KASSERT(bucket->ub_cnt > 0, 818 ("%s: empty bucket in bucket cache", __func__)); 819 zdom->uzd_nitems -= bucket->ub_cnt; 820 821 if (reclaim) { 822 /* 823 * Shift the bounds of the current WSS interval to avoid 824 * perturbing the estimates. 825 */ 826 cnt = lmin(zdom->uzd_bimin, bucket->ub_cnt); 827 atomic_subtract_long(&zdom->uzd_imax, cnt); 828 zdom->uzd_bimin -= cnt; 829 zdom->uzd_imin -= lmin(zdom->uzd_imin, bucket->ub_cnt); 830 if (zdom->uzd_limin >= bucket->ub_cnt) { 831 zdom->uzd_limin -= bucket->ub_cnt; 832 } else { 833 zdom->uzd_limin = 0; 834 zdom->uzd_timin = 0; 835 } 836 } else if (zdom->uzd_bimin > zdom->uzd_nitems) { 837 zdom->uzd_bimin = zdom->uzd_nitems; 838 if (zdom->uzd_imin > zdom->uzd_nitems) 839 zdom->uzd_imin = zdom->uzd_nitems; 840 } 841 842 ZDOM_UNLOCK(zdom); 843 if (dtor) 844 for (i = 0; i < bucket->ub_cnt; i++) 845 item_dtor(zone, bucket->ub_bucket[i], zone->uz_size, 846 NULL, SKIP_NONE); 847 848 return (bucket); 849 } 850 851 /* 852 * Insert a full bucket into the specified cache. The "ws" parameter indicates 853 * whether the bucket's contents should be counted as part of the zone's working 854 * set. The bucket may be freed if it exceeds the bucket limit. 855 */ 856 static void 857 zone_put_bucket(uma_zone_t zone, int domain, uma_bucket_t bucket, void *udata, 858 const bool ws) 859 { 860 uma_zone_domain_t zdom; 861 862 /* We don't cache empty buckets. This can happen after a reclaim. */ 863 if (bucket->ub_cnt == 0) 864 goto out; 865 zdom = zone_domain_lock(zone, domain); 866 867 /* 868 * Conditionally set the maximum number of items. 869 */ 870 zdom->uzd_nitems += bucket->ub_cnt; 871 if (__predict_true(zdom->uzd_nitems < zone->uz_bucket_max)) { 872 bool head; 873 874 if (ws) { 875 zone_domain_imax_set(zdom, zdom->uzd_nitems); 876 } else { 877 /* 878 * Shift the bounds of the current WSS interval to 879 * avoid perturbing the estimates. 880 */ 881 atomic_add_long(&zdom->uzd_imax, bucket->ub_cnt); 882 zdom->uzd_imin += bucket->ub_cnt; 883 zdom->uzd_bimin += bucket->ub_cnt; 884 zdom->uzd_limin += bucket->ub_cnt; 885 } 886 if (STAILQ_EMPTY(&zdom->uzd_buckets)) 887 zdom->uzd_seq = bucket->ub_seq; 888 889 /* 890 * Try to promote reuse of recently used items. For items 891 * protected by SMR, try to defer reuse to minimize polling. 892 * If KASAN is configured, try to defer reuse to improve UAF 893 * detection. 894 */ 895 head = bucket->ub_seq == SMR_SEQ_INVALID; 896 #ifdef KASAN 897 head = head && (zone->uz_flags & UMA_ZONE_NOKASAN) != 0; 898 #endif 899 if (head) 900 STAILQ_INSERT_HEAD(&zdom->uzd_buckets, bucket, ub_link); 901 else 902 STAILQ_INSERT_TAIL(&zdom->uzd_buckets, bucket, ub_link); 903 ZDOM_UNLOCK(zdom); 904 return; 905 } 906 zdom->uzd_nitems -= bucket->ub_cnt; 907 ZDOM_UNLOCK(zdom); 908 out: 909 bucket_free(zone, bucket, udata); 910 } 911 912 /* Pops an item out of a per-cpu cache bucket. */ 913 static inline void * 914 cache_bucket_pop(uma_cache_t cache, uma_cache_bucket_t bucket) 915 { 916 void *item; 917 918 CRITICAL_ASSERT(curthread); 919 920 bucket->ucb_cnt--; 921 item = bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt]; 922 #ifdef INVARIANTS 923 bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = NULL; 924 KASSERT(item != NULL, ("uma_zalloc: Bucket pointer mangled.")); 925 #endif 926 cache->uc_allocs++; 927 928 return (item); 929 } 930 931 /* Pushes an item into a per-cpu cache bucket. */ 932 static inline void 933 cache_bucket_push(uma_cache_t cache, uma_cache_bucket_t bucket, void *item) 934 { 935 936 CRITICAL_ASSERT(curthread); 937 KASSERT(bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] == NULL, 938 ("uma_zfree: Freeing to non free bucket index.")); 939 940 bucket->ucb_bucket->ub_bucket[bucket->ucb_cnt] = item; 941 bucket->ucb_cnt++; 942 cache->uc_frees++; 943 } 944 945 /* 946 * Unload a UMA bucket from a per-cpu cache. 947 */ 948 static inline uma_bucket_t 949 cache_bucket_unload(uma_cache_bucket_t bucket) 950 { 951 uma_bucket_t b; 952 953 b = bucket->ucb_bucket; 954 if (b != NULL) { 955 MPASS(b->ub_entries == bucket->ucb_entries); 956 b->ub_cnt = bucket->ucb_cnt; 957 bucket->ucb_bucket = NULL; 958 bucket->ucb_entries = bucket->ucb_cnt = 0; 959 } 960 961 return (b); 962 } 963 964 static inline uma_bucket_t 965 cache_bucket_unload_alloc(uma_cache_t cache) 966 { 967 968 return (cache_bucket_unload(&cache->uc_allocbucket)); 969 } 970 971 static inline uma_bucket_t 972 cache_bucket_unload_free(uma_cache_t cache) 973 { 974 975 return (cache_bucket_unload(&cache->uc_freebucket)); 976 } 977 978 static inline uma_bucket_t 979 cache_bucket_unload_cross(uma_cache_t cache) 980 { 981 982 return (cache_bucket_unload(&cache->uc_crossbucket)); 983 } 984 985 /* 986 * Load a bucket into a per-cpu cache bucket. 987 */ 988 static inline void 989 cache_bucket_load(uma_cache_bucket_t bucket, uma_bucket_t b) 990 { 991 992 CRITICAL_ASSERT(curthread); 993 MPASS(bucket->ucb_bucket == NULL); 994 MPASS(b->ub_seq == SMR_SEQ_INVALID); 995 996 bucket->ucb_bucket = b; 997 bucket->ucb_cnt = b->ub_cnt; 998 bucket->ucb_entries = b->ub_entries; 999 } 1000 1001 static inline void 1002 cache_bucket_load_alloc(uma_cache_t cache, uma_bucket_t b) 1003 { 1004 1005 cache_bucket_load(&cache->uc_allocbucket, b); 1006 } 1007 1008 static inline void 1009 cache_bucket_load_free(uma_cache_t cache, uma_bucket_t b) 1010 { 1011 1012 cache_bucket_load(&cache->uc_freebucket, b); 1013 } 1014 1015 #ifdef NUMA 1016 static inline void 1017 cache_bucket_load_cross(uma_cache_t cache, uma_bucket_t b) 1018 { 1019 1020 cache_bucket_load(&cache->uc_crossbucket, b); 1021 } 1022 #endif 1023 1024 /* 1025 * Copy and preserve ucb_spare. 1026 */ 1027 static inline void 1028 cache_bucket_copy(uma_cache_bucket_t b1, uma_cache_bucket_t b2) 1029 { 1030 1031 b1->ucb_bucket = b2->ucb_bucket; 1032 b1->ucb_entries = b2->ucb_entries; 1033 b1->ucb_cnt = b2->ucb_cnt; 1034 } 1035 1036 /* 1037 * Swap two cache buckets. 1038 */ 1039 static inline void 1040 cache_bucket_swap(uma_cache_bucket_t b1, uma_cache_bucket_t b2) 1041 { 1042 struct uma_cache_bucket b3; 1043 1044 CRITICAL_ASSERT(curthread); 1045 1046 cache_bucket_copy(&b3, b1); 1047 cache_bucket_copy(b1, b2); 1048 cache_bucket_copy(b2, &b3); 1049 } 1050 1051 /* 1052 * Attempt to fetch a bucket from a zone on behalf of the current cpu cache. 1053 */ 1054 static uma_bucket_t 1055 cache_fetch_bucket(uma_zone_t zone, uma_cache_t cache, int domain) 1056 { 1057 uma_zone_domain_t zdom; 1058 uma_bucket_t bucket; 1059 smr_seq_t seq; 1060 1061 /* 1062 * Avoid the lock if possible. 1063 */ 1064 zdom = ZDOM_GET(zone, domain); 1065 if (zdom->uzd_nitems == 0) 1066 return (NULL); 1067 1068 if ((cache_uz_flags(cache) & UMA_ZONE_SMR) != 0 && 1069 (seq = atomic_load_32(&zdom->uzd_seq)) != SMR_SEQ_INVALID && 1070 !smr_poll(zone->uz_smr, seq, false)) 1071 return (NULL); 1072 1073 /* 1074 * Check the zone's cache of buckets. 1075 */ 1076 zdom = zone_domain_lock(zone, domain); 1077 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) 1078 return (bucket); 1079 ZDOM_UNLOCK(zdom); 1080 1081 return (NULL); 1082 } 1083 1084 static void 1085 zone_log_warning(uma_zone_t zone) 1086 { 1087 static const struct timeval warninterval = { 300, 0 }; 1088 1089 if (!zone_warnings || zone->uz_warning == NULL) 1090 return; 1091 1092 if (ratecheck(&zone->uz_ratecheck, &warninterval)) 1093 printf("[zone: %s] %s\n", zone->uz_name, zone->uz_warning); 1094 } 1095 1096 static inline void 1097 zone_maxaction(uma_zone_t zone) 1098 { 1099 1100 if (zone->uz_maxaction.ta_func != NULL) 1101 taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction); 1102 } 1103 1104 /* 1105 * Routine called by timeout which is used to fire off some time interval 1106 * based calculations. (stats, hash size, etc.) 1107 * 1108 * Arguments: 1109 * arg Unused 1110 * 1111 * Returns: 1112 * Nothing 1113 */ 1114 static void 1115 uma_timeout(void *context __unused, int pending __unused) 1116 { 1117 bucket_enable(); 1118 zone_foreach(zone_timeout, NULL); 1119 1120 /* Reschedule this event */ 1121 taskqueue_enqueue_timeout(taskqueue_thread, &uma_timeout_task, 1122 UMA_TIMEOUT * hz); 1123 } 1124 1125 /* 1126 * Update the working set size estimates for the zone's bucket cache. 1127 * The constants chosen here are somewhat arbitrary. 1128 */ 1129 static void 1130 zone_domain_update_wss(uma_zone_domain_t zdom) 1131 { 1132 long m; 1133 1134 ZDOM_LOCK_ASSERT(zdom); 1135 MPASS(zdom->uzd_imax >= zdom->uzd_nitems); 1136 MPASS(zdom->uzd_nitems >= zdom->uzd_bimin); 1137 MPASS(zdom->uzd_bimin >= zdom->uzd_imin); 1138 1139 /* 1140 * Estimate WSS as modified moving average of biggest allocation 1141 * batches for each period over few minutes (UMA_TIMEOUT of 20s). 1142 */ 1143 zdom->uzd_wss = lmax(zdom->uzd_wss * 3 / 4, 1144 zdom->uzd_imax - zdom->uzd_bimin); 1145 1146 /* 1147 * Estimate longtime minimum item count as a combination of recent 1148 * minimum item count, adjusted by WSS for safety, and the modified 1149 * moving average over the last several hours (UMA_TIMEOUT of 20s). 1150 * timin measures time since limin tried to go negative, that means 1151 * we were dangerously close to or got out of cache. 1152 */ 1153 m = zdom->uzd_imin - zdom->uzd_wss; 1154 if (m >= 0) { 1155 if (zdom->uzd_limin >= m) 1156 zdom->uzd_limin = m; 1157 else 1158 zdom->uzd_limin = (m + zdom->uzd_limin * 255) / 256; 1159 zdom->uzd_timin++; 1160 } else { 1161 zdom->uzd_limin = 0; 1162 zdom->uzd_timin = 0; 1163 } 1164 1165 /* To reduce period edge effects on WSS keep half of the imax. */ 1166 atomic_subtract_long(&zdom->uzd_imax, 1167 (zdom->uzd_imax - zdom->uzd_nitems + 1) / 2); 1168 zdom->uzd_imin = zdom->uzd_bimin = zdom->uzd_nitems; 1169 } 1170 1171 /* 1172 * Routine to perform timeout driven calculations. This expands the 1173 * hashes and does per cpu statistics aggregation. 1174 * 1175 * Returns nothing. 1176 */ 1177 static void 1178 zone_timeout(uma_zone_t zone, void *unused) 1179 { 1180 uma_keg_t keg; 1181 u_int slabs, pages; 1182 1183 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) 1184 goto trim; 1185 1186 keg = zone->uz_keg; 1187 1188 /* 1189 * Hash zones are non-numa by definition so the first domain 1190 * is the only one present. 1191 */ 1192 KEG_LOCK(keg, 0); 1193 pages = keg->uk_domain[0].ud_pages; 1194 1195 /* 1196 * Expand the keg hash table. 1197 * 1198 * This is done if the number of slabs is larger than the hash size. 1199 * What I'm trying to do here is completely reduce collisions. This 1200 * may be a little aggressive. Should I allow for two collisions max? 1201 */ 1202 if ((slabs = pages / keg->uk_ppera) > keg->uk_hash.uh_hashsize) { 1203 struct uma_hash newhash; 1204 struct uma_hash oldhash; 1205 int ret; 1206 1207 /* 1208 * This is so involved because allocating and freeing 1209 * while the keg lock is held will lead to deadlock. 1210 * I have to do everything in stages and check for 1211 * races. 1212 */ 1213 KEG_UNLOCK(keg, 0); 1214 ret = hash_alloc(&newhash, 1 << fls(slabs)); 1215 KEG_LOCK(keg, 0); 1216 if (ret) { 1217 if (hash_expand(&keg->uk_hash, &newhash)) { 1218 oldhash = keg->uk_hash; 1219 keg->uk_hash = newhash; 1220 } else 1221 oldhash = newhash; 1222 1223 KEG_UNLOCK(keg, 0); 1224 hash_free(&oldhash); 1225 goto trim; 1226 } 1227 } 1228 KEG_UNLOCK(keg, 0); 1229 1230 trim: 1231 /* Trim caches not used for a long time. */ 1232 if ((zone->uz_flags & (UMA_ZONE_UNMANAGED | UMA_ZONE_NOTRIM)) == 0) { 1233 for (int i = 0; i < vm_ndomains; i++) { 1234 if (bucket_cache_reclaim_domain(zone, false, false, i) && 1235 (zone->uz_flags & UMA_ZFLAG_CACHE) == 0) 1236 keg_drain(zone->uz_keg, i); 1237 } 1238 } 1239 } 1240 1241 /* 1242 * Allocate and zero fill the next sized hash table from the appropriate 1243 * backing store. 1244 * 1245 * Arguments: 1246 * hash A new hash structure with the old hash size in uh_hashsize 1247 * 1248 * Returns: 1249 * 1 on success and 0 on failure. 1250 */ 1251 static int 1252 hash_alloc(struct uma_hash *hash, u_int size) 1253 { 1254 size_t alloc; 1255 1256 KASSERT(powerof2(size), ("hash size must be power of 2")); 1257 if (size > UMA_HASH_SIZE_INIT) { 1258 hash->uh_hashsize = size; 1259 alloc = sizeof(hash->uh_slab_hash[0]) * hash->uh_hashsize; 1260 hash->uh_slab_hash = malloc(alloc, M_UMAHASH, M_NOWAIT); 1261 } else { 1262 alloc = sizeof(hash->uh_slab_hash[0]) * UMA_HASH_SIZE_INIT; 1263 hash->uh_slab_hash = zone_alloc_item(hashzone, NULL, 1264 UMA_ANYDOMAIN, M_WAITOK); 1265 hash->uh_hashsize = UMA_HASH_SIZE_INIT; 1266 } 1267 if (hash->uh_slab_hash) { 1268 bzero(hash->uh_slab_hash, alloc); 1269 hash->uh_hashmask = hash->uh_hashsize - 1; 1270 return (1); 1271 } 1272 1273 return (0); 1274 } 1275 1276 /* 1277 * Expands the hash table for HASH zones. This is done from zone_timeout 1278 * to reduce collisions. This must not be done in the regular allocation 1279 * path, otherwise, we can recurse on the vm while allocating pages. 1280 * 1281 * Arguments: 1282 * oldhash The hash you want to expand 1283 * newhash The hash structure for the new table 1284 * 1285 * Returns: 1286 * Nothing 1287 * 1288 * Discussion: 1289 */ 1290 static int 1291 hash_expand(struct uma_hash *oldhash, struct uma_hash *newhash) 1292 { 1293 uma_hash_slab_t slab; 1294 u_int hval; 1295 u_int idx; 1296 1297 if (!newhash->uh_slab_hash) 1298 return (0); 1299 1300 if (oldhash->uh_hashsize >= newhash->uh_hashsize) 1301 return (0); 1302 1303 /* 1304 * I need to investigate hash algorithms for resizing without a 1305 * full rehash. 1306 */ 1307 1308 for (idx = 0; idx < oldhash->uh_hashsize; idx++) 1309 while (!LIST_EMPTY(&oldhash->uh_slab_hash[idx])) { 1310 slab = LIST_FIRST(&oldhash->uh_slab_hash[idx]); 1311 LIST_REMOVE(slab, uhs_hlink); 1312 hval = UMA_HASH(newhash, slab->uhs_data); 1313 LIST_INSERT_HEAD(&newhash->uh_slab_hash[hval], 1314 slab, uhs_hlink); 1315 } 1316 1317 return (1); 1318 } 1319 1320 /* 1321 * Free the hash bucket to the appropriate backing store. 1322 * 1323 * Arguments: 1324 * slab_hash The hash bucket we're freeing 1325 * hashsize The number of entries in that hash bucket 1326 * 1327 * Returns: 1328 * Nothing 1329 */ 1330 static void 1331 hash_free(struct uma_hash *hash) 1332 { 1333 if (hash->uh_slab_hash == NULL) 1334 return; 1335 if (hash->uh_hashsize == UMA_HASH_SIZE_INIT) 1336 zone_free_item(hashzone, hash->uh_slab_hash, NULL, SKIP_NONE); 1337 else 1338 free(hash->uh_slab_hash, M_UMAHASH); 1339 } 1340 1341 /* 1342 * Frees all outstanding items in a bucket 1343 * 1344 * Arguments: 1345 * zone The zone to free to, must be unlocked. 1346 * bucket The free/alloc bucket with items. 1347 * 1348 * Returns: 1349 * Nothing 1350 */ 1351 static void 1352 bucket_drain(uma_zone_t zone, uma_bucket_t bucket) 1353 { 1354 int i; 1355 1356 if (bucket->ub_cnt == 0) 1357 return; 1358 1359 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && 1360 bucket->ub_seq != SMR_SEQ_INVALID) { 1361 smr_wait(zone->uz_smr, bucket->ub_seq); 1362 bucket->ub_seq = SMR_SEQ_INVALID; 1363 for (i = 0; i < bucket->ub_cnt; i++) 1364 item_dtor(zone, bucket->ub_bucket[i], 1365 zone->uz_size, NULL, SKIP_NONE); 1366 } 1367 if (zone->uz_fini) 1368 for (i = 0; i < bucket->ub_cnt; i++) { 1369 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); 1370 zone->uz_fini(bucket->ub_bucket[i], zone->uz_size); 1371 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); 1372 } 1373 zone->uz_release(zone->uz_arg, bucket->ub_bucket, bucket->ub_cnt); 1374 if (zone->uz_max_items > 0) 1375 zone_free_limit(zone, bucket->ub_cnt); 1376 #ifdef INVARIANTS 1377 bzero(bucket->ub_bucket, sizeof(void *) * bucket->ub_cnt); 1378 #endif 1379 bucket->ub_cnt = 0; 1380 } 1381 1382 /* 1383 * Drains the per cpu caches for a zone. 1384 * 1385 * NOTE: This may only be called while the zone is being torn down, and not 1386 * during normal operation. This is necessary in order that we do not have 1387 * to migrate CPUs to drain the per-CPU caches. 1388 * 1389 * Arguments: 1390 * zone The zone to drain, must be unlocked. 1391 * 1392 * Returns: 1393 * Nothing 1394 */ 1395 static void 1396 cache_drain(uma_zone_t zone) 1397 { 1398 uma_cache_t cache; 1399 uma_bucket_t bucket; 1400 smr_seq_t seq; 1401 int cpu; 1402 1403 /* 1404 * XXX: It is safe to not lock the per-CPU caches, because we're 1405 * tearing down the zone anyway. I.e., there will be no further use 1406 * of the caches at this point. 1407 * 1408 * XXX: It would good to be able to assert that the zone is being 1409 * torn down to prevent improper use of cache_drain(). 1410 */ 1411 seq = SMR_SEQ_INVALID; 1412 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 1413 seq = smr_advance(zone->uz_smr); 1414 CPU_FOREACH(cpu) { 1415 cache = &zone->uz_cpu[cpu]; 1416 bucket = cache_bucket_unload_alloc(cache); 1417 if (bucket != NULL) 1418 bucket_free(zone, bucket, NULL); 1419 bucket = cache_bucket_unload_free(cache); 1420 if (bucket != NULL) { 1421 bucket->ub_seq = seq; 1422 bucket_free(zone, bucket, NULL); 1423 } 1424 bucket = cache_bucket_unload_cross(cache); 1425 if (bucket != NULL) { 1426 bucket->ub_seq = seq; 1427 bucket_free(zone, bucket, NULL); 1428 } 1429 } 1430 bucket_cache_reclaim(zone, true, UMA_ANYDOMAIN); 1431 } 1432 1433 static void 1434 cache_shrink(uma_zone_t zone, void *unused) 1435 { 1436 1437 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 1438 return; 1439 1440 ZONE_LOCK(zone); 1441 zone->uz_bucket_size = 1442 (zone->uz_bucket_size_min + zone->uz_bucket_size) / 2; 1443 ZONE_UNLOCK(zone); 1444 } 1445 1446 static void 1447 cache_drain_safe_cpu(uma_zone_t zone, void *unused) 1448 { 1449 uma_cache_t cache; 1450 uma_bucket_t b1, b2, b3; 1451 int domain; 1452 1453 if (zone->uz_flags & UMA_ZFLAG_INTERNAL) 1454 return; 1455 1456 b1 = b2 = b3 = NULL; 1457 critical_enter(); 1458 cache = &zone->uz_cpu[curcpu]; 1459 domain = PCPU_GET(domain); 1460 b1 = cache_bucket_unload_alloc(cache); 1461 1462 /* 1463 * Don't flush SMR zone buckets. This leaves the zone without a 1464 * bucket and forces every free to synchronize(). 1465 */ 1466 if ((zone->uz_flags & UMA_ZONE_SMR) == 0) { 1467 b2 = cache_bucket_unload_free(cache); 1468 b3 = cache_bucket_unload_cross(cache); 1469 } 1470 critical_exit(); 1471 1472 if (b1 != NULL) 1473 zone_free_bucket(zone, b1, NULL, domain, false); 1474 if (b2 != NULL) 1475 zone_free_bucket(zone, b2, NULL, domain, false); 1476 if (b3 != NULL) { 1477 /* Adjust the domain so it goes to zone_free_cross. */ 1478 domain = (domain + 1) % vm_ndomains; 1479 zone_free_bucket(zone, b3, NULL, domain, false); 1480 } 1481 } 1482 1483 /* 1484 * Safely drain per-CPU caches of a zone(s) to alloc bucket. 1485 * This is an expensive call because it needs to bind to all CPUs 1486 * one by one and enter a critical section on each of them in order 1487 * to safely access their cache buckets. 1488 * Zone lock must not be held on call this function. 1489 */ 1490 static void 1491 pcpu_cache_drain_safe(uma_zone_t zone) 1492 { 1493 int cpu; 1494 1495 /* 1496 * Polite bucket sizes shrinking was not enough, shrink aggressively. 1497 */ 1498 if (zone) 1499 cache_shrink(zone, NULL); 1500 else 1501 zone_foreach(cache_shrink, NULL); 1502 1503 CPU_FOREACH(cpu) { 1504 thread_lock(curthread); 1505 sched_bind(curthread, cpu); 1506 thread_unlock(curthread); 1507 1508 if (zone) 1509 cache_drain_safe_cpu(zone, NULL); 1510 else 1511 zone_foreach(cache_drain_safe_cpu, NULL); 1512 } 1513 thread_lock(curthread); 1514 sched_unbind(curthread); 1515 thread_unlock(curthread); 1516 } 1517 1518 /* 1519 * Reclaim cached buckets from a zone. All buckets are reclaimed if the caller 1520 * requested a drain, otherwise the per-domain caches are trimmed to either 1521 * estimated working set size. 1522 */ 1523 static bool 1524 bucket_cache_reclaim_domain(uma_zone_t zone, bool drain, bool trim, int domain) 1525 { 1526 uma_zone_domain_t zdom; 1527 uma_bucket_t bucket; 1528 long target; 1529 bool done = false; 1530 1531 /* 1532 * The cross bucket is partially filled and not part of 1533 * the item count. Reclaim it individually here. 1534 */ 1535 zdom = ZDOM_GET(zone, domain); 1536 if ((zone->uz_flags & UMA_ZONE_SMR) == 0 || drain) { 1537 ZONE_CROSS_LOCK(zone); 1538 bucket = zdom->uzd_cross; 1539 zdom->uzd_cross = NULL; 1540 ZONE_CROSS_UNLOCK(zone); 1541 if (bucket != NULL) 1542 bucket_free(zone, bucket, NULL); 1543 } 1544 1545 /* 1546 * If we were asked to drain the zone, we are done only once 1547 * this bucket cache is empty. If trim, we reclaim items in 1548 * excess of the zone's estimated working set size. Multiple 1549 * consecutive calls will shrink the WSS and so reclaim more. 1550 * If neither drain nor trim, then voluntarily reclaim 1/4 1551 * (to reduce first spike) of items not used for a long time. 1552 */ 1553 ZDOM_LOCK(zdom); 1554 zone_domain_update_wss(zdom); 1555 if (drain) 1556 target = 0; 1557 else if (trim) 1558 target = zdom->uzd_wss; 1559 else if (zdom->uzd_timin > 900 / UMA_TIMEOUT) 1560 target = zdom->uzd_nitems - zdom->uzd_limin / 4; 1561 else { 1562 ZDOM_UNLOCK(zdom); 1563 return (done); 1564 } 1565 while ((bucket = STAILQ_FIRST(&zdom->uzd_buckets)) != NULL && 1566 zdom->uzd_nitems >= target + bucket->ub_cnt) { 1567 bucket = zone_fetch_bucket(zone, zdom, true); 1568 if (bucket == NULL) 1569 break; 1570 bucket_free(zone, bucket, NULL); 1571 done = true; 1572 ZDOM_LOCK(zdom); 1573 } 1574 ZDOM_UNLOCK(zdom); 1575 return (done); 1576 } 1577 1578 static void 1579 bucket_cache_reclaim(uma_zone_t zone, bool drain, int domain) 1580 { 1581 int i; 1582 1583 /* 1584 * Shrink the zone bucket size to ensure that the per-CPU caches 1585 * don't grow too large. 1586 */ 1587 if (zone->uz_bucket_size > zone->uz_bucket_size_min) 1588 zone->uz_bucket_size--; 1589 1590 if (domain != UMA_ANYDOMAIN && 1591 (zone->uz_flags & UMA_ZONE_ROUNDROBIN) == 0) { 1592 bucket_cache_reclaim_domain(zone, drain, true, domain); 1593 } else { 1594 for (i = 0; i < vm_ndomains; i++) 1595 bucket_cache_reclaim_domain(zone, drain, true, i); 1596 } 1597 } 1598 1599 static void 1600 keg_free_slab(uma_keg_t keg, uma_slab_t slab, int start) 1601 { 1602 uint8_t *mem; 1603 size_t size; 1604 int i; 1605 uint8_t flags; 1606 1607 CTR4(KTR_UMA, "keg_free_slab keg %s(%p) slab %p, returning %d bytes", 1608 keg->uk_name, keg, slab, PAGE_SIZE * keg->uk_ppera); 1609 1610 mem = slab_data(slab, keg); 1611 size = PAGE_SIZE * keg->uk_ppera; 1612 1613 kasan_mark_slab_valid(keg, mem); 1614 if (keg->uk_fini != NULL) { 1615 for (i = start - 1; i > -1; i--) 1616 #ifdef INVARIANTS 1617 /* 1618 * trash_fini implies that dtor was trash_dtor. trash_fini 1619 * would check that memory hasn't been modified since free, 1620 * which executed trash_dtor. 1621 * That's why we need to run uma_dbg_kskip() check here, 1622 * albeit we don't make skip check for other init/fini 1623 * invocations. 1624 */ 1625 if (!uma_dbg_kskip(keg, slab_item(slab, keg, i)) || 1626 keg->uk_fini != trash_fini) 1627 #endif 1628 keg->uk_fini(slab_item(slab, keg, i), keg->uk_size); 1629 } 1630 flags = slab->us_flags; 1631 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { 1632 zone_free_item(slabzone(keg->uk_ipers), slab_tohashslab(slab), 1633 NULL, SKIP_NONE); 1634 } 1635 keg->uk_freef(mem, size, flags); 1636 uma_total_dec(size); 1637 } 1638 1639 static void 1640 keg_drain_domain(uma_keg_t keg, int domain) 1641 { 1642 struct slabhead freeslabs; 1643 uma_domain_t dom; 1644 uma_slab_t slab, tmp; 1645 uint32_t i, stofree, stokeep, partial; 1646 1647 dom = &keg->uk_domain[domain]; 1648 LIST_INIT(&freeslabs); 1649 1650 CTR4(KTR_UMA, "keg_drain %s(%p) domain %d free items: %u", 1651 keg->uk_name, keg, domain, dom->ud_free_items); 1652 1653 KEG_LOCK(keg, domain); 1654 1655 /* 1656 * Are the free items in partially allocated slabs sufficient to meet 1657 * the reserve? If not, compute the number of fully free slabs that must 1658 * be kept. 1659 */ 1660 partial = dom->ud_free_items - dom->ud_free_slabs * keg->uk_ipers; 1661 if (partial < keg->uk_reserve) { 1662 stokeep = min(dom->ud_free_slabs, 1663 howmany(keg->uk_reserve - partial, keg->uk_ipers)); 1664 } else { 1665 stokeep = 0; 1666 } 1667 stofree = dom->ud_free_slabs - stokeep; 1668 1669 /* 1670 * Partition the free slabs into two sets: those that must be kept in 1671 * order to maintain the reserve, and those that may be released back to 1672 * the system. Since one set may be much larger than the other, 1673 * populate the smaller of the two sets and swap them if necessary. 1674 */ 1675 for (i = min(stofree, stokeep); i > 0; i--) { 1676 slab = LIST_FIRST(&dom->ud_free_slab); 1677 LIST_REMOVE(slab, us_link); 1678 LIST_INSERT_HEAD(&freeslabs, slab, us_link); 1679 } 1680 if (stofree > stokeep) 1681 LIST_SWAP(&freeslabs, &dom->ud_free_slab, uma_slab, us_link); 1682 1683 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) { 1684 LIST_FOREACH(slab, &freeslabs, us_link) 1685 UMA_HASH_REMOVE(&keg->uk_hash, slab); 1686 } 1687 dom->ud_free_items -= stofree * keg->uk_ipers; 1688 dom->ud_free_slabs -= stofree; 1689 dom->ud_pages -= stofree * keg->uk_ppera; 1690 KEG_UNLOCK(keg, domain); 1691 1692 LIST_FOREACH_SAFE(slab, &freeslabs, us_link, tmp) 1693 keg_free_slab(keg, slab, keg->uk_ipers); 1694 } 1695 1696 /* 1697 * Frees pages from a keg back to the system. This is done on demand from 1698 * the pageout daemon. 1699 * 1700 * Returns nothing. 1701 */ 1702 static void 1703 keg_drain(uma_keg_t keg, int domain) 1704 { 1705 int i; 1706 1707 if ((keg->uk_flags & UMA_ZONE_NOFREE) != 0) 1708 return; 1709 if (domain != UMA_ANYDOMAIN) { 1710 keg_drain_domain(keg, domain); 1711 } else { 1712 for (i = 0; i < vm_ndomains; i++) 1713 keg_drain_domain(keg, i); 1714 } 1715 } 1716 1717 static void 1718 zone_reclaim(uma_zone_t zone, int domain, int waitok, bool drain) 1719 { 1720 /* 1721 * Count active reclaim operations in order to interlock with 1722 * zone_dtor(), which removes the zone from global lists before 1723 * attempting to reclaim items itself. 1724 * 1725 * The zone may be destroyed while sleeping, so only zone_dtor() should 1726 * specify M_WAITOK. 1727 */ 1728 ZONE_LOCK(zone); 1729 if (waitok == M_WAITOK) { 1730 while (zone->uz_reclaimers > 0) 1731 msleep(zone, ZONE_LOCKPTR(zone), PVM, "zonedrain", 1); 1732 } 1733 zone->uz_reclaimers++; 1734 ZONE_UNLOCK(zone); 1735 bucket_cache_reclaim(zone, drain, domain); 1736 1737 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) 1738 keg_drain(zone->uz_keg, domain); 1739 ZONE_LOCK(zone); 1740 zone->uz_reclaimers--; 1741 if (zone->uz_reclaimers == 0) 1742 wakeup(zone); 1743 ZONE_UNLOCK(zone); 1744 } 1745 1746 /* 1747 * Allocate a new slab for a keg and inserts it into the partial slab list. 1748 * The keg should be unlocked on entry. If the allocation succeeds it will 1749 * be locked on return. 1750 * 1751 * Arguments: 1752 * flags Wait flags for the item initialization routine 1753 * aflags Wait flags for the slab allocation 1754 * 1755 * Returns: 1756 * The slab that was allocated or NULL if there is no memory and the 1757 * caller specified M_NOWAIT. 1758 */ 1759 static uma_slab_t 1760 keg_alloc_slab(uma_keg_t keg, uma_zone_t zone, int domain, int flags, 1761 int aflags) 1762 { 1763 uma_domain_t dom; 1764 uma_slab_t slab; 1765 unsigned long size; 1766 uint8_t *mem; 1767 uint8_t sflags; 1768 int i; 1769 1770 TSENTER(); 1771 1772 KASSERT(domain >= 0 && domain < vm_ndomains, 1773 ("keg_alloc_slab: domain %d out of range", domain)); 1774 1775 slab = NULL; 1776 mem = NULL; 1777 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) { 1778 uma_hash_slab_t hslab; 1779 hslab = zone_alloc_item(slabzone(keg->uk_ipers), NULL, 1780 domain, aflags); 1781 if (hslab == NULL) 1782 goto fail; 1783 slab = &hslab->uhs_slab; 1784 } 1785 1786 /* 1787 * This reproduces the old vm_zone behavior of zero filling pages the 1788 * first time they are added to a zone. 1789 * 1790 * Malloced items are zeroed in uma_zalloc. 1791 */ 1792 1793 if ((keg->uk_flags & UMA_ZONE_MALLOC) == 0) 1794 aflags |= M_ZERO; 1795 else 1796 aflags &= ~M_ZERO; 1797 1798 if (keg->uk_flags & UMA_ZONE_NODUMP) 1799 aflags |= M_NODUMP; 1800 1801 if (keg->uk_flags & UMA_ZONE_NOFREE) 1802 aflags |= M_NEVERFREED; 1803 1804 /* zone is passed for legacy reasons. */ 1805 size = keg->uk_ppera * PAGE_SIZE; 1806 mem = keg->uk_allocf(zone, size, domain, &sflags, aflags); 1807 if (mem == NULL) { 1808 if (keg->uk_flags & UMA_ZFLAG_OFFPAGE) 1809 zone_free_item(slabzone(keg->uk_ipers), 1810 slab_tohashslab(slab), NULL, SKIP_NONE); 1811 goto fail; 1812 } 1813 uma_total_inc(size); 1814 1815 /* For HASH zones all pages go to the same uma_domain. */ 1816 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) 1817 domain = 0; 1818 1819 kmsan_mark(mem, size, 1820 (aflags & M_ZERO) != 0 ? KMSAN_STATE_INITED : KMSAN_STATE_UNINIT); 1821 1822 /* Point the slab into the allocated memory */ 1823 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) 1824 slab = (uma_slab_t)(mem + keg->uk_pgoff); 1825 else 1826 slab_tohashslab(slab)->uhs_data = mem; 1827 1828 if (keg->uk_flags & UMA_ZFLAG_VTOSLAB) 1829 for (i = 0; i < keg->uk_ppera; i++) 1830 vsetzoneslab((vm_offset_t)mem + (i * PAGE_SIZE), 1831 zone, slab); 1832 1833 slab->us_freecount = keg->uk_ipers; 1834 slab->us_flags = sflags; 1835 slab->us_domain = domain; 1836 1837 BIT_FILL(keg->uk_ipers, &slab->us_free); 1838 #ifdef INVARIANTS 1839 BIT_ZERO(keg->uk_ipers, slab_dbg_bits(slab, keg)); 1840 #endif 1841 1842 if (keg->uk_init != NULL) { 1843 for (i = 0; i < keg->uk_ipers; i++) 1844 if (keg->uk_init(slab_item(slab, keg, i), 1845 keg->uk_size, flags) != 0) 1846 break; 1847 if (i != keg->uk_ipers) { 1848 keg_free_slab(keg, slab, i); 1849 goto fail; 1850 } 1851 } 1852 kasan_mark_slab_invalid(keg, mem); 1853 KEG_LOCK(keg, domain); 1854 1855 CTR3(KTR_UMA, "keg_alloc_slab: allocated slab %p for %s(%p)", 1856 slab, keg->uk_name, keg); 1857 1858 if (keg->uk_flags & UMA_ZFLAG_HASH) 1859 UMA_HASH_INSERT(&keg->uk_hash, slab, mem); 1860 1861 /* 1862 * If we got a slab here it's safe to mark it partially used 1863 * and return. We assume that the caller is going to remove 1864 * at least one item. 1865 */ 1866 dom = &keg->uk_domain[domain]; 1867 LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 1868 dom->ud_pages += keg->uk_ppera; 1869 dom->ud_free_items += keg->uk_ipers; 1870 1871 TSEXIT(); 1872 return (slab); 1873 1874 fail: 1875 return (NULL); 1876 } 1877 1878 /* 1879 * This function is intended to be used early on in place of page_alloc(). It 1880 * performs contiguous physical memory allocations and uses a bump allocator for 1881 * KVA, so is usable before the kernel map is initialized. 1882 */ 1883 static void * 1884 startup_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1885 int wait) 1886 { 1887 vm_paddr_t pa; 1888 vm_page_t m; 1889 int i, pages; 1890 1891 pages = howmany(bytes, PAGE_SIZE); 1892 KASSERT(pages > 0, ("%s can't reserve 0 pages", __func__)); 1893 1894 *pflag = UMA_SLAB_BOOT; 1895 m = vm_page_alloc_noobj_contig_domain(domain, malloc2vm_flags(wait) | 1896 VM_ALLOC_WIRED, pages, (vm_paddr_t)0, ~(vm_paddr_t)0, 1, 0, 1897 VM_MEMATTR_DEFAULT); 1898 if (m == NULL) 1899 return (NULL); 1900 1901 pa = VM_PAGE_TO_PHYS(m); 1902 for (i = 0; i < pages; i++, pa += PAGE_SIZE) { 1903 #if MINIDUMP_PAGE_TRACKING && MINIDUMP_STARTUP_PAGE_TRACKING 1904 if ((wait & M_NODUMP) == 0) 1905 dump_add_page(pa); 1906 #endif 1907 } 1908 1909 /* Allocate KVA and indirectly advance bootmem. */ 1910 return (pmap_map(&bootmem, m->phys_addr, 1911 m->phys_addr + (pages * PAGE_SIZE), VM_PROT_READ | VM_PROT_WRITE)); 1912 } 1913 1914 static void 1915 startup_free(void *mem, vm_size_t bytes) 1916 { 1917 vm_offset_t va; 1918 vm_page_t m; 1919 1920 va = (vm_offset_t)mem; 1921 m = PHYS_TO_VM_PAGE(pmap_kextract(va)); 1922 1923 /* 1924 * startup_alloc() returns direct-mapped slabs on some platforms. Avoid 1925 * unmapping ranges of the direct map. 1926 */ 1927 if (va >= bootstart && va + bytes <= bootmem) 1928 pmap_remove(kernel_pmap, va, va + bytes); 1929 for (; bytes != 0; bytes -= PAGE_SIZE, m++) { 1930 #if MINIDUMP_PAGE_TRACKING && MINIDUMP_STARTUP_PAGE_TRACKING 1931 dump_drop_page(VM_PAGE_TO_PHYS(m)); 1932 #endif 1933 vm_page_unwire_noq(m); 1934 vm_page_free(m); 1935 } 1936 } 1937 1938 /* 1939 * Allocates a number of pages from the system 1940 * 1941 * Arguments: 1942 * bytes The number of bytes requested 1943 * wait Shall we wait? 1944 * 1945 * Returns: 1946 * A pointer to the alloced memory or possibly 1947 * NULL if M_NOWAIT is set. 1948 */ 1949 static void * 1950 page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1951 int wait) 1952 { 1953 void *p; /* Returned page */ 1954 1955 *pflag = UMA_SLAB_KERNEL; 1956 p = kmem_malloc_domainset(DOMAINSET_FIXED(domain), bytes, wait); 1957 1958 return (p); 1959 } 1960 1961 static void * 1962 pcpu_page_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 1963 int wait) 1964 { 1965 struct pglist alloctail; 1966 void *addr; 1967 char *zkva; 1968 int cpu, flags; 1969 vm_page_t p, p_next; 1970 #ifdef NUMA 1971 struct pcpu *pc; 1972 #endif 1973 1974 MPASS(bytes == (mp_maxid + 1) * PAGE_SIZE); 1975 1976 TAILQ_INIT(&alloctail); 1977 flags = VM_ALLOC_SYSTEM | VM_ALLOC_WIRED | malloc2vm_flags(wait); 1978 *pflag = UMA_SLAB_KERNEL; 1979 for (cpu = 0; cpu <= mp_maxid; cpu++) { 1980 if (CPU_ABSENT(cpu)) { 1981 p = vm_page_alloc_noobj(flags); 1982 } else { 1983 #ifndef NUMA 1984 p = vm_page_alloc_noobj(flags); 1985 #else 1986 pc = pcpu_find(cpu); 1987 if (__predict_false(VM_DOMAIN_EMPTY(pc->pc_domain))) 1988 p = NULL; 1989 else 1990 p = vm_page_alloc_noobj_domain(pc->pc_domain, 1991 flags); 1992 if (__predict_false(p == NULL)) 1993 p = vm_page_alloc_noobj(flags); 1994 #endif 1995 } 1996 if (__predict_false(p == NULL)) 1997 goto fail; 1998 TAILQ_INSERT_TAIL(&alloctail, p, plinks.q); 1999 } 2000 if ((addr = kva_alloc(bytes)) == NULL) 2001 goto fail; 2002 zkva = addr; 2003 TAILQ_FOREACH(p, &alloctail, plinks.q) { 2004 pmap_qenter(zkva, &p, 1); 2005 zkva += PAGE_SIZE; 2006 } 2007 return (addr); 2008 fail: 2009 TAILQ_FOREACH_SAFE(p, &alloctail, plinks.q, p_next) { 2010 vm_page_unwire_noq(p); 2011 vm_page_free(p); 2012 } 2013 return (NULL); 2014 } 2015 2016 /* 2017 * Allocates a number of pages not belonging to a VM object 2018 * 2019 * Arguments: 2020 * bytes The number of bytes requested 2021 * wait Shall we wait? 2022 * 2023 * Returns: 2024 * A pointer to the alloced memory or possibly 2025 * NULL if M_NOWAIT is set. 2026 */ 2027 static void * 2028 noobj_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, 2029 int wait) 2030 { 2031 TAILQ_HEAD(, vm_page) alloctail; 2032 u_long npages; 2033 void *retkva; 2034 char *zkva; 2035 vm_page_t p, p_next; 2036 uma_keg_t keg; 2037 int req; 2038 2039 TAILQ_INIT(&alloctail); 2040 keg = zone->uz_keg; 2041 req = VM_ALLOC_INTERRUPT | VM_ALLOC_WIRED; 2042 if ((wait & M_WAITOK) != 0) 2043 req |= VM_ALLOC_WAITOK; 2044 2045 npages = howmany(bytes, PAGE_SIZE); 2046 while (npages > 0) { 2047 p = vm_page_alloc_noobj_domain(domain, req); 2048 if (p != NULL) { 2049 TAILQ_INSERT_TAIL(&alloctail, p, plinks.q); 2050 npages--; 2051 continue; 2052 } 2053 /* 2054 * Page allocation failed, free intermediate pages and 2055 * exit. 2056 */ 2057 TAILQ_FOREACH_SAFE(p, &alloctail, plinks.q, p_next) { 2058 vm_page_unwire_noq(p); 2059 vm_page_free(p); 2060 } 2061 return (NULL); 2062 } 2063 *flags = UMA_SLAB_PRIV; 2064 zkva = (char *)keg->uk_kva + 2065 atomic_fetchadd_long(&keg->uk_offset, round_page(bytes)); 2066 retkva = zkva; 2067 TAILQ_FOREACH(p, &alloctail, plinks.q) { 2068 pmap_qenter(zkva, &p, 1); 2069 zkva += PAGE_SIZE; 2070 } 2071 2072 return (retkva); 2073 } 2074 2075 /* 2076 * Allocate physically contiguous pages. 2077 */ 2078 static void * 2079 contig_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *pflag, 2080 int wait) 2081 { 2082 2083 *pflag = UMA_SLAB_KERNEL; 2084 return ((void *)kmem_alloc_contig_domainset(DOMAINSET_FIXED(domain), 2085 bytes, wait, 0, ~(vm_paddr_t)0, 1, 0, VM_MEMATTR_DEFAULT)); 2086 } 2087 2088 #if defined(UMA_USE_DMAP) && !defined(UMA_MD_SMALL_ALLOC) 2089 void * 2090 uma_small_alloc(uma_zone_t zone, vm_size_t bytes, int domain, uint8_t *flags, 2091 int wait) 2092 { 2093 vm_page_t m; 2094 2095 *flags = UMA_SLAB_PRIV; 2096 m = vm_page_alloc_noobj_domain(domain, 2097 malloc2vm_flags(wait) | VM_ALLOC_WIRED); 2098 if (m == NULL) 2099 return (NULL); 2100 if ((wait & M_NODUMP) == 0) 2101 dump_add_page(VM_PAGE_TO_PHYS(m)); 2102 return (VM_PAGE_TO_DMAP(m)); 2103 } 2104 #endif 2105 2106 /* 2107 * Frees a number of pages to the system 2108 * 2109 * Arguments: 2110 * mem A pointer to the memory to be freed 2111 * size The size of the memory being freed 2112 * flags The original p->us_flags field 2113 * 2114 * Returns: 2115 * Nothing 2116 */ 2117 static void 2118 page_free(void *mem, vm_size_t size, uint8_t flags) 2119 { 2120 2121 if ((flags & UMA_SLAB_BOOT) != 0) { 2122 startup_free(mem, size); 2123 return; 2124 } 2125 2126 KASSERT((flags & UMA_SLAB_KERNEL) != 0, 2127 ("UMA: page_free used with invalid flags %x", flags)); 2128 2129 kmem_free(mem, size); 2130 } 2131 2132 /* 2133 * Frees pcpu zone allocations 2134 * 2135 * Arguments: 2136 * mem A pointer to the memory to be freed 2137 * size The size of the memory being freed 2138 * flags The original p->us_flags field 2139 * 2140 * Returns: 2141 * Nothing 2142 */ 2143 static void 2144 pcpu_page_free(void *mem, vm_size_t size, uint8_t flags) 2145 { 2146 vm_offset_t sva, curva; 2147 vm_paddr_t paddr; 2148 vm_page_t m; 2149 2150 MPASS(size == (mp_maxid+1)*PAGE_SIZE); 2151 2152 if ((flags & UMA_SLAB_BOOT) != 0) { 2153 startup_free(mem, size); 2154 return; 2155 } 2156 2157 sva = (vm_offset_t)mem; 2158 for (curva = sva; curva < sva + size; curva += PAGE_SIZE) { 2159 paddr = pmap_kextract(curva); 2160 m = PHYS_TO_VM_PAGE(paddr); 2161 vm_page_unwire_noq(m); 2162 vm_page_free(m); 2163 } 2164 pmap_qremove(mem, size >> PAGE_SHIFT); 2165 kva_free(mem, size); 2166 } 2167 2168 #if defined(UMA_USE_DMAP) && !defined(UMA_MD_SMALL_ALLOC) 2169 void 2170 uma_small_free(void *mem, vm_size_t size, uint8_t flags) 2171 { 2172 vm_page_t m; 2173 vm_paddr_t pa; 2174 2175 pa = DMAP_TO_PHYS(mem); 2176 dump_drop_page(pa); 2177 m = PHYS_TO_VM_PAGE(pa); 2178 vm_page_unwire_noq(m); 2179 vm_page_free(m); 2180 } 2181 #endif 2182 2183 /* 2184 * Zero fill initializer 2185 * 2186 * Arguments/Returns follow uma_init specifications 2187 */ 2188 static int 2189 zero_init(void *mem, int size, int flags) 2190 { 2191 bzero(mem, size); 2192 return (0); 2193 } 2194 2195 #ifdef INVARIANTS 2196 static struct noslabbits * 2197 slab_dbg_bits(uma_slab_t slab, uma_keg_t keg) 2198 { 2199 2200 return ((void *)((char *)&slab->us_free + BITSET_SIZE(keg->uk_ipers))); 2201 } 2202 #endif 2203 2204 /* 2205 * Actual size of embedded struct slab (!OFFPAGE). 2206 */ 2207 static size_t 2208 slab_sizeof(int nitems) 2209 { 2210 size_t s; 2211 2212 s = sizeof(struct uma_slab) + BITSET_SIZE(nitems) * SLAB_BITSETS; 2213 return (roundup(s, UMA_ALIGN_PTR + 1)); 2214 } 2215 2216 #define UMA_FIXPT_SHIFT 31 2217 #define UMA_FRAC_FIXPT(n, d) \ 2218 ((uint32_t)(((uint64_t)(n) << UMA_FIXPT_SHIFT) / (d))) 2219 #define UMA_FIXPT_PCT(f) \ 2220 ((u_int)(((uint64_t)100 * (f)) >> UMA_FIXPT_SHIFT)) 2221 #define UMA_PCT_FIXPT(pct) UMA_FRAC_FIXPT((pct), 100) 2222 #define UMA_MIN_EFF UMA_PCT_FIXPT(100 - UMA_MAX_WASTE) 2223 2224 /* 2225 * Compute the number of items that will fit in a slab. If hdr is true, the 2226 * item count may be limited to provide space in the slab for an inline slab 2227 * header. Otherwise, all slab space will be provided for item storage. 2228 */ 2229 static u_int 2230 slab_ipers_hdr(u_int size, u_int rsize, u_int slabsize, bool hdr) 2231 { 2232 u_int ipers; 2233 u_int padpi; 2234 2235 /* The padding between items is not needed after the last item. */ 2236 padpi = rsize - size; 2237 2238 if (hdr) { 2239 /* 2240 * Start with the maximum item count and remove items until 2241 * the slab header first alongside the allocatable memory. 2242 */ 2243 for (ipers = MIN(SLAB_MAX_SETSIZE, 2244 (slabsize + padpi - slab_sizeof(1)) / rsize); 2245 ipers > 0 && 2246 ipers * rsize - padpi + slab_sizeof(ipers) > slabsize; 2247 ipers--) 2248 continue; 2249 } else { 2250 ipers = MIN((slabsize + padpi) / rsize, SLAB_MAX_SETSIZE); 2251 } 2252 2253 return (ipers); 2254 } 2255 2256 struct keg_layout_result { 2257 u_int format; 2258 u_int slabsize; 2259 u_int ipers; 2260 u_int eff; 2261 }; 2262 2263 static void 2264 keg_layout_one(uma_keg_t keg, u_int rsize, u_int slabsize, u_int fmt, 2265 struct keg_layout_result *kl) 2266 { 2267 u_int total; 2268 2269 kl->format = fmt; 2270 kl->slabsize = slabsize; 2271 2272 /* Handle INTERNAL as inline with an extra page. */ 2273 if ((fmt & UMA_ZFLAG_INTERNAL) != 0) { 2274 kl->format &= ~UMA_ZFLAG_INTERNAL; 2275 kl->slabsize += PAGE_SIZE; 2276 } 2277 2278 kl->ipers = slab_ipers_hdr(keg->uk_size, rsize, kl->slabsize, 2279 (fmt & UMA_ZFLAG_OFFPAGE) == 0); 2280 2281 /* Account for memory used by an offpage slab header. */ 2282 total = kl->slabsize; 2283 if ((fmt & UMA_ZFLAG_OFFPAGE) != 0) 2284 total += slabzone(kl->ipers)->uz_keg->uk_rsize; 2285 2286 kl->eff = UMA_FRAC_FIXPT(kl->ipers * rsize, total); 2287 } 2288 2289 /* 2290 * Determine the format of a uma keg. This determines where the slab header 2291 * will be placed (inline or offpage) and calculates ipers, rsize, and ppera. 2292 * 2293 * Arguments 2294 * keg The zone we should initialize 2295 * 2296 * Returns 2297 * Nothing 2298 */ 2299 static void 2300 keg_layout(uma_keg_t keg) 2301 { 2302 struct keg_layout_result kl = {}, kl_tmp; 2303 u_int fmts[2]; 2304 u_int alignsize; 2305 u_int nfmt; 2306 u_int pages; 2307 u_int rsize; 2308 u_int slabsize; 2309 u_int i, j; 2310 2311 KASSERT((keg->uk_flags & UMA_ZONE_PCPU) == 0 || 2312 (keg->uk_size <= UMA_PCPU_ALLOC_SIZE && 2313 (keg->uk_flags & UMA_ZONE_CACHESPREAD) == 0), 2314 ("%s: cannot configure for PCPU: keg=%s, size=%u, flags=0x%b", 2315 __func__, keg->uk_name, keg->uk_size, keg->uk_flags, 2316 PRINT_UMA_ZFLAGS)); 2317 KASSERT((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) == 0 || 2318 (keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0, 2319 ("%s: incompatible flags 0x%b", __func__, keg->uk_flags, 2320 PRINT_UMA_ZFLAGS)); 2321 2322 alignsize = keg->uk_align + 1; 2323 #ifdef KASAN 2324 /* 2325 * ASAN requires that each allocation be aligned to the shadow map 2326 * scale factor. 2327 */ 2328 if (alignsize < KASAN_SHADOW_SCALE) 2329 alignsize = KASAN_SHADOW_SCALE; 2330 #endif 2331 2332 /* 2333 * Calculate the size of each allocation (rsize) according to 2334 * alignment. If the requested size is smaller than we have 2335 * allocation bits for we round it up. 2336 */ 2337 rsize = MAX(keg->uk_size, UMA_SMALLEST_UNIT); 2338 rsize = roundup2(rsize, alignsize); 2339 2340 if ((keg->uk_flags & UMA_ZONE_CACHESPREAD) != 0) { 2341 /* 2342 * We want one item to start on every align boundary in a page. 2343 * To do this we will span pages. We will also extend the item 2344 * by the size of align if it is an even multiple of align. 2345 * Otherwise, it would fall on the same boundary every time. 2346 */ 2347 if ((rsize & alignsize) == 0) 2348 rsize += alignsize; 2349 slabsize = rsize * (PAGE_SIZE / alignsize); 2350 slabsize = MIN(slabsize, rsize * SLAB_MAX_SETSIZE); 2351 slabsize = MIN(slabsize, UMA_CACHESPREAD_MAX_SIZE); 2352 slabsize = round_page(slabsize); 2353 } else { 2354 /* 2355 * Start with a slab size of as many pages as it takes to 2356 * represent a single item. We will try to fit as many 2357 * additional items into the slab as possible. 2358 */ 2359 slabsize = round_page(keg->uk_size); 2360 } 2361 2362 /* Build a list of all of the available formats for this keg. */ 2363 nfmt = 0; 2364 2365 /* Evaluate an inline slab layout. */ 2366 if ((keg->uk_flags & (UMA_ZONE_NOTOUCH | UMA_ZONE_PCPU)) == 0) 2367 fmts[nfmt++] = 0; 2368 2369 /* TODO: vm_page-embedded slab. */ 2370 2371 /* 2372 * We can't do OFFPAGE if we're internal or if we've been 2373 * asked to not go to the VM for buckets. If we do this we 2374 * may end up going to the VM for slabs which we do not want 2375 * to do if we're UMA_ZONE_VM, which clearly forbids it. 2376 * In those cases, evaluate a pseudo-format called INTERNAL 2377 * which has an inline slab header and one extra page to 2378 * guarantee that it fits. 2379 * 2380 * Otherwise, see if using an OFFPAGE slab will improve our 2381 * efficiency. 2382 */ 2383 if ((keg->uk_flags & (UMA_ZFLAG_INTERNAL | UMA_ZONE_VM)) != 0) 2384 fmts[nfmt++] = UMA_ZFLAG_INTERNAL; 2385 else 2386 fmts[nfmt++] = UMA_ZFLAG_OFFPAGE; 2387 2388 /* 2389 * Choose a slab size and format which satisfy the minimum efficiency. 2390 * Prefer the smallest slab size that meets the constraints. 2391 * 2392 * Start with a minimum slab size, to accommodate CACHESPREAD. Then, 2393 * for small items (up to PAGE_SIZE), the iteration increment is one 2394 * page; and for large items, the increment is one item. 2395 */ 2396 i = (slabsize + rsize - keg->uk_size) / MAX(PAGE_SIZE, rsize); 2397 KASSERT(i >= 1, ("keg %s(%p) flags=0x%b slabsize=%u, rsize=%u, i=%u", 2398 keg->uk_name, keg, keg->uk_flags, PRINT_UMA_ZFLAGS, slabsize, 2399 rsize, i)); 2400 for ( ; ; i++) { 2401 slabsize = (rsize <= PAGE_SIZE) ? ptoa(i) : 2402 round_page(rsize * (i - 1) + keg->uk_size); 2403 2404 for (j = 0; j < nfmt; j++) { 2405 /* Only if we have no viable format yet. */ 2406 if ((fmts[j] & UMA_ZFLAG_INTERNAL) != 0 && 2407 kl.ipers > 0) 2408 continue; 2409 2410 keg_layout_one(keg, rsize, slabsize, fmts[j], &kl_tmp); 2411 if (kl_tmp.eff <= kl.eff) 2412 continue; 2413 2414 kl = kl_tmp; 2415 2416 CTR6(KTR_UMA, "keg %s layout: format %#x " 2417 "(ipers %u * rsize %u) / slabsize %#x = %u%% eff", 2418 keg->uk_name, kl.format, kl.ipers, rsize, 2419 kl.slabsize, UMA_FIXPT_PCT(kl.eff)); 2420 2421 /* Stop when we reach the minimum efficiency. */ 2422 if (kl.eff >= UMA_MIN_EFF) 2423 break; 2424 } 2425 2426 if (kl.eff >= UMA_MIN_EFF || !multipage_slabs || 2427 slabsize >= SLAB_MAX_SETSIZE * rsize || 2428 (keg->uk_flags & (UMA_ZONE_PCPU | UMA_ZONE_CONTIG)) != 0) 2429 break; 2430 } 2431 2432 pages = atop(kl.slabsize); 2433 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) 2434 pages *= mp_maxid + 1; 2435 2436 keg->uk_rsize = rsize; 2437 keg->uk_ipers = kl.ipers; 2438 keg->uk_ppera = pages; 2439 keg->uk_flags |= kl.format; 2440 2441 /* 2442 * How do we find the slab header if it is offpage or if not all item 2443 * start addresses are in the same page? We could solve the latter 2444 * case with vaddr alignment, but we don't. 2445 */ 2446 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0 || 2447 (keg->uk_ipers - 1) * rsize >= PAGE_SIZE) { 2448 if ((keg->uk_flags & UMA_ZONE_NOTPAGE) != 0) 2449 keg->uk_flags |= UMA_ZFLAG_HASH; 2450 else 2451 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; 2452 } 2453 2454 CTR6(KTR_UMA, "%s: keg=%s, flags=%#x, rsize=%u, ipers=%u, ppera=%u", 2455 __func__, keg->uk_name, keg->uk_flags, rsize, keg->uk_ipers, 2456 pages); 2457 KASSERT(keg->uk_ipers > 0 && keg->uk_ipers <= SLAB_MAX_SETSIZE, 2458 ("%s: keg=%s, flags=0x%b, rsize=%u, ipers=%u, ppera=%u", __func__, 2459 keg->uk_name, keg->uk_flags, PRINT_UMA_ZFLAGS, rsize, 2460 keg->uk_ipers, pages)); 2461 } 2462 2463 /* 2464 * Keg header ctor. This initializes all fields, locks, etc. And inserts 2465 * the keg onto the global keg list. 2466 * 2467 * Arguments/Returns follow uma_ctor specifications 2468 * udata Actually uma_kctor_args 2469 */ 2470 static int 2471 keg_ctor(void *mem, int size, void *udata, int flags) 2472 { 2473 struct uma_kctor_args *arg = udata; 2474 uma_keg_t keg = mem; 2475 uma_zone_t zone; 2476 int i; 2477 2478 bzero(keg, size); 2479 keg->uk_size = arg->size; 2480 keg->uk_init = arg->uminit; 2481 keg->uk_fini = arg->fini; 2482 keg->uk_align = arg->align; 2483 keg->uk_reserve = 0; 2484 keg->uk_flags = arg->flags; 2485 2486 /* 2487 * We use a global round-robin policy by default. Zones with 2488 * UMA_ZONE_FIRSTTOUCH set will use first-touch instead, in which 2489 * case the iterator is never run. 2490 */ 2491 keg->uk_dr.dr_policy = DOMAINSET_RR(); 2492 keg->uk_dr.dr_iter = 0; 2493 2494 /* 2495 * The primary zone is passed to us at keg-creation time. 2496 */ 2497 zone = arg->zone; 2498 keg->uk_name = zone->uz_name; 2499 2500 if (arg->flags & UMA_ZONE_ZINIT) 2501 keg->uk_init = zero_init; 2502 2503 if (arg->flags & UMA_ZONE_MALLOC) 2504 keg->uk_flags |= UMA_ZFLAG_VTOSLAB; 2505 2506 #ifndef SMP 2507 keg->uk_flags &= ~UMA_ZONE_PCPU; 2508 #endif 2509 2510 keg_layout(keg); 2511 2512 /* 2513 * Use a first-touch NUMA policy for kegs that pmap_extract() will 2514 * work on. Use round-robin for everything else. 2515 * 2516 * Zones may override the default by specifying either. 2517 */ 2518 #ifdef NUMA 2519 if ((keg->uk_flags & 2520 (UMA_ZONE_ROUNDROBIN | UMA_ZFLAG_CACHE | UMA_ZONE_NOTPAGE)) == 0) 2521 keg->uk_flags |= UMA_ZONE_FIRSTTOUCH; 2522 else if ((keg->uk_flags & UMA_ZONE_FIRSTTOUCH) == 0) 2523 keg->uk_flags |= UMA_ZONE_ROUNDROBIN; 2524 #endif 2525 2526 /* 2527 * If we haven't booted yet we need allocations to go through the 2528 * startup cache until the vm is ready. 2529 */ 2530 #ifdef UMA_USE_DMAP 2531 if (keg->uk_ppera == 1) 2532 keg->uk_allocf = uma_small_alloc; 2533 else 2534 #endif 2535 if (booted < BOOT_KVA) 2536 keg->uk_allocf = startup_alloc; 2537 else if (keg->uk_flags & UMA_ZONE_PCPU) 2538 keg->uk_allocf = pcpu_page_alloc; 2539 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && keg->uk_ppera > 1) 2540 keg->uk_allocf = contig_alloc; 2541 else 2542 keg->uk_allocf = page_alloc; 2543 #ifdef UMA_USE_DMAP 2544 if (keg->uk_ppera == 1) 2545 keg->uk_freef = uma_small_free; 2546 else 2547 #endif 2548 if (keg->uk_flags & UMA_ZONE_PCPU) 2549 keg->uk_freef = pcpu_page_free; 2550 else 2551 keg->uk_freef = page_free; 2552 2553 /* 2554 * Initialize keg's locks. 2555 */ 2556 for (i = 0; i < vm_ndomains; i++) 2557 KEG_LOCK_INIT(keg, i, (arg->flags & UMA_ZONE_MTXCLASS)); 2558 2559 /* 2560 * If we're putting the slab header in the actual page we need to 2561 * figure out where in each page it goes. See slab_sizeof 2562 * definition. 2563 */ 2564 if (!(keg->uk_flags & UMA_ZFLAG_OFFPAGE)) { 2565 size_t shsize; 2566 2567 shsize = slab_sizeof(keg->uk_ipers); 2568 keg->uk_pgoff = (PAGE_SIZE * keg->uk_ppera) - shsize; 2569 /* 2570 * The only way the following is possible is if with our 2571 * UMA_ALIGN_PTR adjustments we are now bigger than 2572 * UMA_SLAB_SIZE. I haven't checked whether this is 2573 * mathematically possible for all cases, so we make 2574 * sure here anyway. 2575 */ 2576 KASSERT(keg->uk_pgoff + shsize <= PAGE_SIZE * keg->uk_ppera, 2577 ("zone %s ipers %d rsize %d size %d slab won't fit", 2578 zone->uz_name, keg->uk_ipers, keg->uk_rsize, keg->uk_size)); 2579 } 2580 2581 if (keg->uk_flags & UMA_ZFLAG_HASH) 2582 hash_alloc(&keg->uk_hash, 0); 2583 2584 CTR3(KTR_UMA, "keg_ctor %p zone %s(%p)", keg, zone->uz_name, zone); 2585 2586 LIST_INSERT_HEAD(&keg->uk_zones, zone, uz_link); 2587 2588 rw_wlock(&uma_rwlock); 2589 LIST_INSERT_HEAD(&uma_kegs, keg, uk_link); 2590 rw_wunlock(&uma_rwlock); 2591 return (0); 2592 } 2593 2594 static void 2595 zone_kva_available(uma_zone_t zone, void *unused) 2596 { 2597 uma_keg_t keg; 2598 2599 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 2600 return; 2601 KEG_GET(zone, keg); 2602 2603 if (keg->uk_allocf == startup_alloc) { 2604 /* Switch to the real allocator. */ 2605 if (keg->uk_flags & UMA_ZONE_PCPU) 2606 keg->uk_allocf = pcpu_page_alloc; 2607 else if ((keg->uk_flags & UMA_ZONE_CONTIG) != 0 && 2608 keg->uk_ppera > 1) 2609 keg->uk_allocf = contig_alloc; 2610 else 2611 keg->uk_allocf = page_alloc; 2612 } 2613 } 2614 2615 static void 2616 zone_alloc_counters(uma_zone_t zone, void *unused) 2617 { 2618 2619 zone->uz_allocs = counter_u64_alloc(M_WAITOK); 2620 zone->uz_frees = counter_u64_alloc(M_WAITOK); 2621 zone->uz_fails = counter_u64_alloc(M_WAITOK); 2622 zone->uz_xdomain = counter_u64_alloc(M_WAITOK); 2623 } 2624 2625 static void 2626 zone_alloc_sysctl(uma_zone_t zone, void *unused) 2627 { 2628 uma_zone_domain_t zdom; 2629 uma_domain_t dom; 2630 uma_keg_t keg; 2631 struct sysctl_oid *oid, *domainoid; 2632 int domains, i, cnt; 2633 static const char *nokeg = "cache zone"; 2634 char *c; 2635 2636 /* 2637 * Make a sysctl safe copy of the zone name by removing 2638 * any special characters and handling dups by appending 2639 * an index. 2640 */ 2641 if (zone->uz_namecnt != 0) { 2642 /* Count the number of decimal digits and '_' separator. */ 2643 for (i = 1, cnt = zone->uz_namecnt; cnt != 0; i++) 2644 cnt /= 10; 2645 zone->uz_ctlname = malloc(strlen(zone->uz_name) + i + 1, 2646 M_UMA, M_WAITOK); 2647 sprintf(zone->uz_ctlname, "%s_%d", zone->uz_name, 2648 zone->uz_namecnt); 2649 } else 2650 zone->uz_ctlname = strdup(zone->uz_name, M_UMA); 2651 for (c = zone->uz_ctlname; *c != '\0'; c++) 2652 if (strchr("./\\ -", *c) != NULL) 2653 *c = '_'; 2654 2655 /* 2656 * Basic parameters at the root. 2657 */ 2658 zone->uz_oid = SYSCTL_ADD_NODE(NULL, SYSCTL_STATIC_CHILDREN(_vm_uma), 2659 OID_AUTO, zone->uz_ctlname, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2660 oid = zone->uz_oid; 2661 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2662 "size", CTLFLAG_RD, &zone->uz_size, 0, "Allocation size"); 2663 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2664 "flags", CTLFLAG_RD | CTLTYPE_STRING | CTLFLAG_MPSAFE, 2665 zone, 0, sysctl_handle_uma_zone_flags, "A", 2666 "Allocator configuration flags"); 2667 SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2668 "bucket_size", CTLFLAG_RD, &zone->uz_bucket_size, 0, 2669 "Desired per-cpu cache size"); 2670 SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2671 "bucket_size_max", CTLFLAG_RD, &zone->uz_bucket_size_max, 0, 2672 "Maximum allowed per-cpu cache size"); 2673 2674 /* 2675 * keg if present. 2676 */ 2677 if ((zone->uz_flags & UMA_ZFLAG_HASH) == 0) 2678 domains = vm_ndomains; 2679 else 2680 domains = 1; 2681 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 2682 "keg", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2683 keg = zone->uz_keg; 2684 if ((zone->uz_flags & UMA_ZFLAG_CACHE) == 0) { 2685 SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2686 "name", CTLFLAG_RD, keg->uk_name, "Keg name"); 2687 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2688 "rsize", CTLFLAG_RD, &keg->uk_rsize, 0, 2689 "Real object size with alignment"); 2690 SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2691 "ppera", CTLFLAG_RD, &keg->uk_ppera, 0, 2692 "pages per-slab allocation"); 2693 SYSCTL_ADD_U16(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2694 "ipers", CTLFLAG_RD, &keg->uk_ipers, 0, 2695 "items available per-slab"); 2696 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2697 "align", CTLFLAG_RD, &keg->uk_align, 0, 2698 "item alignment mask"); 2699 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2700 "reserve", CTLFLAG_RD, &keg->uk_reserve, 0, 2701 "number of reserved items"); 2702 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2703 "efficiency", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, 2704 keg, 0, sysctl_handle_uma_slab_efficiency, "I", 2705 "Slab utilization (100 - internal fragmentation %)"); 2706 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(oid), 2707 OID_AUTO, "domain", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2708 for (i = 0; i < domains; i++) { 2709 dom = &keg->uk_domain[i]; 2710 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), 2711 OID_AUTO, VM_DOMAIN(i)->vmd_name, 2712 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2713 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2714 "pages", CTLFLAG_RD, &dom->ud_pages, 0, 2715 "Total pages currently allocated from VM"); 2716 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2717 "free_items", CTLFLAG_RD, &dom->ud_free_items, 0, 2718 "Items free in the slab layer"); 2719 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2720 "free_slabs", CTLFLAG_RD, &dom->ud_free_slabs, 0, 2721 "Unused slabs"); 2722 } 2723 } else 2724 SYSCTL_ADD_CONST_STRING(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2725 "name", CTLFLAG_RD, nokeg, "Keg name"); 2726 2727 /* 2728 * Information about zone limits. 2729 */ 2730 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 2731 "limit", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2732 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2733 "items", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 2734 zone, 0, sysctl_handle_uma_zone_items, "QU", 2735 "Current number of allocated items if limit is set"); 2736 SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2737 "max_items", CTLFLAG_RD, &zone->uz_max_items, 0, 2738 "Maximum number of allocated and cached items"); 2739 SYSCTL_ADD_U32(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2740 "sleepers", CTLFLAG_RD, &zone->uz_sleepers, 0, 2741 "Number of threads sleeping at limit"); 2742 SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2743 "sleeps", CTLFLAG_RD, &zone->uz_sleeps, 0, 2744 "Total zone limit sleeps"); 2745 SYSCTL_ADD_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2746 "bucket_max", CTLFLAG_RD, &zone->uz_bucket_max, 0, 2747 "Maximum number of items in each domain's bucket cache"); 2748 2749 /* 2750 * Per-domain zone information. 2751 */ 2752 domainoid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), 2753 OID_AUTO, "domain", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2754 for (i = 0; i < domains; i++) { 2755 zdom = ZDOM_GET(zone, i); 2756 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(domainoid), 2757 OID_AUTO, VM_DOMAIN(i)->vmd_name, 2758 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2759 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2760 "nitems", CTLFLAG_RD, &zdom->uzd_nitems, 2761 "number of items in this domain"); 2762 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2763 "imax", CTLFLAG_RD, &zdom->uzd_imax, 2764 "maximum item count in this period"); 2765 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2766 "imin", CTLFLAG_RD, &zdom->uzd_imin, 2767 "minimum item count in this period"); 2768 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2769 "bimin", CTLFLAG_RD, &zdom->uzd_bimin, 2770 "Minimum item count in this batch"); 2771 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2772 "wss", CTLFLAG_RD, &zdom->uzd_wss, 2773 "Working set size"); 2774 SYSCTL_ADD_LONG(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2775 "limin", CTLFLAG_RD, &zdom->uzd_limin, 2776 "Long time minimum item count"); 2777 SYSCTL_ADD_INT(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2778 "timin", CTLFLAG_RD, &zdom->uzd_timin, 0, 2779 "Time since zero long time minimum item count"); 2780 } 2781 2782 /* 2783 * General statistics. 2784 */ 2785 oid = SYSCTL_ADD_NODE(NULL, SYSCTL_CHILDREN(zone->uz_oid), OID_AUTO, 2786 "stats", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 2787 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2788 "current", CTLFLAG_RD | CTLTYPE_INT | CTLFLAG_MPSAFE, 2789 zone, 1, sysctl_handle_uma_zone_cur, "I", 2790 "Current number of allocated items"); 2791 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2792 "allocs", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 2793 zone, 0, sysctl_handle_uma_zone_allocs, "QU", 2794 "Total allocation calls"); 2795 SYSCTL_ADD_PROC(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2796 "frees", CTLFLAG_RD | CTLTYPE_U64 | CTLFLAG_MPSAFE, 2797 zone, 0, sysctl_handle_uma_zone_frees, "QU", 2798 "Total free calls"); 2799 SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2800 "fails", CTLFLAG_RD, &zone->uz_fails, 2801 "Number of allocation failures"); 2802 SYSCTL_ADD_COUNTER_U64(NULL, SYSCTL_CHILDREN(oid), OID_AUTO, 2803 "xdomain", CTLFLAG_RD, &zone->uz_xdomain, 2804 "Free calls from the wrong domain"); 2805 } 2806 2807 struct uma_zone_count { 2808 const char *name; 2809 int count; 2810 }; 2811 2812 static void 2813 zone_count(uma_zone_t zone, void *arg) 2814 { 2815 struct uma_zone_count *cnt; 2816 2817 cnt = arg; 2818 /* 2819 * Some zones are rapidly created with identical names and 2820 * destroyed out of order. This can lead to gaps in the count. 2821 * Use one greater than the maximum observed for this name. 2822 */ 2823 if (strcmp(zone->uz_name, cnt->name) == 0) 2824 cnt->count = MAX(cnt->count, 2825 zone->uz_namecnt + 1); 2826 } 2827 2828 static void 2829 zone_update_caches(uma_zone_t zone) 2830 { 2831 int i; 2832 2833 for (i = 0; i <= mp_maxid; i++) { 2834 cache_set_uz_size(&zone->uz_cpu[i], zone->uz_size); 2835 cache_set_uz_flags(&zone->uz_cpu[i], zone->uz_flags); 2836 } 2837 } 2838 2839 /* 2840 * Zone header ctor. This initializes all fields, locks, etc. 2841 * 2842 * Arguments/Returns follow uma_ctor specifications 2843 * udata Actually uma_zctor_args 2844 */ 2845 static int 2846 zone_ctor(void *mem, int size, void *udata, int flags) 2847 { 2848 struct uma_zone_count cnt; 2849 struct uma_zctor_args *arg = udata; 2850 uma_zone_domain_t zdom; 2851 uma_zone_t zone = mem; 2852 uma_zone_t z; 2853 uma_keg_t keg; 2854 int i; 2855 2856 bzero(zone, size); 2857 zone->uz_name = arg->name; 2858 zone->uz_ctor = arg->ctor; 2859 zone->uz_dtor = arg->dtor; 2860 zone->uz_init = NULL; 2861 zone->uz_fini = NULL; 2862 zone->uz_sleeps = 0; 2863 zone->uz_bucket_size = 0; 2864 zone->uz_bucket_size_min = 0; 2865 zone->uz_bucket_size_max = BUCKET_MAX; 2866 zone->uz_flags = (arg->flags & UMA_ZONE_SMR); 2867 zone->uz_warning = NULL; 2868 /* The domain structures follow the cpu structures. */ 2869 zone->uz_bucket_max = ULONG_MAX; 2870 timevalclear(&zone->uz_ratecheck); 2871 2872 /* Count the number of duplicate names. */ 2873 cnt.name = arg->name; 2874 cnt.count = 0; 2875 zone_foreach(zone_count, &cnt); 2876 zone->uz_namecnt = cnt.count; 2877 ZONE_CROSS_LOCK_INIT(zone); 2878 2879 for (i = 0; i < vm_ndomains; i++) { 2880 zdom = ZDOM_GET(zone, i); 2881 ZDOM_LOCK_INIT(zone, zdom, (arg->flags & UMA_ZONE_MTXCLASS)); 2882 STAILQ_INIT(&zdom->uzd_buckets); 2883 } 2884 2885 #if defined(INVARIANTS) && !defined(KASAN) && !defined(KMSAN) 2886 if (arg->uminit == trash_init && arg->fini == trash_fini) 2887 zone->uz_flags |= UMA_ZFLAG_TRASH | UMA_ZFLAG_CTORDTOR; 2888 #elif defined(KASAN) 2889 if ((arg->flags & (UMA_ZONE_NOFREE | UMA_ZFLAG_CACHE)) != 0) 2890 arg->flags |= UMA_ZONE_NOKASAN; 2891 #endif 2892 2893 /* 2894 * This is a pure cache zone, no kegs. 2895 */ 2896 if (arg->import) { 2897 KASSERT((arg->flags & UMA_ZFLAG_CACHE) != 0, 2898 ("zone_ctor: Import specified for non-cache zone.")); 2899 zone->uz_flags = arg->flags; 2900 zone->uz_size = arg->size; 2901 zone->uz_import = arg->import; 2902 zone->uz_release = arg->release; 2903 zone->uz_arg = arg->arg; 2904 #ifdef NUMA 2905 /* 2906 * Cache zones are round-robin unless a policy is 2907 * specified because they may have incompatible 2908 * constraints. 2909 */ 2910 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) == 0) 2911 zone->uz_flags |= UMA_ZONE_ROUNDROBIN; 2912 #endif 2913 rw_wlock(&uma_rwlock); 2914 LIST_INSERT_HEAD(&uma_cachezones, zone, uz_link); 2915 rw_wunlock(&uma_rwlock); 2916 goto out; 2917 } 2918 2919 /* 2920 * Use the regular zone/keg/slab allocator. 2921 */ 2922 zone->uz_import = zone_import; 2923 zone->uz_release = zone_release; 2924 zone->uz_arg = zone; 2925 keg = arg->keg; 2926 2927 if (arg->flags & UMA_ZONE_SECONDARY) { 2928 KASSERT((zone->uz_flags & UMA_ZONE_SECONDARY) == 0, 2929 ("Secondary zone requested UMA_ZFLAG_INTERNAL")); 2930 KASSERT(arg->keg != NULL, ("Secondary zone on zero'd keg")); 2931 zone->uz_init = arg->uminit; 2932 zone->uz_fini = arg->fini; 2933 zone->uz_flags |= UMA_ZONE_SECONDARY; 2934 rw_wlock(&uma_rwlock); 2935 ZONE_LOCK(zone); 2936 LIST_FOREACH(z, &keg->uk_zones, uz_link) { 2937 if (LIST_NEXT(z, uz_link) == NULL) { 2938 LIST_INSERT_AFTER(z, zone, uz_link); 2939 break; 2940 } 2941 } 2942 ZONE_UNLOCK(zone); 2943 rw_wunlock(&uma_rwlock); 2944 } else if (keg == NULL) { 2945 if ((keg = uma_kcreate(zone, arg->size, arg->uminit, arg->fini, 2946 arg->align, arg->flags)) == NULL) 2947 return (ENOMEM); 2948 } else { 2949 struct uma_kctor_args karg; 2950 int error; 2951 2952 /* We should only be here from uma_startup() */ 2953 karg.size = arg->size; 2954 karg.uminit = arg->uminit; 2955 karg.fini = arg->fini; 2956 karg.align = arg->align; 2957 karg.flags = (arg->flags & ~UMA_ZONE_SMR); 2958 karg.zone = zone; 2959 error = keg_ctor(arg->keg, sizeof(struct uma_keg), &karg, 2960 flags); 2961 if (error) 2962 return (error); 2963 } 2964 2965 /* Inherit properties from the keg. */ 2966 zone->uz_keg = keg; 2967 zone->uz_size = keg->uk_size; 2968 zone->uz_flags |= (keg->uk_flags & 2969 (UMA_ZONE_INHERIT | UMA_ZFLAG_INHERIT)); 2970 2971 out: 2972 if (booted >= BOOT_PCPU) { 2973 zone_alloc_counters(zone, NULL); 2974 if (booted >= BOOT_RUNNING) 2975 zone_alloc_sysctl(zone, NULL); 2976 } else { 2977 zone->uz_allocs = EARLY_COUNTER; 2978 zone->uz_frees = EARLY_COUNTER; 2979 zone->uz_fails = EARLY_COUNTER; 2980 } 2981 2982 /* Caller requests a private SMR context. */ 2983 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 2984 zone->uz_smr = smr_create(zone->uz_name, 0, 0); 2985 2986 KASSERT((arg->flags & (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET)) != 2987 (UMA_ZONE_MAXBUCKET | UMA_ZONE_NOBUCKET), 2988 ("Invalid zone flag combination")); 2989 if (arg->flags & UMA_ZFLAG_INTERNAL) 2990 zone->uz_bucket_size_max = zone->uz_bucket_size = 0; 2991 if ((arg->flags & UMA_ZONE_MAXBUCKET) != 0) 2992 zone->uz_bucket_size = BUCKET_MAX; 2993 else if ((arg->flags & UMA_ZONE_NOBUCKET) != 0) 2994 zone->uz_bucket_size = 0; 2995 else 2996 zone->uz_bucket_size = bucket_select(zone->uz_size); 2997 zone->uz_bucket_size_min = zone->uz_bucket_size; 2998 if (zone->uz_dtor != NULL || zone->uz_ctor != NULL) 2999 zone->uz_flags |= UMA_ZFLAG_CTORDTOR; 3000 zone_update_caches(zone); 3001 3002 return (0); 3003 } 3004 3005 /* 3006 * Keg header dtor. This frees all data, destroys locks, frees the hash 3007 * table and removes the keg from the global list. 3008 * 3009 * Arguments/Returns follow uma_dtor specifications 3010 * udata unused 3011 */ 3012 static void 3013 keg_dtor(void *arg, int size, void *udata) 3014 { 3015 uma_keg_t keg; 3016 uint32_t free, pages; 3017 int i; 3018 3019 keg = (uma_keg_t)arg; 3020 free = pages = 0; 3021 for (i = 0; i < vm_ndomains; i++) { 3022 free += keg->uk_domain[i].ud_free_items; 3023 pages += keg->uk_domain[i].ud_pages; 3024 KEG_LOCK_FINI(keg, i); 3025 } 3026 if (pages != 0) 3027 printf("Freed UMA keg (%s) was not empty (%u items). " 3028 " Lost %u pages of memory.\n", 3029 keg->uk_name ? keg->uk_name : "", 3030 pages / keg->uk_ppera * keg->uk_ipers - free, pages); 3031 3032 hash_free(&keg->uk_hash); 3033 } 3034 3035 /* 3036 * Zone header dtor. 3037 * 3038 * Arguments/Returns follow uma_dtor specifications 3039 * udata unused 3040 */ 3041 static void 3042 zone_dtor(void *arg, int size, void *udata) 3043 { 3044 uma_zone_t zone; 3045 uma_keg_t keg; 3046 int i; 3047 3048 zone = (uma_zone_t)arg; 3049 3050 sysctl_remove_oid(zone->uz_oid, 1, 1); 3051 3052 if (!(zone->uz_flags & UMA_ZFLAG_INTERNAL)) 3053 cache_drain(zone); 3054 3055 rw_wlock(&uma_rwlock); 3056 LIST_REMOVE(zone, uz_link); 3057 rw_wunlock(&uma_rwlock); 3058 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { 3059 keg = zone->uz_keg; 3060 keg->uk_reserve = 0; 3061 } 3062 zone_reclaim(zone, UMA_ANYDOMAIN, M_WAITOK, true); 3063 3064 /* 3065 * We only destroy kegs from non secondary/non cache zones. 3066 */ 3067 if ((zone->uz_flags & (UMA_ZONE_SECONDARY | UMA_ZFLAG_CACHE)) == 0) { 3068 keg = zone->uz_keg; 3069 rw_wlock(&uma_rwlock); 3070 LIST_REMOVE(keg, uk_link); 3071 rw_wunlock(&uma_rwlock); 3072 zone_free_item(kegs, keg, NULL, SKIP_NONE); 3073 } 3074 counter_u64_free(zone->uz_allocs); 3075 counter_u64_free(zone->uz_frees); 3076 counter_u64_free(zone->uz_fails); 3077 counter_u64_free(zone->uz_xdomain); 3078 free(zone->uz_ctlname, M_UMA); 3079 for (i = 0; i < vm_ndomains; i++) 3080 ZDOM_LOCK_FINI(ZDOM_GET(zone, i)); 3081 ZONE_CROSS_LOCK_FINI(zone); 3082 } 3083 3084 static void 3085 zone_foreach_unlocked(void (*zfunc)(uma_zone_t, void *arg), void *arg) 3086 { 3087 uma_keg_t keg; 3088 uma_zone_t zone; 3089 3090 LIST_FOREACH(keg, &uma_kegs, uk_link) { 3091 LIST_FOREACH(zone, &keg->uk_zones, uz_link) 3092 zfunc(zone, arg); 3093 } 3094 LIST_FOREACH(zone, &uma_cachezones, uz_link) 3095 zfunc(zone, arg); 3096 } 3097 3098 /* 3099 * Traverses every zone in the system and calls a callback 3100 * 3101 * Arguments: 3102 * zfunc A pointer to a function which accepts a zone 3103 * as an argument. 3104 * 3105 * Returns: 3106 * Nothing 3107 */ 3108 static void 3109 zone_foreach(void (*zfunc)(uma_zone_t, void *arg), void *arg) 3110 { 3111 3112 rw_rlock(&uma_rwlock); 3113 zone_foreach_unlocked(zfunc, arg); 3114 rw_runlock(&uma_rwlock); 3115 } 3116 3117 /* 3118 * Initialize the kernel memory allocator. This is done after pages can be 3119 * allocated but before general KVA is available. 3120 */ 3121 void 3122 uma_startup1(vm_offset_t virtual_avail) 3123 { 3124 struct uma_zctor_args args; 3125 size_t ksize, zsize, size; 3126 uma_keg_t primarykeg; 3127 uintptr_t m; 3128 int domain; 3129 uint8_t pflag; 3130 3131 bootstart = bootmem = virtual_avail; 3132 3133 rw_init(&uma_rwlock, "UMA lock"); 3134 sx_init(&uma_reclaim_lock, "umareclaim"); 3135 3136 ksize = sizeof(struct uma_keg) + 3137 (sizeof(struct uma_domain) * vm_ndomains); 3138 ksize = roundup(ksize, UMA_SUPER_ALIGN); 3139 zsize = sizeof(struct uma_zone) + 3140 (sizeof(struct uma_cache) * (mp_maxid + 1)) + 3141 (sizeof(struct uma_zone_domain) * vm_ndomains); 3142 zsize = roundup(zsize, UMA_SUPER_ALIGN); 3143 3144 /* Allocate the zone of zones, zone of kegs, and zone of zones keg. */ 3145 size = (zsize * 2) + ksize; 3146 for (domain = 0; domain < vm_ndomains; domain++) { 3147 m = (uintptr_t)startup_alloc(NULL, size, domain, &pflag, 3148 M_NOWAIT | M_ZERO); 3149 if (m != 0) 3150 break; 3151 } 3152 zones = (uma_zone_t)m; 3153 m += zsize; 3154 kegs = (uma_zone_t)m; 3155 m += zsize; 3156 primarykeg = (uma_keg_t)m; 3157 3158 /* "manually" create the initial zone */ 3159 memset(&args, 0, sizeof(args)); 3160 args.name = "UMA Kegs"; 3161 args.size = ksize; 3162 args.ctor = keg_ctor; 3163 args.dtor = keg_dtor; 3164 args.uminit = zero_init; 3165 args.fini = NULL; 3166 args.keg = primarykeg; 3167 args.align = UMA_SUPER_ALIGN - 1; 3168 args.flags = UMA_ZFLAG_INTERNAL; 3169 zone_ctor(kegs, zsize, &args, M_WAITOK); 3170 3171 args.name = "UMA Zones"; 3172 args.size = zsize; 3173 args.ctor = zone_ctor; 3174 args.dtor = zone_dtor; 3175 args.uminit = zero_init; 3176 args.fini = NULL; 3177 args.keg = NULL; 3178 args.align = UMA_SUPER_ALIGN - 1; 3179 args.flags = UMA_ZFLAG_INTERNAL; 3180 zone_ctor(zones, zsize, &args, M_WAITOK); 3181 3182 /* Now make zones for slab headers */ 3183 slabzones[0] = uma_zcreate("UMA Slabs 0", SLABZONE0_SIZE, 3184 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 3185 slabzones[1] = uma_zcreate("UMA Slabs 1", SLABZONE1_SIZE, 3186 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 3187 3188 hashzone = uma_zcreate("UMA Hash", 3189 sizeof(struct slabhead *) * UMA_HASH_SIZE_INIT, 3190 NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZFLAG_INTERNAL); 3191 3192 bucket_init(); 3193 smr_init(); 3194 } 3195 3196 #ifndef UMA_USE_DMAP 3197 extern void vm_radix_reserve_kva(void); 3198 #endif 3199 3200 /* 3201 * Advertise the availability of normal kva allocations and switch to 3202 * the default back-end allocator. Marks the KVA we consumed on startup 3203 * as used in the map. 3204 */ 3205 void 3206 uma_startup2(void) 3207 { 3208 3209 if (bootstart != bootmem) { 3210 vm_map_lock(kernel_map); 3211 (void)vm_map_insert(kernel_map, NULL, 0, bootstart, bootmem, 3212 VM_PROT_RW, VM_PROT_RW, MAP_NOFAULT); 3213 vm_map_unlock(kernel_map); 3214 } 3215 3216 #ifndef UMA_USE_DMAP 3217 /* Set up radix zone to use noobj_alloc. */ 3218 vm_radix_reserve_kva(); 3219 #endif 3220 3221 booted = BOOT_KVA; 3222 zone_foreach_unlocked(zone_kva_available, NULL); 3223 bucket_enable(); 3224 } 3225 3226 /* 3227 * Allocate counters as early as possible so that boot-time allocations are 3228 * accounted more precisely. 3229 */ 3230 static void 3231 uma_startup_pcpu(void *arg __unused) 3232 { 3233 3234 zone_foreach_unlocked(zone_alloc_counters, NULL); 3235 booted = BOOT_PCPU; 3236 } 3237 SYSINIT(uma_startup_pcpu, SI_SUB_COUNTER, SI_ORDER_ANY, uma_startup_pcpu, NULL); 3238 3239 /* 3240 * Finish our initialization steps. 3241 */ 3242 static void 3243 uma_startup3(void *arg __unused) 3244 { 3245 3246 #ifdef INVARIANTS 3247 TUNABLE_INT_FETCH("vm.debug.divisor", &dbg_divisor); 3248 uma_dbg_cnt = counter_u64_alloc(M_WAITOK); 3249 uma_skip_cnt = counter_u64_alloc(M_WAITOK); 3250 #endif 3251 zone_foreach_unlocked(zone_alloc_sysctl, NULL); 3252 booted = BOOT_RUNNING; 3253 3254 EVENTHANDLER_REGISTER(shutdown_post_sync, uma_shutdown, NULL, 3255 EVENTHANDLER_PRI_FIRST); 3256 } 3257 SYSINIT(uma_startup3, SI_SUB_VM_CONF, SI_ORDER_SECOND, uma_startup3, NULL); 3258 3259 static void 3260 uma_startup4(void *arg __unused) 3261 { 3262 TIMEOUT_TASK_INIT(taskqueue_thread, &uma_timeout_task, 0, uma_timeout, 3263 NULL); 3264 taskqueue_enqueue_timeout(taskqueue_thread, &uma_timeout_task, 3265 UMA_TIMEOUT * hz); 3266 } 3267 SYSINIT(uma_startup4, SI_SUB_TASKQ, SI_ORDER_ANY, uma_startup4, NULL); 3268 3269 static void 3270 uma_shutdown(void) 3271 { 3272 3273 booted = BOOT_SHUTDOWN; 3274 } 3275 3276 static uma_keg_t 3277 uma_kcreate(uma_zone_t zone, size_t size, uma_init uminit, uma_fini fini, 3278 int align, uint32_t flags) 3279 { 3280 struct uma_kctor_args args; 3281 3282 args.size = size; 3283 args.uminit = uminit; 3284 args.fini = fini; 3285 args.align = align; 3286 args.flags = flags; 3287 args.zone = zone; 3288 return (zone_alloc_item(kegs, &args, UMA_ANYDOMAIN, M_WAITOK)); 3289 } 3290 3291 3292 static void 3293 check_align_mask(unsigned int mask) 3294 { 3295 3296 KASSERT(powerof2(mask + 1), 3297 ("UMA: %s: Not the mask of a power of 2 (%#x)", __func__, mask)); 3298 /* 3299 * Make sure the stored align mask doesn't have its highest bit set, 3300 * which would cause implementation-defined behavior when passing it as 3301 * the 'align' argument of uma_zcreate(). Such very large alignments do 3302 * not make sense anyway. 3303 */ 3304 KASSERT(mask <= INT_MAX, 3305 ("UMA: %s: Mask too big (%#x)", __func__, mask)); 3306 } 3307 3308 /* Public functions */ 3309 /* See uma.h */ 3310 void 3311 uma_set_cache_align_mask(unsigned int mask) 3312 { 3313 3314 check_align_mask(mask); 3315 uma_cache_align_mask = mask; 3316 } 3317 3318 /* Returns the alignment mask to use to request cache alignment. */ 3319 unsigned int 3320 uma_get_cache_align_mask(void) 3321 { 3322 return (uma_cache_align_mask); 3323 } 3324 3325 /* See uma.h */ 3326 uma_zone_t 3327 uma_zcreate(const char *name, size_t size, uma_ctor ctor, uma_dtor dtor, 3328 uma_init uminit, uma_fini fini, int align, uint32_t flags) 3329 3330 { 3331 struct uma_zctor_args args; 3332 uma_zone_t res; 3333 3334 check_align_mask(align); 3335 3336 /* This stuff is essential for the zone ctor */ 3337 memset(&args, 0, sizeof(args)); 3338 args.name = name; 3339 args.size = size; 3340 args.ctor = ctor; 3341 args.dtor = dtor; 3342 args.uminit = uminit; 3343 args.fini = fini; 3344 #if defined(INVARIANTS) && !defined(KASAN) && !defined(KMSAN) 3345 /* 3346 * Inject procedures which check for memory use after free if we are 3347 * allowed to scramble the memory while it is not allocated. This 3348 * requires that: UMA is actually able to access the memory, no init 3349 * or fini procedures, no dependency on the initial value of the 3350 * memory, and no (legitimate) use of the memory after free. Note, 3351 * the ctor and dtor do not need to be empty. 3352 */ 3353 if ((!(flags & (UMA_ZONE_ZINIT | UMA_ZONE_NOTOUCH | 3354 UMA_ZONE_NOFREE))) && uminit == NULL && fini == NULL) { 3355 args.uminit = trash_init; 3356 args.fini = trash_fini; 3357 } 3358 #endif 3359 args.align = align; 3360 args.flags = flags; 3361 args.keg = NULL; 3362 3363 sx_xlock(&uma_reclaim_lock); 3364 res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 3365 sx_xunlock(&uma_reclaim_lock); 3366 3367 return (res); 3368 } 3369 3370 /* See uma.h */ 3371 uma_zone_t 3372 uma_zsecond_create(const char *name, uma_ctor ctor, uma_dtor dtor, 3373 uma_init zinit, uma_fini zfini, uma_zone_t primary) 3374 { 3375 struct uma_zctor_args args; 3376 uma_keg_t keg; 3377 uma_zone_t res; 3378 3379 keg = primary->uz_keg; 3380 memset(&args, 0, sizeof(args)); 3381 args.name = name; 3382 args.size = keg->uk_size; 3383 args.ctor = ctor; 3384 args.dtor = dtor; 3385 args.uminit = zinit; 3386 args.fini = zfini; 3387 args.align = keg->uk_align; 3388 args.flags = keg->uk_flags | UMA_ZONE_SECONDARY; 3389 args.keg = keg; 3390 3391 sx_xlock(&uma_reclaim_lock); 3392 res = zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK); 3393 sx_xunlock(&uma_reclaim_lock); 3394 3395 return (res); 3396 } 3397 3398 /* See uma.h */ 3399 uma_zone_t 3400 uma_zcache_create(const char *name, int size, uma_ctor ctor, uma_dtor dtor, 3401 uma_init zinit, uma_fini zfini, uma_import zimport, uma_release zrelease, 3402 void *arg, int flags) 3403 { 3404 struct uma_zctor_args args; 3405 3406 memset(&args, 0, sizeof(args)); 3407 args.name = name; 3408 args.size = size; 3409 args.ctor = ctor; 3410 args.dtor = dtor; 3411 args.uminit = zinit; 3412 args.fini = zfini; 3413 args.import = zimport; 3414 args.release = zrelease; 3415 args.arg = arg; 3416 args.align = 0; 3417 args.flags = flags | UMA_ZFLAG_CACHE; 3418 3419 return (zone_alloc_item(zones, &args, UMA_ANYDOMAIN, M_WAITOK)); 3420 } 3421 3422 /* See uma.h */ 3423 void 3424 uma_zdestroy(uma_zone_t zone) 3425 { 3426 3427 /* 3428 * Large slabs are expensive to reclaim, so don't bother doing 3429 * unnecessary work if we're shutting down. 3430 */ 3431 if (booted == BOOT_SHUTDOWN && 3432 zone->uz_fini == NULL && zone->uz_release == zone_release) 3433 return; 3434 sx_xlock(&uma_reclaim_lock); 3435 zone_free_item(zones, zone, NULL, SKIP_NONE); 3436 sx_xunlock(&uma_reclaim_lock); 3437 } 3438 3439 void 3440 uma_zwait(uma_zone_t zone) 3441 { 3442 3443 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 3444 uma_zfree_smr(zone, uma_zalloc_smr(zone, M_WAITOK)); 3445 else if ((zone->uz_flags & UMA_ZONE_PCPU) != 0) 3446 uma_zfree_pcpu(zone, uma_zalloc_pcpu(zone, M_WAITOK)); 3447 else 3448 uma_zfree(zone, uma_zalloc(zone, M_WAITOK)); 3449 } 3450 3451 void * 3452 uma_zalloc_pcpu_arg(uma_zone_t zone, void *udata, int flags) 3453 { 3454 void *item, *pcpu_item; 3455 #ifdef SMP 3456 int i; 3457 3458 MPASS(zone->uz_flags & UMA_ZONE_PCPU); 3459 #endif 3460 item = uma_zalloc_arg(zone, udata, flags & ~M_ZERO); 3461 if (item == NULL) 3462 return (NULL); 3463 pcpu_item = zpcpu_base_to_offset(item); 3464 if (flags & M_ZERO) { 3465 #ifdef SMP 3466 for (i = 0; i <= mp_maxid; i++) 3467 bzero(zpcpu_get_cpu(pcpu_item, i), zone->uz_size); 3468 #else 3469 bzero(item, zone->uz_size); 3470 #endif 3471 } 3472 return (pcpu_item); 3473 } 3474 3475 /* 3476 * A stub while both regular and pcpu cases are identical. 3477 */ 3478 void 3479 uma_zfree_pcpu_arg(uma_zone_t zone, void *pcpu_item, void *udata) 3480 { 3481 void *item; 3482 3483 #ifdef SMP 3484 MPASS(zone->uz_flags & UMA_ZONE_PCPU); 3485 #endif 3486 3487 /* uma_zfree_pcu_*(..., NULL) does nothing, to match free(9). */ 3488 if (pcpu_item == NULL) 3489 return; 3490 3491 item = zpcpu_offset_to_base(pcpu_item); 3492 uma_zfree_arg(zone, item, udata); 3493 } 3494 3495 static inline void * 3496 item_ctor(uma_zone_t zone, int uz_flags, int size, void *udata, int flags, 3497 void *item) 3498 { 3499 #ifdef INVARIANTS 3500 bool skipdbg; 3501 #endif 3502 3503 kasan_mark_item_valid(zone, item); 3504 kmsan_mark_item_uninitialized(zone, item); 3505 3506 #ifdef INVARIANTS 3507 skipdbg = uma_dbg_zskip(zone, item); 3508 if (!skipdbg && (uz_flags & UMA_ZFLAG_TRASH) != 0 && 3509 zone->uz_ctor != trash_ctor) 3510 trash_ctor(item, size, zone, flags); 3511 #endif 3512 3513 /* Check flags before loading ctor pointer. */ 3514 if (__predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0) && 3515 __predict_false(zone->uz_ctor != NULL) && 3516 zone->uz_ctor(item, size, udata, flags) != 0) { 3517 counter_u64_add(zone->uz_fails, 1); 3518 zone_free_item(zone, item, udata, SKIP_DTOR | SKIP_CNT); 3519 return (NULL); 3520 } 3521 #ifdef INVARIANTS 3522 if (!skipdbg) 3523 uma_dbg_alloc(zone, NULL, item); 3524 #endif 3525 if (__predict_false(flags & M_ZERO)) 3526 return (memset(item, 0, size)); 3527 3528 return (item); 3529 } 3530 3531 static inline void 3532 item_dtor(uma_zone_t zone, void *item, int size, void *udata, 3533 enum zfreeskip skip) 3534 { 3535 #ifdef INVARIANTS 3536 bool skipdbg; 3537 3538 skipdbg = uma_dbg_zskip(zone, item); 3539 if (skip == SKIP_NONE && !skipdbg) { 3540 if ((zone->uz_flags & UMA_ZONE_MALLOC) != 0) 3541 uma_dbg_free(zone, udata, item); 3542 else 3543 uma_dbg_free(zone, NULL, item); 3544 } 3545 #endif 3546 if (__predict_true(skip < SKIP_DTOR)) { 3547 if (zone->uz_dtor != NULL) 3548 zone->uz_dtor(item, size, udata); 3549 #ifdef INVARIANTS 3550 if (!skipdbg && (zone->uz_flags & UMA_ZFLAG_TRASH) != 0 && 3551 zone->uz_dtor != trash_dtor) 3552 trash_dtor(item, size, zone); 3553 #endif 3554 } 3555 kasan_mark_item_invalid(zone, item); 3556 } 3557 3558 #ifdef NUMA 3559 static int 3560 item_domain(void *item) 3561 { 3562 int domain; 3563 3564 domain = vm_phys_domain(vtophys(item)); 3565 KASSERT(domain >= 0 && domain < vm_ndomains, 3566 ("%s: unknown domain for item %p", __func__, item)); 3567 return (domain); 3568 } 3569 #endif 3570 3571 #if defined(INVARIANTS) || defined(DEBUG_MEMGUARD) || defined(WITNESS) 3572 #if defined(INVARIANTS) && (defined(DDB) || defined(STACK)) 3573 #include <sys/stack.h> 3574 #endif 3575 #define UMA_ZALLOC_DEBUG 3576 static int 3577 uma_zalloc_debug(uma_zone_t zone, void **itemp, void *udata, int flags) 3578 { 3579 int error; 3580 3581 error = 0; 3582 #ifdef WITNESS 3583 if (flags & M_WAITOK) { 3584 WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK, NULL, 3585 "uma_zalloc_debug: zone \"%s\"", zone->uz_name); 3586 } 3587 #endif 3588 3589 #ifdef INVARIANTS 3590 KASSERT((flags & M_EXEC) == 0, 3591 ("uma_zalloc_debug: called with M_EXEC")); 3592 KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3593 ("uma_zalloc_debug: called within spinlock or critical section")); 3594 KASSERT((zone->uz_flags & UMA_ZONE_PCPU) == 0 || (flags & M_ZERO) == 0, 3595 ("uma_zalloc_debug: allocating from a pcpu zone with M_ZERO")); 3596 3597 _Static_assert(M_NOWAIT != 0 && M_WAITOK != 0, 3598 "M_NOWAIT and M_WAITOK must be non-zero for this assertion:"); 3599 #if 0 3600 /* 3601 * Give the #elif clause time to find problems, then remove it 3602 * and enable this. (Remove <sys/stack.h> above, too.) 3603 */ 3604 KASSERT((flags & (M_NOWAIT|M_WAITOK)) == M_NOWAIT || 3605 (flags & (M_NOWAIT|M_WAITOK)) == M_WAITOK, 3606 ("uma_zalloc_debug: must pass one of M_NOWAIT or M_WAITOK")); 3607 #elif defined(DDB) || defined(STACK) 3608 if (__predict_false((flags & (M_NOWAIT|M_WAITOK)) != M_NOWAIT && 3609 (flags & (M_NOWAIT|M_WAITOK)) != M_WAITOK)) { 3610 static int stack_count; 3611 struct stack st; 3612 3613 if (stack_count < 10) { 3614 ++stack_count; 3615 printf("uma_zalloc* called with bad WAIT flags:\n"); 3616 stack_save(&st); 3617 stack_print(&st); 3618 } 3619 } 3620 #endif 3621 #endif 3622 3623 #ifdef DEBUG_MEMGUARD 3624 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && 3625 memguard_cmp_zone(zone)) { 3626 void *item; 3627 item = memguard_alloc(zone->uz_size, flags); 3628 if (item != NULL) { 3629 error = EJUSTRETURN; 3630 if (zone->uz_init != NULL && 3631 zone->uz_init(item, zone->uz_size, flags) != 0) { 3632 *itemp = NULL; 3633 return (error); 3634 } 3635 if (zone->uz_ctor != NULL && 3636 zone->uz_ctor(item, zone->uz_size, udata, 3637 flags) != 0) { 3638 counter_u64_add(zone->uz_fails, 1); 3639 if (zone->uz_fini != NULL) 3640 zone->uz_fini(item, zone->uz_size); 3641 *itemp = NULL; 3642 return (error); 3643 } 3644 *itemp = item; 3645 return (error); 3646 } 3647 /* This is unfortunate but should not be fatal. */ 3648 } 3649 #endif 3650 return (error); 3651 } 3652 3653 static int 3654 uma_zfree_debug(uma_zone_t zone, void *item, void *udata) 3655 { 3656 KASSERT(curthread->td_critnest == 0 || SCHEDULER_STOPPED(), 3657 ("uma_zfree_debug: called with spinlock or critical section held")); 3658 3659 #ifdef DEBUG_MEMGUARD 3660 if ((zone->uz_flags & (UMA_ZONE_SMR | UMA_ZFLAG_CACHE)) == 0 && 3661 is_memguard_addr(item)) { 3662 if (zone->uz_dtor != NULL) 3663 zone->uz_dtor(item, zone->uz_size, udata); 3664 if (zone->uz_fini != NULL) 3665 zone->uz_fini(item, zone->uz_size); 3666 memguard_free(item); 3667 return (EJUSTRETURN); 3668 } 3669 #endif 3670 return (0); 3671 } 3672 #endif 3673 3674 static inline void * 3675 cache_alloc_item(uma_zone_t zone, uma_cache_t cache, uma_cache_bucket_t bucket, 3676 void *udata, int flags) 3677 { 3678 void *item; 3679 int size, uz_flags; 3680 3681 item = cache_bucket_pop(cache, bucket); 3682 size = cache_uz_size(cache); 3683 uz_flags = cache_uz_flags(cache); 3684 critical_exit(); 3685 return (item_ctor(zone, uz_flags, size, udata, flags, item)); 3686 } 3687 3688 static __noinline void * 3689 cache_alloc_retry(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) 3690 { 3691 uma_cache_bucket_t bucket; 3692 int domain; 3693 3694 while (cache_alloc(zone, cache, udata, flags)) { 3695 cache = &zone->uz_cpu[curcpu]; 3696 bucket = &cache->uc_allocbucket; 3697 if (__predict_false(bucket->ucb_cnt == 0)) 3698 continue; 3699 return (cache_alloc_item(zone, cache, bucket, udata, flags)); 3700 } 3701 critical_exit(); 3702 3703 /* 3704 * We can not get a bucket so try to return a single item. 3705 */ 3706 if (zone->uz_flags & UMA_ZONE_FIRSTTOUCH) 3707 domain = PCPU_GET(domain); 3708 else 3709 domain = UMA_ANYDOMAIN; 3710 return (zone_alloc_item(zone, udata, domain, flags)); 3711 } 3712 3713 /* See uma.h */ 3714 void * 3715 uma_zalloc_smr(uma_zone_t zone, int flags) 3716 { 3717 uma_cache_bucket_t bucket; 3718 uma_cache_t cache; 3719 3720 CTR3(KTR_UMA, "uma_zalloc_smr zone %s(%p) flags %d", zone->uz_name, 3721 zone, flags); 3722 3723 #ifdef UMA_ZALLOC_DEBUG 3724 void *item; 3725 3726 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, 3727 ("uma_zalloc_arg: called with non-SMR zone.")); 3728 if (uma_zalloc_debug(zone, &item, NULL, flags) == EJUSTRETURN) 3729 return (item); 3730 #endif 3731 3732 critical_enter(); 3733 cache = &zone->uz_cpu[curcpu]; 3734 bucket = &cache->uc_allocbucket; 3735 if (__predict_false(bucket->ucb_cnt == 0)) 3736 return (cache_alloc_retry(zone, cache, NULL, flags)); 3737 return (cache_alloc_item(zone, cache, bucket, NULL, flags)); 3738 } 3739 3740 /* See uma.h */ 3741 void * 3742 uma_zalloc_arg(uma_zone_t zone, void *udata, int flags) 3743 { 3744 uma_cache_bucket_t bucket; 3745 uma_cache_t cache; 3746 3747 /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 3748 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 3749 3750 /* This is the fast path allocation */ 3751 CTR3(KTR_UMA, "uma_zalloc_arg zone %s(%p) flags %d", zone->uz_name, 3752 zone, flags); 3753 3754 #ifdef UMA_ZALLOC_DEBUG 3755 void *item; 3756 3757 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 3758 ("uma_zalloc_arg: called with SMR zone.")); 3759 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) 3760 return (item); 3761 #endif 3762 3763 /* 3764 * If possible, allocate from the per-CPU cache. There are two 3765 * requirements for safe access to the per-CPU cache: (1) the thread 3766 * accessing the cache must not be preempted or yield during access, 3767 * and (2) the thread must not migrate CPUs without switching which 3768 * cache it accesses. We rely on a critical section to prevent 3769 * preemption and migration. We release the critical section in 3770 * order to acquire the zone mutex if we are unable to allocate from 3771 * the current cache; when we re-acquire the critical section, we 3772 * must detect and handle migration if it has occurred. 3773 */ 3774 critical_enter(); 3775 cache = &zone->uz_cpu[curcpu]; 3776 bucket = &cache->uc_allocbucket; 3777 if (__predict_false(bucket->ucb_cnt == 0)) 3778 return (cache_alloc_retry(zone, cache, udata, flags)); 3779 return (cache_alloc_item(zone, cache, bucket, udata, flags)); 3780 } 3781 3782 /* 3783 * Replenish an alloc bucket and possibly restore an old one. Called in 3784 * a critical section. Returns in a critical section. 3785 * 3786 * A false return value indicates an allocation failure. 3787 * A true return value indicates success and the caller should retry. 3788 */ 3789 static __noinline bool 3790 cache_alloc(uma_zone_t zone, uma_cache_t cache, void *udata, int flags) 3791 { 3792 uma_bucket_t bucket; 3793 uint32_t zflags; 3794 int curdomain, domain; 3795 bool new; 3796 3797 CRITICAL_ASSERT(curthread); 3798 3799 /* 3800 * If we have run out of items in our alloc bucket see 3801 * if we can switch with the free bucket. 3802 * 3803 * SMR zones can't re-use the free bucket until the sequence has 3804 * expired. When KASAN is enabled, we want to avoid re-using free 3805 * items in order to improve reliability of use-after-free detection. 3806 */ 3807 zflags = cache_uz_flags(cache); 3808 if ((zflags & UMA_ZONE_SMR) == 0 && 3809 #ifdef KASAN 3810 (zflags & UMA_ZONE_NOKASAN) != 0 && 3811 #endif 3812 cache->uc_freebucket.ucb_cnt != 0) { 3813 cache_bucket_swap(&cache->uc_freebucket, 3814 &cache->uc_allocbucket); 3815 return (true); 3816 } 3817 3818 /* 3819 * Discard any empty allocation bucket while we hold no locks. 3820 */ 3821 bucket = cache_bucket_unload_alloc(cache); 3822 critical_exit(); 3823 3824 if (bucket != NULL) { 3825 KASSERT(bucket->ub_cnt == 0, 3826 ("cache_alloc: Entered with non-empty alloc bucket.")); 3827 bucket_free(zone, bucket, udata); 3828 } 3829 3830 /* 3831 * Attempt to retrieve the item from the per-CPU cache has failed, so 3832 * we must go back to the zone. This requires the zdom lock, so we 3833 * must drop the critical section, then re-acquire it when we go back 3834 * to the cache. Since the critical section is released, we may be 3835 * preempted or migrate. As such, make sure not to maintain any 3836 * thread-local state specific to the cache from prior to releasing 3837 * the critical section. 3838 */ 3839 domain = PCPU_GET(domain); 3840 if ((zflags & UMA_ZONE_ROUNDROBIN) != 0 || VM_DOMAIN_EMPTY(domain)) 3841 domain = zone_domain_highest(zone, domain); 3842 bucket = cache_fetch_bucket(zone, cache, domain); 3843 if (bucket == NULL && zone->uz_bucket_size != 0 && !bucketdisable) { 3844 bucket = zone_alloc_bucket(zone, udata, domain, flags); 3845 new = true; 3846 } else { 3847 new = false; 3848 } 3849 3850 CTR3(KTR_UMA, "uma_zalloc: zone %s(%p) bucket zone returned %p", 3851 zone->uz_name, zone, bucket); 3852 if (bucket == NULL) { 3853 critical_enter(); 3854 return (false); 3855 } 3856 3857 /* 3858 * See if we lost the race or were migrated. Cache the 3859 * initialized bucket to make this less likely or claim 3860 * the memory directly. 3861 */ 3862 critical_enter(); 3863 cache = &zone->uz_cpu[curcpu]; 3864 if (cache->uc_allocbucket.ucb_bucket == NULL && 3865 ((zflags & UMA_ZONE_FIRSTTOUCH) == 0 || 3866 (curdomain = PCPU_GET(domain)) == domain || 3867 VM_DOMAIN_EMPTY(curdomain))) { 3868 if (new) 3869 atomic_add_long(&ZDOM_GET(zone, domain)->uzd_imax, 3870 bucket->ub_cnt); 3871 cache_bucket_load_alloc(cache, bucket); 3872 return (true); 3873 } 3874 3875 /* 3876 * We lost the race, release this bucket and start over. 3877 */ 3878 critical_exit(); 3879 zone_put_bucket(zone, domain, bucket, udata, !new); 3880 critical_enter(); 3881 3882 return (true); 3883 } 3884 3885 void * 3886 uma_zalloc_domain(uma_zone_t zone, void *udata, int domain, int flags) 3887 { 3888 #ifdef NUMA 3889 uma_bucket_t bucket; 3890 uma_zone_domain_t zdom; 3891 void *item; 3892 #endif 3893 3894 /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 3895 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 3896 3897 /* This is the fast path allocation */ 3898 CTR4(KTR_UMA, "uma_zalloc_domain zone %s(%p) domain %d flags %d", 3899 zone->uz_name, zone, domain, flags); 3900 3901 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 3902 ("uma_zalloc_domain: called with SMR zone.")); 3903 #ifdef NUMA 3904 KASSERT((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0, 3905 ("uma_zalloc_domain: called with non-FIRSTTOUCH zone.")); 3906 3907 if (vm_ndomains == 1) 3908 return (uma_zalloc_arg(zone, udata, flags)); 3909 3910 #ifdef UMA_ZALLOC_DEBUG 3911 if (uma_zalloc_debug(zone, &item, udata, flags) == EJUSTRETURN) 3912 return (item); 3913 #endif 3914 3915 /* 3916 * Try to allocate from the bucket cache before falling back to the keg. 3917 * We could try harder and attempt to allocate from per-CPU caches or 3918 * the per-domain cross-domain buckets, but the complexity is probably 3919 * not worth it. It is more important that frees of previous 3920 * cross-domain allocations do not blow up the cache. 3921 */ 3922 zdom = zone_domain_lock(zone, domain); 3923 if ((bucket = zone_fetch_bucket(zone, zdom, false)) != NULL) { 3924 item = bucket->ub_bucket[bucket->ub_cnt - 1]; 3925 #ifdef INVARIANTS 3926 bucket->ub_bucket[bucket->ub_cnt - 1] = NULL; 3927 #endif 3928 bucket->ub_cnt--; 3929 zone_put_bucket(zone, domain, bucket, udata, true); 3930 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, 3931 flags, item); 3932 if (item != NULL) { 3933 KASSERT(item_domain(item) == domain, 3934 ("%s: bucket cache item %p from wrong domain", 3935 __func__, item)); 3936 counter_u64_add(zone->uz_allocs, 1); 3937 } 3938 return (item); 3939 } 3940 ZDOM_UNLOCK(zdom); 3941 return (zone_alloc_item(zone, udata, domain, flags)); 3942 #else 3943 return (uma_zalloc_arg(zone, udata, flags)); 3944 #endif 3945 } 3946 3947 /* 3948 * Find a slab with some space. Prefer slabs that are partially used over those 3949 * that are totally full. This helps to reduce fragmentation. 3950 * 3951 * If 'rr' is 1, search all domains starting from 'domain'. Otherwise check 3952 * only 'domain'. 3953 */ 3954 static uma_slab_t 3955 keg_first_slab(uma_keg_t keg, int domain, bool rr) 3956 { 3957 uma_domain_t dom; 3958 uma_slab_t slab; 3959 int start; 3960 3961 KASSERT(domain >= 0 && domain < vm_ndomains, 3962 ("keg_first_slab: domain %d out of range", domain)); 3963 KEG_LOCK_ASSERT(keg, domain); 3964 3965 slab = NULL; 3966 start = domain; 3967 do { 3968 dom = &keg->uk_domain[domain]; 3969 if ((slab = LIST_FIRST(&dom->ud_part_slab)) != NULL) 3970 return (slab); 3971 if ((slab = LIST_FIRST(&dom->ud_free_slab)) != NULL) { 3972 LIST_REMOVE(slab, us_link); 3973 dom->ud_free_slabs--; 3974 LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 3975 return (slab); 3976 } 3977 if (rr) 3978 domain = (domain + 1) % vm_ndomains; 3979 } while (domain != start); 3980 3981 return (NULL); 3982 } 3983 3984 /* 3985 * Fetch an existing slab from a free or partial list. Returns with the 3986 * keg domain lock held if a slab was found or unlocked if not. 3987 */ 3988 static uma_slab_t 3989 keg_fetch_free_slab(uma_keg_t keg, int domain, bool rr, int flags) 3990 { 3991 uma_slab_t slab; 3992 uint32_t reserve; 3993 3994 /* HASH has a single free list. */ 3995 if ((keg->uk_flags & UMA_ZFLAG_HASH) != 0) 3996 domain = 0; 3997 3998 KEG_LOCK(keg, domain); 3999 reserve = (flags & M_USE_RESERVE) != 0 ? 0 : keg->uk_reserve; 4000 if (keg->uk_domain[domain].ud_free_items <= reserve || 4001 (slab = keg_first_slab(keg, domain, rr)) == NULL) { 4002 KEG_UNLOCK(keg, domain); 4003 return (NULL); 4004 } 4005 return (slab); 4006 } 4007 4008 static uma_slab_t 4009 keg_fetch_slab(uma_keg_t keg, uma_zone_t zone, int rdomain, const int flags) 4010 { 4011 struct vm_domainset_iter di; 4012 uma_slab_t slab; 4013 int aflags, domain; 4014 bool rr; 4015 4016 KASSERT((flags & (M_WAITOK | M_NOVM)) != (M_WAITOK | M_NOVM), 4017 ("%s: invalid flags %#x", __func__, flags)); 4018 4019 restart: 4020 /* 4021 * Use the keg's policy if upper layers haven't already specified a 4022 * domain (as happens with first-touch zones). 4023 */ 4024 rr = rdomain == UMA_ANYDOMAIN; 4025 aflags = flags; 4026 if (rr) { 4027 if (vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 4028 &aflags) != 0) 4029 return (NULL); 4030 } else 4031 domain = rdomain; 4032 4033 for (;;) { 4034 slab = keg_fetch_free_slab(keg, domain, rr, flags); 4035 if (slab != NULL) 4036 return (slab); 4037 4038 /* 4039 * M_NOVM is used to break the recursion that can otherwise 4040 * occur if low-level memory management routines use UMA. 4041 */ 4042 if ((flags & M_NOVM) == 0) { 4043 slab = keg_alloc_slab(keg, zone, domain, flags, aflags); 4044 if (slab != NULL) 4045 return (slab); 4046 } 4047 4048 if (!rr) { 4049 if ((flags & M_USE_RESERVE) != 0) { 4050 /* 4051 * Drain reserves from other domains before 4052 * giving up or sleeping. It may be useful to 4053 * support per-domain reserves eventually. 4054 */ 4055 rdomain = UMA_ANYDOMAIN; 4056 goto restart; 4057 } 4058 if ((flags & M_WAITOK) == 0) 4059 break; 4060 vm_wait_domain(domain); 4061 } else if (vm_domainset_iter_policy(&di, &domain) != 0) 4062 break; 4063 } 4064 4065 /* 4066 * We might not have been able to get a slab but another cpu 4067 * could have while we were unlocked. Check again before we 4068 * fail. 4069 */ 4070 if ((slab = keg_fetch_free_slab(keg, domain, rr, flags)) != NULL) 4071 return (slab); 4072 4073 return (NULL); 4074 } 4075 4076 static void * 4077 slab_alloc_item(uma_keg_t keg, uma_slab_t slab) 4078 { 4079 uma_domain_t dom; 4080 void *item; 4081 int freei; 4082 4083 KEG_LOCK_ASSERT(keg, slab->us_domain); 4084 4085 dom = &keg->uk_domain[slab->us_domain]; 4086 freei = BIT_FFS(keg->uk_ipers, &slab->us_free) - 1; 4087 BIT_CLR(keg->uk_ipers, freei, &slab->us_free); 4088 item = slab_item(slab, keg, freei); 4089 slab->us_freecount--; 4090 dom->ud_free_items--; 4091 4092 /* 4093 * Move this slab to the full list. It must be on the partial list, so 4094 * we do not need to update the free slab count. In particular, 4095 * keg_fetch_slab() always returns slabs on the partial list. 4096 */ 4097 if (slab->us_freecount == 0) { 4098 LIST_REMOVE(slab, us_link); 4099 LIST_INSERT_HEAD(&dom->ud_full_slab, slab, us_link); 4100 } 4101 4102 return (item); 4103 } 4104 4105 static int 4106 zone_import(void *arg, void **bucket, int max, int domain, int flags) 4107 { 4108 uma_domain_t dom; 4109 uma_zone_t zone; 4110 uma_slab_t slab; 4111 uma_keg_t keg; 4112 #ifdef NUMA 4113 int stripe; 4114 #endif 4115 int i; 4116 4117 zone = arg; 4118 slab = NULL; 4119 keg = zone->uz_keg; 4120 /* Try to keep the buckets totally full */ 4121 for (i = 0; i < max; ) { 4122 if ((slab = keg_fetch_slab(keg, zone, domain, flags)) == NULL) 4123 break; 4124 #ifdef NUMA 4125 stripe = howmany(max, vm_ndomains); 4126 #endif 4127 dom = &keg->uk_domain[slab->us_domain]; 4128 do { 4129 bucket[i++] = slab_alloc_item(keg, slab); 4130 if (keg->uk_reserve > 0 && 4131 dom->ud_free_items <= keg->uk_reserve) { 4132 /* 4133 * Avoid depleting the reserve after a 4134 * successful item allocation, even if 4135 * M_USE_RESERVE is specified. 4136 */ 4137 KEG_UNLOCK(keg, slab->us_domain); 4138 goto out; 4139 } 4140 #ifdef NUMA 4141 /* 4142 * If the zone is striped we pick a new slab for every 4143 * N allocations. Eliminating this conditional will 4144 * instead pick a new domain for each bucket rather 4145 * than stripe within each bucket. The current option 4146 * produces more fragmentation and requires more cpu 4147 * time but yields better distribution. 4148 */ 4149 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0 && 4150 vm_ndomains > 1 && --stripe == 0) 4151 break; 4152 #endif 4153 } while (slab->us_freecount != 0 && i < max); 4154 KEG_UNLOCK(keg, slab->us_domain); 4155 4156 /* Don't block if we allocated any successfully. */ 4157 flags &= ~M_WAITOK; 4158 flags |= M_NOWAIT; 4159 } 4160 out: 4161 return i; 4162 } 4163 4164 static int 4165 zone_alloc_limit_hard(uma_zone_t zone, int count, int flags) 4166 { 4167 uint64_t old, new, total, max; 4168 4169 /* 4170 * The hard case. We're going to sleep because there were existing 4171 * sleepers or because we ran out of items. This routine enforces 4172 * fairness by keeping fifo order. 4173 * 4174 * First release our ill gotten gains and make some noise. 4175 */ 4176 for (;;) { 4177 zone_free_limit(zone, count); 4178 zone_log_warning(zone); 4179 zone_maxaction(zone); 4180 if (flags & M_NOWAIT) 4181 return (0); 4182 4183 /* 4184 * We need to allocate an item or set ourself as a sleeper 4185 * while the sleepq lock is held to avoid wakeup races. This 4186 * is essentially a home rolled semaphore. 4187 */ 4188 sleepq_lock(&zone->uz_max_items); 4189 old = zone->uz_items; 4190 do { 4191 MPASS(UZ_ITEMS_SLEEPERS(old) < UZ_ITEMS_SLEEPERS_MAX); 4192 /* Cache the max since we will evaluate twice. */ 4193 max = zone->uz_max_items; 4194 if (UZ_ITEMS_SLEEPERS(old) != 0 || 4195 UZ_ITEMS_COUNT(old) >= max) 4196 new = old + UZ_ITEMS_SLEEPER; 4197 else 4198 new = old + MIN(count, max - old); 4199 } while (atomic_fcmpset_64(&zone->uz_items, &old, new) == 0); 4200 4201 /* We may have successfully allocated under the sleepq lock. */ 4202 if (UZ_ITEMS_SLEEPERS(new) == 0) { 4203 sleepq_release(&zone->uz_max_items); 4204 return (new - old); 4205 } 4206 4207 /* 4208 * This is in a different cacheline from uz_items so that we 4209 * don't constantly invalidate the fastpath cacheline when we 4210 * adjust item counts. This could be limited to toggling on 4211 * transitions. 4212 */ 4213 atomic_add_32(&zone->uz_sleepers, 1); 4214 atomic_add_64(&zone->uz_sleeps, 1); 4215 4216 /* 4217 * We have added ourselves as a sleeper. The sleepq lock 4218 * protects us from wakeup races. Sleep now and then retry. 4219 */ 4220 sleepq_add(&zone->uz_max_items, NULL, "zonelimit", 0, 0); 4221 sleepq_wait(&zone->uz_max_items, PVM); 4222 4223 /* 4224 * After wakeup, remove ourselves as a sleeper and try 4225 * again. We no longer have the sleepq lock for protection. 4226 * 4227 * Subract ourselves as a sleeper while attempting to add 4228 * our count. 4229 */ 4230 atomic_subtract_32(&zone->uz_sleepers, 1); 4231 old = atomic_fetchadd_64(&zone->uz_items, 4232 -(UZ_ITEMS_SLEEPER - count)); 4233 /* We're no longer a sleeper. */ 4234 old -= UZ_ITEMS_SLEEPER; 4235 4236 /* 4237 * If we're still at the limit, restart. Notably do not 4238 * block on other sleepers. Cache the max value to protect 4239 * against changes via sysctl. 4240 */ 4241 total = UZ_ITEMS_COUNT(old); 4242 max = zone->uz_max_items; 4243 if (total >= max) 4244 continue; 4245 /* Truncate if necessary, otherwise wake other sleepers. */ 4246 if (total + count > max) { 4247 zone_free_limit(zone, total + count - max); 4248 count = max - total; 4249 } else if (total + count < max && UZ_ITEMS_SLEEPERS(old) != 0) 4250 wakeup_one(&zone->uz_max_items); 4251 4252 return (count); 4253 } 4254 } 4255 4256 /* 4257 * Allocate 'count' items from our max_items limit. Returns the number 4258 * available. If M_NOWAIT is not specified it will sleep until at least 4259 * one item can be allocated. 4260 */ 4261 static int 4262 zone_alloc_limit(uma_zone_t zone, int count, int flags) 4263 { 4264 uint64_t old; 4265 uint64_t max; 4266 4267 max = zone->uz_max_items; 4268 MPASS(max > 0); 4269 4270 /* 4271 * We expect normal allocations to succeed with a simple 4272 * fetchadd. 4273 */ 4274 old = atomic_fetchadd_64(&zone->uz_items, count); 4275 if (__predict_true(old + count <= max)) 4276 return (count); 4277 4278 /* 4279 * If we had some items and no sleepers just return the 4280 * truncated value. We have to release the excess space 4281 * though because that may wake sleepers who weren't woken 4282 * because we were temporarily over the limit. 4283 */ 4284 if (old < max) { 4285 zone_free_limit(zone, (old + count) - max); 4286 return (max - old); 4287 } 4288 return (zone_alloc_limit_hard(zone, count, flags)); 4289 } 4290 4291 /* 4292 * Free a number of items back to the limit. 4293 */ 4294 static void 4295 zone_free_limit(uma_zone_t zone, int count) 4296 { 4297 uint64_t old; 4298 4299 MPASS(count > 0); 4300 4301 /* 4302 * In the common case we either have no sleepers or 4303 * are still over the limit and can just return. 4304 */ 4305 old = atomic_fetchadd_64(&zone->uz_items, -count); 4306 if (__predict_true(UZ_ITEMS_SLEEPERS(old) == 0 || 4307 UZ_ITEMS_COUNT(old) - count >= zone->uz_max_items)) 4308 return; 4309 4310 /* 4311 * Moderate the rate of wakeups. Sleepers will continue 4312 * to generate wakeups if necessary. 4313 */ 4314 wakeup_one(&zone->uz_max_items); 4315 } 4316 4317 static uma_bucket_t 4318 zone_alloc_bucket(uma_zone_t zone, void *udata, int domain, int flags) 4319 { 4320 uma_bucket_t bucket; 4321 int error, maxbucket, cnt; 4322 4323 CTR3(KTR_UMA, "zone_alloc_bucket zone %s(%p) domain %d", zone->uz_name, 4324 zone, domain); 4325 4326 /* Avoid allocs targeting empty domains. */ 4327 if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) 4328 domain = UMA_ANYDOMAIN; 4329 else if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) 4330 domain = UMA_ANYDOMAIN; 4331 4332 if (zone->uz_max_items > 0) 4333 maxbucket = zone_alloc_limit(zone, zone->uz_bucket_size, 4334 M_NOWAIT); 4335 else 4336 maxbucket = zone->uz_bucket_size; 4337 if (maxbucket == 0) 4338 return (NULL); 4339 4340 /* Don't wait for buckets, preserve caller's NOVM setting. */ 4341 bucket = bucket_alloc(zone, udata, M_NOWAIT | (flags & M_NOVM)); 4342 if (bucket == NULL) { 4343 cnt = 0; 4344 goto out; 4345 } 4346 4347 bucket->ub_cnt = zone->uz_import(zone->uz_arg, bucket->ub_bucket, 4348 MIN(maxbucket, bucket->ub_entries), domain, flags); 4349 4350 /* 4351 * Initialize the memory if necessary. 4352 */ 4353 if (bucket->ub_cnt != 0 && zone->uz_init != NULL) { 4354 int i; 4355 4356 for (i = 0; i < bucket->ub_cnt; i++) { 4357 kasan_mark_item_valid(zone, bucket->ub_bucket[i]); 4358 error = zone->uz_init(bucket->ub_bucket[i], 4359 zone->uz_size, flags); 4360 kasan_mark_item_invalid(zone, bucket->ub_bucket[i]); 4361 if (error != 0) 4362 break; 4363 } 4364 4365 /* 4366 * If we couldn't initialize the whole bucket, put the 4367 * rest back onto the freelist. 4368 */ 4369 if (i != bucket->ub_cnt) { 4370 zone->uz_release(zone->uz_arg, &bucket->ub_bucket[i], 4371 bucket->ub_cnt - i); 4372 #ifdef INVARIANTS 4373 bzero(&bucket->ub_bucket[i], 4374 sizeof(void *) * (bucket->ub_cnt - i)); 4375 #endif 4376 bucket->ub_cnt = i; 4377 } 4378 } 4379 4380 cnt = bucket->ub_cnt; 4381 if (bucket->ub_cnt == 0) { 4382 bucket_free(zone, bucket, udata); 4383 counter_u64_add(zone->uz_fails, 1); 4384 bucket = NULL; 4385 } 4386 out: 4387 if (zone->uz_max_items > 0 && cnt < maxbucket) 4388 zone_free_limit(zone, maxbucket - cnt); 4389 4390 return (bucket); 4391 } 4392 4393 /* 4394 * Allocates a single item from a zone. 4395 * 4396 * Arguments 4397 * zone The zone to alloc for. 4398 * udata The data to be passed to the constructor. 4399 * domain The domain to allocate from or UMA_ANYDOMAIN. 4400 * flags M_WAITOK, M_NOWAIT, M_ZERO. 4401 * 4402 * Returns 4403 * NULL if there is no memory and M_NOWAIT is set 4404 * An item if successful 4405 */ 4406 4407 static void * 4408 zone_alloc_item(uma_zone_t zone, void *udata, int domain, int flags) 4409 { 4410 void *item; 4411 4412 if (zone->uz_max_items > 0 && zone_alloc_limit(zone, 1, flags) == 0) { 4413 counter_u64_add(zone->uz_fails, 1); 4414 return (NULL); 4415 } 4416 4417 /* Avoid allocs targeting empty domains. */ 4418 if (domain != UMA_ANYDOMAIN && VM_DOMAIN_EMPTY(domain)) 4419 domain = UMA_ANYDOMAIN; 4420 4421 if (zone->uz_import(zone->uz_arg, &item, 1, domain, flags) != 1) 4422 goto fail_cnt; 4423 4424 /* 4425 * We have to call both the zone's init (not the keg's init) 4426 * and the zone's ctor. This is because the item is going from 4427 * a keg slab directly to the user, and the user is expecting it 4428 * to be both zone-init'd as well as zone-ctor'd. 4429 */ 4430 if (zone->uz_init != NULL) { 4431 int error; 4432 4433 kasan_mark_item_valid(zone, item); 4434 error = zone->uz_init(item, zone->uz_size, flags); 4435 kasan_mark_item_invalid(zone, item); 4436 if (error != 0) { 4437 zone_free_item(zone, item, udata, SKIP_FINI | SKIP_CNT); 4438 goto fail_cnt; 4439 } 4440 } 4441 item = item_ctor(zone, zone->uz_flags, zone->uz_size, udata, flags, 4442 item); 4443 if (item == NULL) 4444 goto fail; 4445 4446 counter_u64_add(zone->uz_allocs, 1); 4447 CTR3(KTR_UMA, "zone_alloc_item item %p from %s(%p)", item, 4448 zone->uz_name, zone); 4449 4450 return (item); 4451 4452 fail_cnt: 4453 counter_u64_add(zone->uz_fails, 1); 4454 fail: 4455 if (zone->uz_max_items > 0) 4456 zone_free_limit(zone, 1); 4457 CTR2(KTR_UMA, "zone_alloc_item failed from %s(%p)", 4458 zone->uz_name, zone); 4459 4460 return (NULL); 4461 } 4462 4463 /* 4464 * Try to free an item to the per-CPU cache, promoting its quick reuse. 4465 */ 4466 static __always_inline bool 4467 cache_free_reuse(uma_zone_t zone, int uz_flags, void *item, void *udata) 4468 { 4469 uma_cache_t cache; 4470 int itemdomain; 4471 4472 /* 4473 * If possible, free to the per-CPU cache. There are two 4474 * requirements for safe access to the per-CPU cache: (1) the thread 4475 * accessing the cache must not be preempted or yield during access, 4476 * and (2) the thread must not migrate CPUs without switching which 4477 * cache it accesses. We rely on a critical section to prevent 4478 * preemption and migration. We release the critical section in 4479 * order to acquire the zone mutex if we are unable to free to the 4480 * current cache; when we re-acquire the critical section, we must 4481 * detect and handle migration if it has occurred. 4482 */ 4483 itemdomain = 0; 4484 #ifdef NUMA 4485 if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 4486 itemdomain = item_domain(item); 4487 #endif 4488 4489 critical_enter(); 4490 do { 4491 uma_cache_bucket_t bucket; 4492 4493 cache = &zone->uz_cpu[curcpu]; 4494 /* 4495 * Try to free into the allocbucket first to give LIFO 4496 * ordering for cache-hot datastructures. Spill over 4497 * into the freebucket if necessary. Alloc will swap 4498 * them if one runs dry. 4499 */ 4500 bucket = &cache->uc_allocbucket; 4501 #ifdef NUMA 4502 if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4503 PCPU_GET(domain) != itemdomain) { 4504 bucket = &cache->uc_crossbucket; 4505 } else 4506 #endif 4507 if (bucket->ucb_cnt == bucket->ucb_entries && 4508 cache->uc_freebucket.ucb_cnt < 4509 cache->uc_freebucket.ucb_entries) 4510 cache_bucket_swap(&cache->uc_freebucket, 4511 &cache->uc_allocbucket); 4512 if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { 4513 cache_bucket_push(cache, bucket, item); 4514 critical_exit(); 4515 return (true); 4516 } 4517 } while (cache_free(zone, cache, udata, itemdomain)); 4518 critical_exit(); 4519 4520 return (false); 4521 } 4522 4523 /* 4524 * Try to free an object to the per-CPU cache, deferring its reuse. This is 4525 * used by the SMR-protected allocator, which cannot reuse the item until 4526 * smr_poll() guarantees that no threads are still accessing it, and by 4527 * sanitizers, which wish to defer reuse to make UAF detection more effective. 4528 */ 4529 static __always_inline bool 4530 cache_free_defer(uma_zone_t zone, void *item, void *udata) 4531 { 4532 uma_cache_t cache; 4533 int itemdomain; 4534 #ifdef NUMA 4535 int uz_flags; 4536 #endif 4537 4538 itemdomain = 0; 4539 #ifdef NUMA 4540 uz_flags = cache_uz_flags(&zone->uz_cpu[curcpu]); 4541 if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0) 4542 itemdomain = item_domain(item); 4543 #endif 4544 critical_enter(); 4545 do { 4546 uma_cache_bucket_t bucket; 4547 4548 cache = &zone->uz_cpu[curcpu]; 4549 bucket = &cache->uc_freebucket; 4550 #ifdef NUMA 4551 if ((uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4552 PCPU_GET(domain) != itemdomain) { 4553 bucket = &cache->uc_crossbucket; 4554 } 4555 #endif 4556 if (__predict_true(bucket->ucb_cnt < bucket->ucb_entries)) { 4557 cache_bucket_push(cache, bucket, item); 4558 critical_exit(); 4559 return (true); 4560 } 4561 } while (cache_free(zone, cache, udata, itemdomain)); 4562 critical_exit(); 4563 4564 return (false); 4565 } 4566 4567 /* See uma.h */ 4568 void 4569 uma_zfree_smr(uma_zone_t zone, void *item) 4570 { 4571 CTR3(KTR_UMA, "uma_zfree_smr zone %s(%p) item %p", 4572 zone->uz_name, zone, item); 4573 4574 #ifdef UMA_ZALLOC_DEBUG 4575 KASSERT((zone->uz_flags & UMA_ZONE_SMR) != 0, 4576 ("uma_zfree_smr: called with non-SMR zone.")); 4577 KASSERT(item != NULL, ("uma_zfree_smr: Called with NULL pointer.")); 4578 SMR_ASSERT_NOT_ENTERED(zone->uz_smr); 4579 if (uma_zfree_debug(zone, item, NULL) == EJUSTRETURN) 4580 return; 4581 #endif 4582 4583 if (cache_free_defer(zone, item, NULL)) 4584 return; 4585 4586 /* 4587 * If nothing else caught this, we'll just do an internal free. 4588 */ 4589 zone_free_item(zone, item, NULL, SKIP_NONE); 4590 } 4591 4592 /* See uma.h */ 4593 void 4594 uma_zfree_arg(uma_zone_t zone, void *item, void *udata) 4595 { 4596 uma_cache_t cache; 4597 int uz_flags; 4598 4599 /* Enable entropy collection for RANDOM_ENABLE_UMA kernel option */ 4600 random_harvest_fast_uma(&zone, sizeof(zone), RANDOM_UMA); 4601 4602 CTR3(KTR_UMA, "uma_zfree_arg zone %s(%p) item %p", 4603 zone->uz_name, zone, item); 4604 4605 #ifdef UMA_ZALLOC_DEBUG 4606 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 4607 ("uma_zfree_arg: called with SMR zone.")); 4608 if (uma_zfree_debug(zone, item, udata) == EJUSTRETURN) 4609 return; 4610 #endif 4611 /* uma_zfree(..., NULL) does nothing, to match free(9). */ 4612 if (item == NULL) 4613 return; 4614 4615 /* 4616 * We are accessing the per-cpu cache without a critical section to 4617 * fetch size and flags. This is acceptable, if we are preempted we 4618 * will simply read another cpu's line. 4619 */ 4620 cache = &zone->uz_cpu[curcpu]; 4621 uz_flags = cache_uz_flags(cache); 4622 if (UMA_ALWAYS_CTORDTOR || 4623 __predict_false((uz_flags & UMA_ZFLAG_CTORDTOR) != 0)) 4624 item_dtor(zone, item, cache_uz_size(cache), udata, SKIP_NONE); 4625 4626 /* 4627 * The race here is acceptable. If we miss it we'll just have to wait 4628 * a little longer for the limits to be reset. 4629 */ 4630 if (__predict_false(uz_flags & UMA_ZFLAG_LIMIT) && 4631 atomic_load_32(&zone->uz_sleepers) > 0) { 4632 /* We will free directly to the zone. */ 4633 } 4634 #ifdef KASAN 4635 else if ((uz_flags & UMA_ZONE_NOKASAN) == 0) { 4636 if (cache_free_defer(zone, item, udata)) 4637 return; 4638 } 4639 #endif 4640 else if (cache_free_reuse(zone, uz_flags, item, udata)) { 4641 return; 4642 } 4643 4644 /* 4645 * If nothing else caught this, we'll just do an internal free. 4646 */ 4647 zone_free_item(zone, item, udata, SKIP_DTOR); 4648 } 4649 4650 #ifdef NUMA 4651 /* 4652 * sort crossdomain free buckets to domain correct buckets and cache 4653 * them. 4654 */ 4655 static void 4656 zone_free_cross(uma_zone_t zone, uma_bucket_t bucket, void *udata) 4657 { 4658 struct uma_bucketlist emptybuckets, fullbuckets; 4659 uma_zone_domain_t zdom; 4660 uma_bucket_t b; 4661 smr_seq_t seq; 4662 void *item; 4663 int domain; 4664 4665 CTR3(KTR_UMA, 4666 "uma_zfree: zone %s(%p) draining cross bucket %p", 4667 zone->uz_name, zone, bucket); 4668 4669 /* 4670 * It is possible for buckets to arrive here out of order so we fetch 4671 * the current smr seq rather than accepting the bucket's. 4672 */ 4673 seq = SMR_SEQ_INVALID; 4674 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) 4675 seq = smr_advance(zone->uz_smr); 4676 4677 /* 4678 * To avoid having ndomain * ndomain buckets for sorting we have a 4679 * lock on the current crossfree bucket. A full matrix with 4680 * per-domain locking could be used if necessary. 4681 */ 4682 STAILQ_INIT(&emptybuckets); 4683 STAILQ_INIT(&fullbuckets); 4684 ZONE_CROSS_LOCK(zone); 4685 for (; bucket->ub_cnt > 0; bucket->ub_cnt--) { 4686 item = bucket->ub_bucket[bucket->ub_cnt - 1]; 4687 domain = item_domain(item); 4688 zdom = ZDOM_GET(zone, domain); 4689 if (zdom->uzd_cross == NULL) { 4690 if ((b = STAILQ_FIRST(&emptybuckets)) != NULL) { 4691 STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); 4692 zdom->uzd_cross = b; 4693 } else { 4694 /* 4695 * Avoid allocating a bucket with the cross lock 4696 * held, since allocation can trigger a 4697 * cross-domain free and bucket zones may 4698 * allocate from each other. 4699 */ 4700 ZONE_CROSS_UNLOCK(zone); 4701 b = bucket_alloc(zone, udata, M_NOWAIT); 4702 if (b == NULL) 4703 goto out; 4704 ZONE_CROSS_LOCK(zone); 4705 if (zdom->uzd_cross != NULL) { 4706 STAILQ_INSERT_HEAD(&emptybuckets, b, 4707 ub_link); 4708 } else { 4709 zdom->uzd_cross = b; 4710 } 4711 } 4712 } 4713 b = zdom->uzd_cross; 4714 b->ub_bucket[b->ub_cnt++] = item; 4715 b->ub_seq = seq; 4716 if (b->ub_cnt == b->ub_entries) { 4717 STAILQ_INSERT_HEAD(&fullbuckets, b, ub_link); 4718 if ((b = STAILQ_FIRST(&emptybuckets)) != NULL) 4719 STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); 4720 zdom->uzd_cross = b; 4721 } 4722 } 4723 ZONE_CROSS_UNLOCK(zone); 4724 out: 4725 if (bucket->ub_cnt == 0) 4726 bucket->ub_seq = SMR_SEQ_INVALID; 4727 bucket_free(zone, bucket, udata); 4728 4729 while ((b = STAILQ_FIRST(&emptybuckets)) != NULL) { 4730 STAILQ_REMOVE_HEAD(&emptybuckets, ub_link); 4731 bucket_free(zone, b, udata); 4732 } 4733 while ((b = STAILQ_FIRST(&fullbuckets)) != NULL) { 4734 STAILQ_REMOVE_HEAD(&fullbuckets, ub_link); 4735 domain = item_domain(b->ub_bucket[0]); 4736 zone_put_bucket(zone, domain, b, udata, true); 4737 } 4738 } 4739 #endif 4740 4741 static void 4742 zone_free_bucket(uma_zone_t zone, uma_bucket_t bucket, void *udata, 4743 int itemdomain, bool ws) 4744 { 4745 4746 #ifdef NUMA 4747 /* 4748 * Buckets coming from the wrong domain will be entirely for the 4749 * only other domain on two domain systems. In this case we can 4750 * simply cache them. Otherwise we need to sort them back to 4751 * correct domains. 4752 */ 4753 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && 4754 vm_ndomains > 2 && PCPU_GET(domain) != itemdomain) { 4755 zone_free_cross(zone, bucket, udata); 4756 return; 4757 } 4758 #endif 4759 4760 /* 4761 * Attempt to save the bucket in the zone's domain bucket cache. 4762 */ 4763 CTR3(KTR_UMA, 4764 "uma_zfree: zone %s(%p) putting bucket %p on free list", 4765 zone->uz_name, zone, bucket); 4766 /* ub_cnt is pointing to the last free item */ 4767 if ((zone->uz_flags & UMA_ZONE_ROUNDROBIN) != 0) 4768 itemdomain = zone_domain_lowest(zone, itemdomain); 4769 zone_put_bucket(zone, itemdomain, bucket, udata, ws); 4770 } 4771 4772 /* 4773 * Populate a free or cross bucket for the current cpu cache. Free any 4774 * existing full bucket either to the zone cache or back to the slab layer. 4775 * 4776 * Enters and returns in a critical section. false return indicates that 4777 * we can not satisfy this free in the cache layer. true indicates that 4778 * the caller should retry. 4779 */ 4780 static __noinline bool 4781 cache_free(uma_zone_t zone, uma_cache_t cache, void *udata, int itemdomain) 4782 { 4783 uma_cache_bucket_t cbucket; 4784 uma_bucket_t newbucket, bucket; 4785 4786 CRITICAL_ASSERT(curthread); 4787 4788 if (zone->uz_bucket_size == 0) 4789 return false; 4790 4791 cache = &zone->uz_cpu[curcpu]; 4792 newbucket = NULL; 4793 4794 /* 4795 * FIRSTTOUCH domains need to free to the correct zdom. When 4796 * enabled this is the zdom of the item. The bucket is the 4797 * cross bucket if the current domain and itemdomain do not match. 4798 */ 4799 cbucket = &cache->uc_freebucket; 4800 #ifdef NUMA 4801 if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) { 4802 if (PCPU_GET(domain) != itemdomain) { 4803 cbucket = &cache->uc_crossbucket; 4804 if (cbucket->ucb_cnt != 0) 4805 counter_u64_add(zone->uz_xdomain, 4806 cbucket->ucb_cnt); 4807 } 4808 } 4809 #endif 4810 bucket = cache_bucket_unload(cbucket); 4811 KASSERT(bucket == NULL || bucket->ub_cnt == bucket->ub_entries, 4812 ("cache_free: Entered with non-full free bucket.")); 4813 4814 /* We are no longer associated with this CPU. */ 4815 critical_exit(); 4816 4817 /* 4818 * Don't let SMR zones operate without a free bucket. Force 4819 * a synchronize and re-use this one. We will only degrade 4820 * to a synchronize every bucket_size items rather than every 4821 * item if we fail to allocate a bucket. 4822 */ 4823 if ((zone->uz_flags & UMA_ZONE_SMR) != 0) { 4824 if (bucket != NULL) 4825 bucket->ub_seq = smr_advance(zone->uz_smr); 4826 newbucket = bucket_alloc(zone, udata, M_NOWAIT); 4827 if (newbucket == NULL && bucket != NULL) { 4828 bucket_drain(zone, bucket); 4829 newbucket = bucket; 4830 bucket = NULL; 4831 } 4832 } else if (!bucketdisable) 4833 newbucket = bucket_alloc(zone, udata, M_NOWAIT); 4834 4835 if (bucket != NULL) 4836 zone_free_bucket(zone, bucket, udata, itemdomain, true); 4837 4838 critical_enter(); 4839 if ((bucket = newbucket) == NULL) 4840 return (false); 4841 cache = &zone->uz_cpu[curcpu]; 4842 #ifdef NUMA 4843 /* 4844 * Check to see if we should be populating the cross bucket. If it 4845 * is already populated we will fall through and attempt to populate 4846 * the free bucket. 4847 */ 4848 if ((cache_uz_flags(cache) & UMA_ZONE_FIRSTTOUCH) != 0) { 4849 if (PCPU_GET(domain) != itemdomain && 4850 cache->uc_crossbucket.ucb_bucket == NULL) { 4851 cache_bucket_load_cross(cache, bucket); 4852 return (true); 4853 } 4854 } 4855 #endif 4856 /* 4857 * We may have lost the race to fill the bucket or switched CPUs. 4858 */ 4859 if (cache->uc_freebucket.ucb_bucket != NULL) { 4860 critical_exit(); 4861 bucket_free(zone, bucket, udata); 4862 critical_enter(); 4863 } else 4864 cache_bucket_load_free(cache, bucket); 4865 4866 return (true); 4867 } 4868 4869 static void 4870 slab_free_item(uma_zone_t zone, uma_slab_t slab, void *item) 4871 { 4872 uma_keg_t keg; 4873 uma_domain_t dom; 4874 int freei; 4875 4876 keg = zone->uz_keg; 4877 KEG_LOCK_ASSERT(keg, slab->us_domain); 4878 4879 /* Do we need to remove from any lists? */ 4880 dom = &keg->uk_domain[slab->us_domain]; 4881 if (slab->us_freecount + 1 == keg->uk_ipers) { 4882 LIST_REMOVE(slab, us_link); 4883 LIST_INSERT_HEAD(&dom->ud_free_slab, slab, us_link); 4884 dom->ud_free_slabs++; 4885 } else if (slab->us_freecount == 0) { 4886 LIST_REMOVE(slab, us_link); 4887 LIST_INSERT_HEAD(&dom->ud_part_slab, slab, us_link); 4888 } 4889 4890 /* Slab management. */ 4891 freei = slab_item_index(slab, keg, item); 4892 BIT_SET(keg->uk_ipers, freei, &slab->us_free); 4893 slab->us_freecount++; 4894 4895 /* Keg statistics. */ 4896 dom->ud_free_items++; 4897 } 4898 4899 static void 4900 zone_release(void *arg, void **bucket, int cnt) 4901 { 4902 struct mtx *lock; 4903 uma_zone_t zone; 4904 uma_slab_t slab; 4905 uma_keg_t keg; 4906 uint8_t *mem; 4907 void *item; 4908 int i; 4909 4910 zone = arg; 4911 keg = zone->uz_keg; 4912 lock = NULL; 4913 if (__predict_false((zone->uz_flags & UMA_ZFLAG_HASH) != 0)) 4914 lock = KEG_LOCK(keg, 0); 4915 for (i = 0; i < cnt; i++) { 4916 item = bucket[i]; 4917 if (__predict_true((zone->uz_flags & UMA_ZFLAG_VTOSLAB) != 0)) { 4918 slab = vtoslab((vm_offset_t)item); 4919 } else { 4920 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 4921 if ((zone->uz_flags & UMA_ZFLAG_HASH) != 0) 4922 slab = hash_sfind(&keg->uk_hash, mem); 4923 else 4924 slab = (uma_slab_t)(mem + keg->uk_pgoff); 4925 } 4926 if (lock != KEG_LOCKPTR(keg, slab->us_domain)) { 4927 if (lock != NULL) 4928 mtx_unlock(lock); 4929 lock = KEG_LOCK(keg, slab->us_domain); 4930 } 4931 slab_free_item(zone, slab, item); 4932 } 4933 if (lock != NULL) 4934 mtx_unlock(lock); 4935 } 4936 4937 /* 4938 * Frees a single item to any zone. 4939 * 4940 * Arguments: 4941 * zone The zone to free to 4942 * item The item we're freeing 4943 * udata User supplied data for the dtor 4944 * skip Skip dtors and finis 4945 */ 4946 static __noinline void 4947 zone_free_item(uma_zone_t zone, void *item, void *udata, enum zfreeskip skip) 4948 { 4949 4950 /* 4951 * If a free is sent directly to an SMR zone we have to 4952 * synchronize immediately because the item can instantly 4953 * be reallocated. This should only happen in degenerate 4954 * cases when no memory is available for per-cpu caches. 4955 */ 4956 if ((zone->uz_flags & UMA_ZONE_SMR) != 0 && skip == SKIP_NONE) 4957 smr_synchronize(zone->uz_smr); 4958 4959 item_dtor(zone, item, zone->uz_size, udata, skip); 4960 4961 if (skip < SKIP_FINI && zone->uz_fini) { 4962 kasan_mark_item_valid(zone, item); 4963 zone->uz_fini(item, zone->uz_size); 4964 kasan_mark_item_invalid(zone, item); 4965 } 4966 4967 zone->uz_release(zone->uz_arg, &item, 1); 4968 4969 if (skip & SKIP_CNT) 4970 return; 4971 4972 counter_u64_add(zone->uz_frees, 1); 4973 4974 if (zone->uz_max_items > 0) 4975 zone_free_limit(zone, 1); 4976 } 4977 4978 /* See uma.h */ 4979 int 4980 uma_zone_set_max(uma_zone_t zone, int nitems) 4981 { 4982 4983 /* 4984 * If the limit is small, we may need to constrain the maximum per-CPU 4985 * cache size, or disable caching entirely. 4986 */ 4987 uma_zone_set_maxcache(zone, nitems); 4988 4989 /* 4990 * XXX This can misbehave if the zone has any allocations with 4991 * no limit and a limit is imposed. There is currently no 4992 * way to clear a limit. 4993 */ 4994 ZONE_LOCK(zone); 4995 if (zone->uz_max_items == 0) 4996 ZONE_ASSERT_COLD(zone); 4997 zone->uz_max_items = nitems; 4998 zone->uz_flags |= UMA_ZFLAG_LIMIT; 4999 zone_update_caches(zone); 5000 /* We may need to wake waiters. */ 5001 wakeup(&zone->uz_max_items); 5002 ZONE_UNLOCK(zone); 5003 5004 return (nitems); 5005 } 5006 5007 /* See uma.h */ 5008 void 5009 uma_zone_set_maxcache(uma_zone_t zone, int nitems) 5010 { 5011 int bpcpu, bpdom, bsize, nb; 5012 5013 ZONE_LOCK(zone); 5014 5015 /* 5016 * Compute a lower bound on the number of items that may be cached in 5017 * the zone. Each CPU gets at least two buckets, and for cross-domain 5018 * frees we use an additional bucket per CPU and per domain. Select the 5019 * largest bucket size that does not exceed half of the requested limit, 5020 * with the left over space given to the full bucket cache. 5021 */ 5022 bpdom = 0; 5023 bpcpu = 2; 5024 #ifdef NUMA 5025 if ((zone->uz_flags & UMA_ZONE_FIRSTTOUCH) != 0 && vm_ndomains > 1) { 5026 bpcpu++; 5027 bpdom++; 5028 } 5029 #endif 5030 nb = bpcpu * mp_ncpus + bpdom * vm_ndomains; 5031 bsize = nitems / nb / 2; 5032 if (bsize > BUCKET_MAX) 5033 bsize = BUCKET_MAX; 5034 else if (bsize == 0 && nitems / nb > 0) 5035 bsize = 1; 5036 zone->uz_bucket_size_max = zone->uz_bucket_size = bsize; 5037 if (zone->uz_bucket_size_min > zone->uz_bucket_size_max) 5038 zone->uz_bucket_size_min = zone->uz_bucket_size_max; 5039 zone->uz_bucket_max = nitems - nb * bsize; 5040 ZONE_UNLOCK(zone); 5041 } 5042 5043 /* See uma.h */ 5044 int 5045 uma_zone_get_max(uma_zone_t zone) 5046 { 5047 int nitems; 5048 5049 nitems = atomic_load_64(&zone->uz_max_items); 5050 5051 return (nitems); 5052 } 5053 5054 /* See uma.h */ 5055 void 5056 uma_zone_set_warning(uma_zone_t zone, const char *warning) 5057 { 5058 5059 ZONE_ASSERT_COLD(zone); 5060 zone->uz_warning = warning; 5061 } 5062 5063 /* See uma.h */ 5064 void 5065 uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction) 5066 { 5067 5068 ZONE_ASSERT_COLD(zone); 5069 TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone); 5070 } 5071 5072 /* See uma.h */ 5073 int 5074 uma_zone_get_cur(uma_zone_t zone) 5075 { 5076 int64_t nitems; 5077 u_int i; 5078 5079 nitems = 0; 5080 if (zone->uz_allocs != EARLY_COUNTER && zone->uz_frees != EARLY_COUNTER) 5081 nitems = counter_u64_fetch(zone->uz_allocs) - 5082 counter_u64_fetch(zone->uz_frees); 5083 CPU_FOREACH(i) 5084 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs) - 5085 atomic_load_64(&zone->uz_cpu[i].uc_frees); 5086 5087 return (nitems < 0 ? 0 : nitems); 5088 } 5089 5090 static uint64_t 5091 uma_zone_get_allocs(uma_zone_t zone) 5092 { 5093 uint64_t nitems; 5094 u_int i; 5095 5096 nitems = 0; 5097 if (zone->uz_allocs != EARLY_COUNTER) 5098 nitems = counter_u64_fetch(zone->uz_allocs); 5099 CPU_FOREACH(i) 5100 nitems += atomic_load_64(&zone->uz_cpu[i].uc_allocs); 5101 5102 return (nitems); 5103 } 5104 5105 static uint64_t 5106 uma_zone_get_frees(uma_zone_t zone) 5107 { 5108 uint64_t nitems; 5109 u_int i; 5110 5111 nitems = 0; 5112 if (zone->uz_frees != EARLY_COUNTER) 5113 nitems = counter_u64_fetch(zone->uz_frees); 5114 CPU_FOREACH(i) 5115 nitems += atomic_load_64(&zone->uz_cpu[i].uc_frees); 5116 5117 return (nitems); 5118 } 5119 5120 #ifdef INVARIANTS 5121 /* Used only for KEG_ASSERT_COLD(). */ 5122 static uint64_t 5123 uma_keg_get_allocs(uma_keg_t keg) 5124 { 5125 uma_zone_t z; 5126 uint64_t nitems; 5127 5128 nitems = 0; 5129 LIST_FOREACH(z, &keg->uk_zones, uz_link) 5130 nitems += uma_zone_get_allocs(z); 5131 5132 return (nitems); 5133 } 5134 #endif 5135 5136 /* See uma.h */ 5137 void 5138 uma_zone_set_init(uma_zone_t zone, uma_init uminit) 5139 { 5140 uma_keg_t keg; 5141 5142 KEG_GET(zone, keg); 5143 KEG_ASSERT_COLD(keg); 5144 keg->uk_init = uminit; 5145 } 5146 5147 /* See uma.h */ 5148 void 5149 uma_zone_set_fini(uma_zone_t zone, uma_fini fini) 5150 { 5151 uma_keg_t keg; 5152 5153 KEG_GET(zone, keg); 5154 KEG_ASSERT_COLD(keg); 5155 keg->uk_fini = fini; 5156 } 5157 5158 /* See uma.h */ 5159 void 5160 uma_zone_set_zinit(uma_zone_t zone, uma_init zinit) 5161 { 5162 5163 ZONE_ASSERT_COLD(zone); 5164 zone->uz_init = zinit; 5165 } 5166 5167 /* See uma.h */ 5168 void 5169 uma_zone_set_zfini(uma_zone_t zone, uma_fini zfini) 5170 { 5171 5172 ZONE_ASSERT_COLD(zone); 5173 zone->uz_fini = zfini; 5174 } 5175 5176 /* See uma.h */ 5177 void 5178 uma_zone_set_freef(uma_zone_t zone, uma_free freef) 5179 { 5180 uma_keg_t keg; 5181 5182 KEG_GET(zone, keg); 5183 KEG_ASSERT_COLD(keg); 5184 keg->uk_freef = freef; 5185 } 5186 5187 /* See uma.h */ 5188 void 5189 uma_zone_set_allocf(uma_zone_t zone, uma_alloc allocf) 5190 { 5191 uma_keg_t keg; 5192 5193 KEG_GET(zone, keg); 5194 KEG_ASSERT_COLD(keg); 5195 keg->uk_allocf = allocf; 5196 } 5197 5198 /* See uma.h */ 5199 void 5200 uma_zone_set_smr(uma_zone_t zone, smr_t smr) 5201 { 5202 5203 ZONE_ASSERT_COLD(zone); 5204 5205 KASSERT(smr != NULL, ("Got NULL smr")); 5206 KASSERT((zone->uz_flags & UMA_ZONE_SMR) == 0, 5207 ("zone %p (%s) already uses SMR", zone, zone->uz_name)); 5208 zone->uz_flags |= UMA_ZONE_SMR; 5209 zone->uz_smr = smr; 5210 zone_update_caches(zone); 5211 } 5212 5213 smr_t 5214 uma_zone_get_smr(uma_zone_t zone) 5215 { 5216 5217 return (zone->uz_smr); 5218 } 5219 5220 /* See uma.h */ 5221 void 5222 uma_zone_reserve(uma_zone_t zone, int items) 5223 { 5224 uma_keg_t keg; 5225 5226 KEG_GET(zone, keg); 5227 KEG_ASSERT_COLD(keg); 5228 keg->uk_reserve = items; 5229 } 5230 5231 /* See uma.h */ 5232 int 5233 uma_zone_reserve_kva(uma_zone_t zone, int count) 5234 { 5235 uma_keg_t keg; 5236 void *kva; 5237 u_int pages; 5238 5239 KEG_GET(zone, keg); 5240 KEG_ASSERT_COLD(keg); 5241 ZONE_ASSERT_COLD(zone); 5242 5243 pages = howmany(count, keg->uk_ipers) * keg->uk_ppera; 5244 5245 #ifdef UMA_USE_DMAP 5246 if (keg->uk_ppera > 1) { 5247 #else 5248 if (1) { 5249 #endif 5250 kva = kva_alloc((vm_size_t)pages * PAGE_SIZE); 5251 if (kva == NULL) 5252 return (0); 5253 } else 5254 kva = NULL; 5255 5256 MPASS(keg->uk_kva == NULL); 5257 keg->uk_kva = kva; 5258 keg->uk_offset = 0; 5259 zone->uz_max_items = pages * keg->uk_ipers; 5260 #ifdef UMA_USE_DMAP 5261 keg->uk_allocf = (keg->uk_ppera > 1) ? noobj_alloc : uma_small_alloc; 5262 #else 5263 keg->uk_allocf = noobj_alloc; 5264 #endif 5265 keg->uk_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; 5266 zone->uz_flags |= UMA_ZFLAG_LIMIT | UMA_ZONE_NOFREE; 5267 zone_update_caches(zone); 5268 5269 return (1); 5270 } 5271 5272 /* See uma.h */ 5273 void 5274 uma_prealloc(uma_zone_t zone, int items) 5275 { 5276 struct vm_domainset_iter di; 5277 uma_domain_t dom; 5278 uma_slab_t slab; 5279 uma_keg_t keg; 5280 int aflags, domain, slabs; 5281 5282 KEG_GET(zone, keg); 5283 slabs = howmany(items, keg->uk_ipers); 5284 while (slabs-- > 0) { 5285 aflags = M_WAITOK; 5286 if (vm_domainset_iter_policy_ref_init(&di, &keg->uk_dr, &domain, 5287 &aflags) != 0) 5288 panic("%s: Domainset is empty", __func__); 5289 for (;;) { 5290 slab = keg_alloc_slab(keg, zone, domain, M_WAITOK, 5291 aflags); 5292 if (slab != NULL) { 5293 dom = &keg->uk_domain[slab->us_domain]; 5294 /* 5295 * keg_alloc_slab() always returns a slab on the 5296 * partial list. 5297 */ 5298 LIST_REMOVE(slab, us_link); 5299 LIST_INSERT_HEAD(&dom->ud_free_slab, slab, 5300 us_link); 5301 dom->ud_free_slabs++; 5302 KEG_UNLOCK(keg, slab->us_domain); 5303 break; 5304 } 5305 if (vm_domainset_iter_policy(&di, &domain) != 0) 5306 panic("%s: Cannot allocate from any domain", 5307 __func__); 5308 } 5309 } 5310 } 5311 5312 /* 5313 * Returns a snapshot of memory consumption in bytes. 5314 */ 5315 size_t 5316 uma_zone_memory(uma_zone_t zone) 5317 { 5318 size_t sz; 5319 int i; 5320 5321 sz = 0; 5322 if (zone->uz_flags & UMA_ZFLAG_CACHE) { 5323 for (i = 0; i < vm_ndomains; i++) 5324 sz += ZDOM_GET(zone, i)->uzd_nitems; 5325 return (sz * zone->uz_size); 5326 } 5327 for (i = 0; i < vm_ndomains; i++) 5328 sz += zone->uz_keg->uk_domain[i].ud_pages; 5329 5330 return (sz * PAGE_SIZE); 5331 } 5332 5333 struct uma_reclaim_args { 5334 int domain; 5335 int req; 5336 }; 5337 5338 static void 5339 uma_reclaim_domain_cb(uma_zone_t zone, void *arg) 5340 { 5341 struct uma_reclaim_args *args; 5342 5343 args = arg; 5344 if ((zone->uz_flags & UMA_ZONE_UNMANAGED) != 0) 5345 return; 5346 if ((args->req == UMA_RECLAIM_TRIM) && 5347 (zone->uz_flags & UMA_ZONE_NOTRIM) !=0) 5348 return; 5349 5350 uma_zone_reclaim_domain(zone, args->req, args->domain); 5351 } 5352 5353 /* See uma.h */ 5354 void 5355 uma_reclaim(int req) 5356 { 5357 uma_reclaim_domain(req, UMA_ANYDOMAIN); 5358 } 5359 5360 void 5361 uma_reclaim_domain(int req, int domain) 5362 { 5363 struct uma_reclaim_args args; 5364 5365 bucket_enable(); 5366 5367 args.domain = domain; 5368 args.req = req; 5369 5370 sx_slock(&uma_reclaim_lock); 5371 switch (req) { 5372 case UMA_RECLAIM_TRIM: 5373 case UMA_RECLAIM_DRAIN: 5374 zone_foreach(uma_reclaim_domain_cb, &args); 5375 break; 5376 case UMA_RECLAIM_DRAIN_CPU: 5377 /* 5378 * Reclaim globally visible free items from all zones, then drain 5379 * per-CPU buckets, then reclaim items freed while draining. 5380 * This approach minimizes expensive context switching needed to 5381 * drain each zone's per-CPU buckets. 5382 */ 5383 args.req = UMA_RECLAIM_DRAIN; 5384 zone_foreach(uma_reclaim_domain_cb, &args); 5385 pcpu_cache_drain_safe(NULL); 5386 zone_foreach(uma_reclaim_domain_cb, &args); 5387 break; 5388 default: 5389 panic("unhandled reclamation request %d", req); 5390 } 5391 5392 /* 5393 * Some slabs may have been freed but this zone will be visited early 5394 * we visit again so that we can free pages that are empty once other 5395 * zones are drained. We have to do the same for buckets. 5396 */ 5397 uma_zone_reclaim_domain(slabzones[0], UMA_RECLAIM_DRAIN, domain); 5398 uma_zone_reclaim_domain(slabzones[1], UMA_RECLAIM_DRAIN, domain); 5399 bucket_zone_drain(domain); 5400 sx_sunlock(&uma_reclaim_lock); 5401 } 5402 5403 static volatile int uma_reclaim_needed; 5404 5405 void 5406 uma_reclaim_wakeup(void) 5407 { 5408 5409 if (atomic_fetchadd_int(&uma_reclaim_needed, 1) == 0) 5410 wakeup(uma_reclaim); 5411 } 5412 5413 void 5414 uma_reclaim_worker(void *arg __unused) 5415 { 5416 5417 for (;;) { 5418 sx_xlock(&uma_reclaim_lock); 5419 while (atomic_load_int(&uma_reclaim_needed) == 0) 5420 sx_sleep(uma_reclaim, &uma_reclaim_lock, PVM, "umarcl", 5421 hz); 5422 sx_xunlock(&uma_reclaim_lock); 5423 EVENTHANDLER_INVOKE(vm_lowmem, VM_LOW_KMEM); 5424 uma_reclaim(UMA_RECLAIM_DRAIN_CPU); 5425 atomic_store_int(&uma_reclaim_needed, 0); 5426 /* Don't fire more than once per-second. */ 5427 pause("umarclslp", hz); 5428 } 5429 } 5430 5431 /* See uma.h */ 5432 void 5433 uma_zone_reclaim(uma_zone_t zone, int req) 5434 { 5435 uma_zone_reclaim_domain(zone, req, UMA_ANYDOMAIN); 5436 } 5437 5438 void 5439 uma_zone_reclaim_domain(uma_zone_t zone, int req, int domain) 5440 { 5441 switch (req) { 5442 case UMA_RECLAIM_TRIM: 5443 zone_reclaim(zone, domain, M_NOWAIT, false); 5444 break; 5445 case UMA_RECLAIM_DRAIN: 5446 zone_reclaim(zone, domain, M_NOWAIT, true); 5447 break; 5448 case UMA_RECLAIM_DRAIN_CPU: 5449 pcpu_cache_drain_safe(zone); 5450 zone_reclaim(zone, domain, M_NOWAIT, true); 5451 break; 5452 default: 5453 panic("unhandled reclamation request %d", req); 5454 } 5455 } 5456 5457 /* See uma.h */ 5458 int 5459 uma_zone_exhausted(uma_zone_t zone) 5460 { 5461 5462 return (atomic_load_32(&zone->uz_sleepers) > 0); 5463 } 5464 5465 unsigned long 5466 uma_limit(void) 5467 { 5468 5469 return (uma_kmem_limit); 5470 } 5471 5472 void 5473 uma_set_limit(unsigned long limit) 5474 { 5475 5476 uma_kmem_limit = limit; 5477 } 5478 5479 unsigned long 5480 uma_size(void) 5481 { 5482 5483 return (atomic_load_long(&uma_kmem_total)); 5484 } 5485 5486 long 5487 uma_avail(void) 5488 { 5489 5490 return (uma_kmem_limit - uma_size()); 5491 } 5492 5493 #ifdef DDB 5494 /* 5495 * Generate statistics across both the zone and its per-cpu cache's. Return 5496 * desired statistics if the pointer is non-NULL for that statistic. 5497 * 5498 * Note: does not update the zone statistics, as it can't safely clear the 5499 * per-CPU cache statistic. 5500 * 5501 */ 5502 static void 5503 uma_zone_sumstat(uma_zone_t z, long *cachefreep, uint64_t *allocsp, 5504 uint64_t *freesp, uint64_t *sleepsp, uint64_t *xdomainp) 5505 { 5506 uma_cache_t cache; 5507 uint64_t allocs, frees, sleeps, xdomain; 5508 int cachefree, cpu; 5509 5510 allocs = frees = sleeps = xdomain = 0; 5511 cachefree = 0; 5512 CPU_FOREACH(cpu) { 5513 cache = &z->uz_cpu[cpu]; 5514 cachefree += cache->uc_allocbucket.ucb_cnt; 5515 cachefree += cache->uc_freebucket.ucb_cnt; 5516 xdomain += cache->uc_crossbucket.ucb_cnt; 5517 cachefree += cache->uc_crossbucket.ucb_cnt; 5518 allocs += cache->uc_allocs; 5519 frees += cache->uc_frees; 5520 } 5521 allocs += counter_u64_fetch(z->uz_allocs); 5522 frees += counter_u64_fetch(z->uz_frees); 5523 xdomain += counter_u64_fetch(z->uz_xdomain); 5524 sleeps += z->uz_sleeps; 5525 if (cachefreep != NULL) 5526 *cachefreep = cachefree; 5527 if (allocsp != NULL) 5528 *allocsp = allocs; 5529 if (freesp != NULL) 5530 *freesp = frees; 5531 if (sleepsp != NULL) 5532 *sleepsp = sleeps; 5533 if (xdomainp != NULL) 5534 *xdomainp = xdomain; 5535 } 5536 #endif /* DDB */ 5537 5538 static int 5539 sysctl_vm_zone_count(SYSCTL_HANDLER_ARGS) 5540 { 5541 uma_keg_t kz; 5542 uma_zone_t z; 5543 int count; 5544 5545 count = 0; 5546 rw_rlock(&uma_rwlock); 5547 LIST_FOREACH(kz, &uma_kegs, uk_link) { 5548 LIST_FOREACH(z, &kz->uk_zones, uz_link) 5549 count++; 5550 } 5551 LIST_FOREACH(z, &uma_cachezones, uz_link) 5552 count++; 5553 5554 rw_runlock(&uma_rwlock); 5555 return (sysctl_handle_int(oidp, &count, 0, req)); 5556 } 5557 5558 static void 5559 uma_vm_zone_stats(struct uma_type_header *uth, uma_zone_t z, struct sbuf *sbuf, 5560 struct uma_percpu_stat *ups, bool internal) 5561 { 5562 uma_zone_domain_t zdom; 5563 uma_cache_t cache; 5564 int i; 5565 5566 for (i = 0; i < vm_ndomains; i++) { 5567 zdom = ZDOM_GET(z, i); 5568 uth->uth_zone_free += zdom->uzd_nitems; 5569 } 5570 uth->uth_allocs = counter_u64_fetch(z->uz_allocs); 5571 uth->uth_frees = counter_u64_fetch(z->uz_frees); 5572 uth->uth_fails = counter_u64_fetch(z->uz_fails); 5573 uth->uth_xdomain = counter_u64_fetch(z->uz_xdomain); 5574 uth->uth_sleeps = z->uz_sleeps; 5575 5576 for (i = 0; i < mp_maxid + 1; i++) { 5577 bzero(&ups[i], sizeof(*ups)); 5578 if (internal || CPU_ABSENT(i)) 5579 continue; 5580 cache = &z->uz_cpu[i]; 5581 ups[i].ups_cache_free += cache->uc_allocbucket.ucb_cnt; 5582 ups[i].ups_cache_free += cache->uc_freebucket.ucb_cnt; 5583 ups[i].ups_cache_free += cache->uc_crossbucket.ucb_cnt; 5584 ups[i].ups_allocs = cache->uc_allocs; 5585 ups[i].ups_frees = cache->uc_frees; 5586 } 5587 } 5588 5589 static int 5590 sysctl_vm_zone_stats(SYSCTL_HANDLER_ARGS) 5591 { 5592 struct uma_stream_header ush; 5593 struct uma_type_header uth; 5594 struct uma_percpu_stat *ups; 5595 struct sbuf sbuf; 5596 uma_keg_t kz; 5597 uma_zone_t z; 5598 uint64_t items; 5599 uint32_t kfree, pages; 5600 int count, error, i; 5601 5602 error = sysctl_wire_old_buffer(req, 0); 5603 if (error != 0) 5604 return (error); 5605 sbuf_new_for_sysctl(&sbuf, NULL, 128, req); 5606 sbuf_clear_flags(&sbuf, SBUF_INCLUDENUL); 5607 ups = malloc((mp_maxid + 1) * sizeof(*ups), M_TEMP, M_WAITOK); 5608 5609 count = 0; 5610 rw_rlock(&uma_rwlock); 5611 LIST_FOREACH(kz, &uma_kegs, uk_link) { 5612 LIST_FOREACH(z, &kz->uk_zones, uz_link) 5613 count++; 5614 } 5615 5616 LIST_FOREACH(z, &uma_cachezones, uz_link) 5617 count++; 5618 5619 /* 5620 * Insert stream header. 5621 */ 5622 bzero(&ush, sizeof(ush)); 5623 ush.ush_version = UMA_STREAM_VERSION; 5624 ush.ush_maxcpus = (mp_maxid + 1); 5625 ush.ush_count = count; 5626 (void)sbuf_bcat(&sbuf, &ush, sizeof(ush)); 5627 5628 LIST_FOREACH(kz, &uma_kegs, uk_link) { 5629 kfree = pages = 0; 5630 for (i = 0; i < vm_ndomains; i++) { 5631 kfree += kz->uk_domain[i].ud_free_items; 5632 pages += kz->uk_domain[i].ud_pages; 5633 } 5634 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 5635 bzero(&uth, sizeof(uth)); 5636 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 5637 uth.uth_align = kz->uk_align; 5638 uth.uth_size = kz->uk_size; 5639 uth.uth_rsize = kz->uk_rsize; 5640 if (z->uz_max_items > 0) { 5641 items = UZ_ITEMS_COUNT(z->uz_items); 5642 uth.uth_pages = (items / kz->uk_ipers) * 5643 kz->uk_ppera; 5644 } else 5645 uth.uth_pages = pages; 5646 uth.uth_maxpages = (z->uz_max_items / kz->uk_ipers) * 5647 kz->uk_ppera; 5648 uth.uth_limit = z->uz_max_items; 5649 uth.uth_keg_free = kfree; 5650 5651 /* 5652 * A zone is secondary is it is not the first entry 5653 * on the keg's zone list. 5654 */ 5655 if ((z->uz_flags & UMA_ZONE_SECONDARY) && 5656 (LIST_FIRST(&kz->uk_zones) != z)) 5657 uth.uth_zone_flags = UTH_ZONE_SECONDARY; 5658 uma_vm_zone_stats(&uth, z, &sbuf, ups, 5659 kz->uk_flags & UMA_ZFLAG_INTERNAL); 5660 (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 5661 for (i = 0; i < mp_maxid + 1; i++) 5662 (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 5663 } 5664 } 5665 LIST_FOREACH(z, &uma_cachezones, uz_link) { 5666 bzero(&uth, sizeof(uth)); 5667 strlcpy(uth.uth_name, z->uz_name, UTH_MAX_NAME); 5668 uth.uth_size = z->uz_size; 5669 uma_vm_zone_stats(&uth, z, &sbuf, ups, false); 5670 (void)sbuf_bcat(&sbuf, &uth, sizeof(uth)); 5671 for (i = 0; i < mp_maxid + 1; i++) 5672 (void)sbuf_bcat(&sbuf, &ups[i], sizeof(ups[i])); 5673 } 5674 5675 rw_runlock(&uma_rwlock); 5676 error = sbuf_finish(&sbuf); 5677 sbuf_delete(&sbuf); 5678 free(ups, M_TEMP); 5679 return (error); 5680 } 5681 5682 int 5683 sysctl_handle_uma_zone_max(SYSCTL_HANDLER_ARGS) 5684 { 5685 uma_zone_t zone = *(uma_zone_t *)arg1; 5686 int error, max; 5687 5688 max = uma_zone_get_max(zone); 5689 error = sysctl_handle_int(oidp, &max, 0, req); 5690 if (error || !req->newptr) 5691 return (error); 5692 5693 uma_zone_set_max(zone, max); 5694 5695 return (0); 5696 } 5697 5698 int 5699 sysctl_handle_uma_zone_cur(SYSCTL_HANDLER_ARGS) 5700 { 5701 uma_zone_t zone; 5702 int cur; 5703 5704 /* 5705 * Some callers want to add sysctls for global zones that 5706 * may not yet exist so they pass a pointer to a pointer. 5707 */ 5708 if (arg2 == 0) 5709 zone = *(uma_zone_t *)arg1; 5710 else 5711 zone = arg1; 5712 cur = uma_zone_get_cur(zone); 5713 return (sysctl_handle_int(oidp, &cur, 0, req)); 5714 } 5715 5716 static int 5717 sysctl_handle_uma_zone_allocs(SYSCTL_HANDLER_ARGS) 5718 { 5719 uma_zone_t zone = arg1; 5720 uint64_t cur; 5721 5722 cur = uma_zone_get_allocs(zone); 5723 return (sysctl_handle_64(oidp, &cur, 0, req)); 5724 } 5725 5726 static int 5727 sysctl_handle_uma_zone_frees(SYSCTL_HANDLER_ARGS) 5728 { 5729 uma_zone_t zone = arg1; 5730 uint64_t cur; 5731 5732 cur = uma_zone_get_frees(zone); 5733 return (sysctl_handle_64(oidp, &cur, 0, req)); 5734 } 5735 5736 static int 5737 sysctl_handle_uma_zone_flags(SYSCTL_HANDLER_ARGS) 5738 { 5739 struct sbuf sbuf; 5740 uma_zone_t zone = arg1; 5741 int error; 5742 5743 sbuf_new_for_sysctl(&sbuf, NULL, 0, req); 5744 if (zone->uz_flags != 0) 5745 sbuf_printf(&sbuf, "0x%b", zone->uz_flags, PRINT_UMA_ZFLAGS); 5746 else 5747 sbuf_printf(&sbuf, "0"); 5748 error = sbuf_finish(&sbuf); 5749 sbuf_delete(&sbuf); 5750 5751 return (error); 5752 } 5753 5754 static int 5755 sysctl_handle_uma_slab_efficiency(SYSCTL_HANDLER_ARGS) 5756 { 5757 uma_keg_t keg = arg1; 5758 int avail, effpct, total; 5759 5760 total = keg->uk_ppera * PAGE_SIZE; 5761 if ((keg->uk_flags & UMA_ZFLAG_OFFPAGE) != 0) 5762 total += slabzone(keg->uk_ipers)->uz_keg->uk_rsize; 5763 /* 5764 * We consider the client's requested size and alignment here, not the 5765 * real size determination uk_rsize, because we also adjust the real 5766 * size for internal implementation reasons (max bitset size). 5767 */ 5768 avail = keg->uk_ipers * roundup2(keg->uk_size, keg->uk_align + 1); 5769 if ((keg->uk_flags & UMA_ZONE_PCPU) != 0) 5770 avail *= mp_maxid + 1; 5771 effpct = 100 * avail / total; 5772 return (sysctl_handle_int(oidp, &effpct, 0, req)); 5773 } 5774 5775 static int 5776 sysctl_handle_uma_zone_items(SYSCTL_HANDLER_ARGS) 5777 { 5778 uma_zone_t zone = arg1; 5779 uint64_t cur; 5780 5781 cur = UZ_ITEMS_COUNT(atomic_load_64(&zone->uz_items)); 5782 return (sysctl_handle_64(oidp, &cur, 0, req)); 5783 } 5784 5785 #ifdef INVARIANTS 5786 static uma_slab_t 5787 uma_dbg_getslab(uma_zone_t zone, void *item) 5788 { 5789 uma_slab_t slab; 5790 uma_keg_t keg; 5791 uint8_t *mem; 5792 5793 /* 5794 * It is safe to return the slab here even though the 5795 * zone is unlocked because the item's allocation state 5796 * essentially holds a reference. 5797 */ 5798 mem = (uint8_t *)((uintptr_t)item & (~UMA_SLAB_MASK)); 5799 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 5800 return (NULL); 5801 if (zone->uz_flags & UMA_ZFLAG_VTOSLAB) 5802 return (vtoslab((vm_offset_t)mem)); 5803 keg = zone->uz_keg; 5804 if ((keg->uk_flags & UMA_ZFLAG_HASH) == 0) 5805 return ((uma_slab_t)(mem + keg->uk_pgoff)); 5806 KEG_LOCK(keg, 0); 5807 slab = hash_sfind(&keg->uk_hash, mem); 5808 KEG_UNLOCK(keg, 0); 5809 5810 return (slab); 5811 } 5812 5813 static bool 5814 uma_dbg_zskip(uma_zone_t zone, void *mem) 5815 { 5816 5817 if ((zone->uz_flags & UMA_ZFLAG_CACHE) != 0) 5818 return (true); 5819 5820 return (uma_dbg_kskip(zone->uz_keg, mem)); 5821 } 5822 5823 static bool 5824 uma_dbg_kskip(uma_keg_t keg, void *mem) 5825 { 5826 uintptr_t idx; 5827 5828 if (dbg_divisor == 0) 5829 return (true); 5830 5831 if (dbg_divisor == 1) 5832 return (false); 5833 5834 idx = (uintptr_t)mem >> PAGE_SHIFT; 5835 if (keg->uk_ipers > 1) { 5836 idx *= keg->uk_ipers; 5837 idx += ((uintptr_t)mem & PAGE_MASK) / keg->uk_rsize; 5838 } 5839 5840 if ((idx / dbg_divisor) * dbg_divisor != idx) { 5841 counter_u64_add(uma_skip_cnt, 1); 5842 return (true); 5843 } 5844 counter_u64_add(uma_dbg_cnt, 1); 5845 5846 return (false); 5847 } 5848 5849 /* 5850 * Set up the slab's freei data such that uma_dbg_free can function. 5851 * 5852 */ 5853 static void 5854 uma_dbg_alloc(uma_zone_t zone, uma_slab_t slab, void *item) 5855 { 5856 uma_keg_t keg; 5857 int freei; 5858 5859 if (slab == NULL) { 5860 slab = uma_dbg_getslab(zone, item); 5861 if (slab == NULL) 5862 panic("uma: item %p did not belong to zone %s", 5863 item, zone->uz_name); 5864 } 5865 keg = zone->uz_keg; 5866 freei = slab_item_index(slab, keg, item); 5867 5868 if (BIT_TEST_SET_ATOMIC(keg->uk_ipers, freei, 5869 slab_dbg_bits(slab, keg))) 5870 panic("Duplicate alloc of %p from zone %p(%s) slab %p(%d)", 5871 item, zone, zone->uz_name, slab, freei); 5872 } 5873 5874 /* 5875 * Verifies freed addresses. Checks for alignment, valid slab membership 5876 * and duplicate frees. 5877 * 5878 */ 5879 static void 5880 uma_dbg_free(uma_zone_t zone, uma_slab_t slab, void *item) 5881 { 5882 uma_keg_t keg; 5883 int freei; 5884 5885 if (slab == NULL) { 5886 slab = uma_dbg_getslab(zone, item); 5887 if (slab == NULL) 5888 panic("uma: Freed item %p did not belong to zone %s", 5889 item, zone->uz_name); 5890 } 5891 keg = zone->uz_keg; 5892 freei = slab_item_index(slab, keg, item); 5893 5894 if (freei >= keg->uk_ipers) 5895 panic("Invalid free of %p from zone %p(%s) slab %p(%d)", 5896 item, zone, zone->uz_name, slab, freei); 5897 5898 if (slab_item(slab, keg, freei) != item) 5899 panic("Unaligned free of %p from zone %p(%s) slab %p(%d)", 5900 item, zone, zone->uz_name, slab, freei); 5901 5902 if (!BIT_TEST_CLR_ATOMIC(keg->uk_ipers, freei, 5903 slab_dbg_bits(slab, keg))) 5904 panic("Duplicate free of %p from zone %p(%s) slab %p(%d)", 5905 item, zone, zone->uz_name, slab, freei); 5906 } 5907 #endif /* INVARIANTS */ 5908 5909 #ifdef DDB 5910 static int64_t 5911 get_uma_stats(uma_keg_t kz, uma_zone_t z, uint64_t *allocs, uint64_t *used, 5912 uint64_t *sleeps, long *cachefree, uint64_t *xdomain) 5913 { 5914 uint64_t frees; 5915 int i; 5916 5917 if (kz->uk_flags & UMA_ZFLAG_INTERNAL) { 5918 *allocs = counter_u64_fetch(z->uz_allocs); 5919 frees = counter_u64_fetch(z->uz_frees); 5920 *sleeps = z->uz_sleeps; 5921 *cachefree = 0; 5922 *xdomain = 0; 5923 } else 5924 uma_zone_sumstat(z, cachefree, allocs, &frees, sleeps, 5925 xdomain); 5926 for (i = 0; i < vm_ndomains; i++) { 5927 *cachefree += ZDOM_GET(z, i)->uzd_nitems; 5928 if (!((z->uz_flags & UMA_ZONE_SECONDARY) && 5929 (LIST_FIRST(&kz->uk_zones) != z))) 5930 *cachefree += kz->uk_domain[i].ud_free_items; 5931 } 5932 *used = *allocs - frees; 5933 return (((int64_t)*used + *cachefree) * kz->uk_size); 5934 } 5935 5936 DB_SHOW_COMMAND_FLAGS(uma, db_show_uma, DB_CMD_MEMSAFE) 5937 { 5938 const char *fmt_hdr, *fmt_entry; 5939 uma_keg_t kz; 5940 uma_zone_t z; 5941 uint64_t allocs, used, sleeps, xdomain; 5942 long cachefree; 5943 /* variables for sorting */ 5944 uma_keg_t cur_keg; 5945 uma_zone_t cur_zone, last_zone; 5946 int64_t cur_size, last_size, size; 5947 int ties; 5948 5949 /* /i option produces machine-parseable CSV output */ 5950 if (modif[0] == 'i') { 5951 fmt_hdr = "%s,%s,%s,%s,%s,%s,%s,%s,%s\n"; 5952 fmt_entry = "\"%s\",%ju,%jd,%ld,%ju,%ju,%u,%jd,%ju\n"; 5953 } else { 5954 fmt_hdr = "%18s %6s %7s %7s %11s %7s %7s %10s %8s\n"; 5955 fmt_entry = "%18s %6ju %7jd %7ld %11ju %7ju %7u %10jd %8ju\n"; 5956 } 5957 5958 db_printf(fmt_hdr, "Zone", "Size", "Used", "Free", "Requests", 5959 "Sleeps", "Bucket", "Total Mem", "XFree"); 5960 5961 /* Sort the zones with largest size first. */ 5962 last_zone = NULL; 5963 last_size = INT64_MAX; 5964 for (;;) { 5965 cur_zone = NULL; 5966 cur_size = -1; 5967 ties = 0; 5968 LIST_FOREACH(kz, &uma_kegs, uk_link) { 5969 LIST_FOREACH(z, &kz->uk_zones, uz_link) { 5970 /* 5971 * In the case of size ties, print out zones 5972 * in the order they are encountered. That is, 5973 * when we encounter the most recently output 5974 * zone, we have already printed all preceding 5975 * ties, and we must print all following ties. 5976 */ 5977 if (z == last_zone) { 5978 ties = 1; 5979 continue; 5980 } 5981 size = get_uma_stats(kz, z, &allocs, &used, 5982 &sleeps, &cachefree, &xdomain); 5983 if (size > cur_size && size < last_size + ties) 5984 { 5985 cur_size = size; 5986 cur_zone = z; 5987 cur_keg = kz; 5988 } 5989 } 5990 } 5991 if (cur_zone == NULL) 5992 break; 5993 5994 size = get_uma_stats(cur_keg, cur_zone, &allocs, &used, 5995 &sleeps, &cachefree, &xdomain); 5996 db_printf(fmt_entry, cur_zone->uz_name, 5997 (uintmax_t)cur_keg->uk_size, (intmax_t)used, cachefree, 5998 (uintmax_t)allocs, (uintmax_t)sleeps, 5999 (unsigned)cur_zone->uz_bucket_size, (intmax_t)size, 6000 xdomain); 6001 6002 if (db_pager_quit) 6003 return; 6004 last_zone = cur_zone; 6005 last_size = cur_size; 6006 } 6007 } 6008 6009 DB_SHOW_COMMAND_FLAGS(umacache, db_show_umacache, DB_CMD_MEMSAFE) 6010 { 6011 uma_zone_t z; 6012 uint64_t allocs, frees; 6013 long cachefree; 6014 int i; 6015 6016 db_printf("%18s %8s %8s %8s %12s %8s\n", "Zone", "Size", "Used", "Free", 6017 "Requests", "Bucket"); 6018 LIST_FOREACH(z, &uma_cachezones, uz_link) { 6019 uma_zone_sumstat(z, &cachefree, &allocs, &frees, NULL, NULL); 6020 for (i = 0; i < vm_ndomains; i++) 6021 cachefree += ZDOM_GET(z, i)->uzd_nitems; 6022 db_printf("%18s %8ju %8jd %8ld %12ju %8u\n", 6023 z->uz_name, (uintmax_t)z->uz_size, 6024 (intmax_t)(allocs - frees), cachefree, 6025 (uintmax_t)allocs, z->uz_bucket_size); 6026 if (db_pager_quit) 6027 return; 6028 } 6029 } 6030 #endif /* DDB */ 6031