Lines Matching +full:pcs +full:- +full:handle
1 // SPDX-License-Identifier: GPL-2.0
38 #include <linux/fault-inject.h>
45 #include <kunit/test-bug.h>
57 * 2. node->list_lock (Spinlock)
58 * 3. kmem_cache->cpu_slab->lock (Local lock)
76 * A. slab->freelist -> List of free objects in a slab
77 * B. slab->inuse -> Number of objects in use
78 * C. slab->objects -> Number of objects in slab
79 * D. slab->frozen -> frozen state
100 * - node partial slab: SL_partial && !frozen
101 * - cpu partial slab: !SL_partial && !frozen
102 * - cpu slab: !SL_partial && frozen
103 * - full slab: !SL_partial && !frozen
114 * much as possible. As long as SLUB does not have to handle partial
122 * cpu_slab->lock local lock
131 * an in-progress slow path operations. In this case the local lock is always
167 * slab->frozen The slab is frozen and exempt from list processing.
189 * enum slab_flags - How the slab flags bits are used.
191 * @SL_partial: On the per-node partial list
259 p += s->red_left_pad; in fixup_red_left()
276 * - Support PAGE_ALLOC_DEBUG. Should be easy to do.
278 * - Variable sizing of the per node arrays
321 #define OO_MASK ((1 << OO_SHIFT) - 1)
342 depot_stack_handle_t handle; member
442 * avoid this_cpu_add()'s irq-disable overhead. in stat()
444 raw_cpu_inc(s->cpu_slab->stat[si]); in stat()
452 raw_cpu_add(s->cpu_slab->stat[si], v); in stat_add()
504 return s->node[node]; in get_node()
518 return n->barn; in get_barn()
557 * with an XOR of the address where the pointer is held and a per-cache
566 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr); in freelist_ptr_encode()
579 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr)); in freelist_ptr_decode()
592 ptr_addr = (unsigned long)object + s->offset; in get_freepointer()
600 prefetchw(object + s->offset); in prefetch_freepointer()
624 freepointer_addr = (unsigned long)object + s->offset; in get_freepointer_safe()
631 unsigned long freeptr_addr = (unsigned long)object + s->offset; in set_freepointer()
646 return s->offset >= s->inuse; in freeptr_outside_object()
656 return s->inuse + sizeof(void *); in get_info_end()
658 return s->inuse; in get_info_end()
664 __p < (__addr) + (__objects) * (__s)->size; \
665 __p += (__s)->size)
697 s->cpu_partial = nr_objects; in slub_set_cpu_partial()
703 * be half-full. in slub_set_cpu_partial()
705 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo)); in slub_set_cpu_partial()
706 s->cpu_partial_slabs = nr_slabs; in slub_set_cpu_partial()
711 return s->cpu_partial_slabs; in slub_get_cpu_partial()
726 * If network-based swap is enabled, slub must keep track of whether memory
731 return test_bit(SL_pfmemalloc, &slab->flags.f); in slab_test_pfmemalloc()
736 set_bit(SL_pfmemalloc, &slab->flags.f); in slab_set_pfmemalloc()
741 __clear_bit(SL_pfmemalloc, &slab->flags.f); in __slab_clear_pfmemalloc()
749 bit_spin_lock(SL_locked, &slab->flags.f); in slab_lock()
754 bit_spin_unlock(SL_locked, &slab->flags.f); in slab_unlock()
766 return try_cmpxchg_freelist(&slab->freelist_counter.full, &old.full, new.full); in __update_freelist_fast()
780 if (slab->freelist == freelist_old && in __update_freelist_slow()
781 slab->counters == counters_old) { in __update_freelist_slow()
782 slab->freelist = freelist_new; in __update_freelist_slow()
783 slab->counters = counters_new; in __update_freelist_slow()
808 if (s->flags & __CMPXCHG_DOUBLE) { in __slab_update_freelist()
822 pr_info("%s %s: cmpxchg double redo ", n, s->name); in __slab_update_freelist()
835 if (s->flags & __CMPXCHG_DOUBLE) { in slab_update_freelist()
853 pr_info("%s %s: cmpxchg double redo ", n, s->name); in slab_update_freelist()
887 return s->object_size; in get_orig_size()
915 bitmap_zero(obj_map, slab->objects); in __fill_map()
917 for (p = slab->freelist; p; p = get_freepointer(s, p)) in __fill_map()
929 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); in slab_add_kunit_errors()
933 (*(int *)resource->data)++; in slab_add_kunit_errors()
945 resource = kunit_find_named_resource(current->kunit_test, "slab_errors"); in slab_in_kunit_test()
958 if (s->flags & SLAB_RED_ZONE) in size_from_object()
959 return s->size - s->red_left_pad; in size_from_object()
961 return s->size; in size_from_object()
966 if (s->flags & SLAB_RED_ZONE) in restore_red_left()
967 p -= s->red_left_pad; in restore_red_left()
1018 if (object < base || object >= base + slab->objects * s->size || in check_valid_pointer()
1019 (object - base) % s->size) { in check_valid_pointer()
1048 depot_stack_handle_t handle; in set_track_prepare() local
1053 handle = stack_depot_save(entries, nr_entries, gfp_flags); in set_track_prepare()
1055 return handle; in set_track_prepare()
1066 depot_stack_handle_t handle) in set_track_update() argument
1071 p->handle = handle; in set_track_update()
1073 p->addr = addr; in set_track_update()
1074 p->cpu = smp_processor_id(); in set_track_update()
1075 p->pid = current->pid; in set_track_update()
1076 p->when = jiffies; in set_track_update()
1082 depot_stack_handle_t handle = set_track_prepare(gfp_flags); in set_track() local
1084 set_track_update(s, object, alloc, addr, handle); in set_track()
1091 if (!(s->flags & SLAB_STORE_USER)) in init_tracking()
1100 depot_stack_handle_t handle __maybe_unused; in print_track()
1102 if (!t->addr) in print_track()
1106 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid); in print_track()
1108 handle = READ_ONCE(t->handle); in print_track()
1109 if (handle) in print_track()
1110 stack_depot_print(handle); in print_track()
1119 if (!(s->flags & SLAB_STORE_USER)) in print_tracking()
1129 slab, slab->objects, slab->inuse, slab->freelist, in print_slab_info()
1130 &slab->flags.f); in print_slab_info()
1135 set_orig_size(s, (void *)object, s->object_size); in skip_orig_size_check()
1147 pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf); in __slab_bug()
1148 pr_err("-----------------------------------------------------------------------------\n\n"); in __slab_bug()
1173 pr_err("FIX %s: %pV\n", s->name, &vaf); in slab_fix()
1187 p, p - addr, get_freepointer(s, p)); in print_trailer()
1189 if (s->flags & SLAB_RED_ZONE) in print_trailer()
1190 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad, in print_trailer()
1191 s->red_left_pad); in print_trailer()
1193 print_section(KERN_ERR, "Bytes b4 ", p - 16, 16); in print_trailer()
1196 min_t(unsigned int, s->object_size, PAGE_SIZE)); in print_trailer()
1197 if (s->flags & SLAB_RED_ZONE) in print_trailer()
1198 print_section(KERN_ERR, "Redzone ", p + s->object_size, in print_trailer()
1199 s->inuse - s->object_size); in print_trailer()
1203 if (s->flags & SLAB_STORE_USER) in print_trailer()
1214 size_from_object(s) - off); in print_trailer()
1238 if ((s->flags & SLAB_CONSISTENCY_CHECKS) && in freelist_corrupted()
1278 unsigned int poison_size = s->object_size; in init_object()
1280 if (s->flags & SLAB_RED_ZONE) { in init_object()
1283 * the shadow makes it possible to distinguish uninit-value in init_object()
1284 * from use-after-free. in init_object()
1286 memset_no_sanitize_memory(p - s->red_left_pad, val, in init_object()
1287 s->red_left_pad); in init_object()
1299 if (s->flags & __OBJECT_POISON) { in init_object()
1300 memset_no_sanitize_memory(p, POISON_FREE, poison_size - 1); in init_object()
1301 memset_no_sanitize_memory(p + poison_size - 1, POISON_END, 1); in init_object()
1304 if (s->flags & SLAB_RED_ZONE) in init_object()
1306 s->inuse - poison_size); in init_object()
1312 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data); in restore_bytes()
1313 memset(from, data, to - from); in restore_bytes()
1338 while (end > fault && end[-1] == value) in check_bytes_and_report()
1339 end--; in check_bytes_and_report()
1344 pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n", in check_bytes_and_report()
1345 what, fault, end - 1, fault - addr, fault[0], value); in check_bytes_and_report()
1366 * object + s->object_size
1374 * object + s->inuse
1386 * object + s->size
1387 * Nothing is used beyond s->size.
1398 if (s->flags & SLAB_STORE_USER) { in check_pad_bytes()
1402 if (s->flags & SLAB_KMALLOC) in check_pad_bytes()
1412 p + off, POISON_INUSE, size_from_object(s) - off, true); in check_pad_bytes()
1426 if (!(s->flags & SLAB_POISON)) in slab_pad_check()
1432 remainder = length % s->size; in slab_pad_check()
1436 pad = end - remainder; in slab_pad_check()
1442 while (end > fault && end[-1] == POISON_INUSE) in slab_pad_check()
1443 end--; in slab_pad_check()
1445 slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu", in slab_pad_check()
1446 fault, end - 1, fault - start); in slab_pad_check()
1457 u8 *endobject = object + s->object_size; in check_object()
1461 if (s->flags & SLAB_RED_ZONE) { in check_object()
1463 object - s->red_left_pad, val, s->red_left_pad, ret)) in check_object()
1467 endobject, val, s->inuse - s->object_size, ret)) in check_object()
1473 if (s->object_size > orig_size && in check_object()
1476 val, s->object_size - orig_size, ret)) { in check_object()
1481 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) { in check_object()
1484 s->inuse - s->object_size, ret)) in check_object()
1489 if (s->flags & SLAB_POISON) { in check_object()
1490 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) { in check_object()
1497 if (kasan_meta_size < s->object_size - 1 && in check_object()
1500 s->object_size - kasan_meta_size - 1, ret)) in check_object()
1502 if (kasan_meta_size < s->object_size && in check_object()
1504 p + s->object_size - 1, POISON_END, 1, ret)) in check_object()
1542 maxobj = order_objects(slab_order(slab), s->size); in check_slab()
1543 if (slab->objects > maxobj) { in check_slab()
1545 slab->objects, maxobj); in check_slab()
1548 if (slab->inuse > slab->objects) { in check_slab()
1550 slab->inuse, slab->objects); in check_slab()
1553 if (slab->frozen) { in check_slab()
1574 fp = slab->freelist; in on_freelist()
1575 while (fp && nr <= slab->objects) { in on_freelist()
1586 slab->freelist = NULL; in on_freelist()
1587 slab->inuse = slab->objects; in on_freelist()
1597 if (nr > slab->objects) { in on_freelist()
1599 slab->freelist = NULL; in on_freelist()
1600 slab->inuse = slab->objects; in on_freelist()
1605 max_objects = order_objects(slab_order(slab), s->size); in on_freelist()
1609 if (slab->objects != max_objects) { in on_freelist()
1611 slab->objects, max_objects); in on_freelist()
1612 slab->objects = max_objects; in on_freelist()
1615 if (slab->inuse != slab->objects - nr) { in on_freelist()
1617 slab->inuse, slab->objects - nr); in on_freelist()
1618 slab->inuse = slab->objects - nr; in on_freelist()
1627 if (s->flags & SLAB_TRACE) { in trace()
1629 s->name, in trace()
1631 object, slab->inuse, in trace()
1632 slab->freelist); in trace()
1636 s->object_size); in trace()
1648 if (!(s->flags & SLAB_STORE_USER)) in add_full()
1651 lockdep_assert_held(&n->list_lock); in add_full()
1652 list_add(&slab->slab_list, &n->full); in add_full()
1657 if (!(s->flags & SLAB_STORE_USER)) in remove_full()
1660 lockdep_assert_held(&n->list_lock); in remove_full()
1661 list_del(&slab->slab_list); in remove_full()
1666 return atomic_long_read(&n->nr_slabs); in node_nr_slabs()
1673 atomic_long_inc(&n->nr_slabs); in inc_slabs_node()
1674 atomic_long_add(objects, &n->total_objects); in inc_slabs_node()
1680 atomic_long_dec(&n->nr_slabs); in dec_slabs_node()
1681 atomic_long_sub(objects, &n->total_objects); in dec_slabs_node()
1725 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in alloc_debug_processing()
1742 slab->inuse = slab->objects; in alloc_debug_processing()
1743 slab->freelist = NULL; in alloc_debug_processing()
1744 slab->frozen = 1; /* mark consistency-failed slab as frozen */ in alloc_debug_processing()
1765 if (unlikely(s != slab->slab_cache)) { in free_consistency_checks()
1766 if (!slab->slab_cache) { in free_consistency_checks()
1784 * @init: assume this is initial parsing and not per-kmem-create parsing
1810 case '-': in parse_slub_debug_flags()
1919 pr_info("mem auto-init: SLAB_POISON will take precedence over init_on_alloc/init_on_free\n"); in setup_slub_debug()
1927 * kmem_cache_flags - apply debugging options to the cache
1933 * slab_debug=<Debug-Options>,<slab name1>,<slab name2> ...
1969 end = next_block - 1; in kmem_cache_flags()
1971 glob = strnchr(iter, end - iter, '*'); in kmem_cache_flags()
1973 cmplen = glob - iter; in kmem_cache_flags()
1975 cmplen = max_t(size_t, len, (end - iter)); in kmem_cache_flags()
2000 unsigned long addr, depot_stack_handle_t handle) { return true; } in free_debug_processing() argument
2047 unsigned int offs = obj_to_index(obj_exts_slab->slab_cache, in mark_objexts_empty()
2061 return cmpxchg(&slab->obj_exts, 0, OBJEXTS_ALLOC_FAIL) == 0; in mark_failed_objexts_alloc()
2099 slab->obj_exts = 0; in init_slab_obj_exts()
2140 return -ENOMEM; in alloc_slab_obj_exts()
2150 old_exts = READ_ONCE(slab->obj_exts); in alloc_slab_obj_exts()
2158 slab->obj_exts = new_exts; in alloc_slab_obj_exts()
2171 } else if (cmpxchg(&slab->obj_exts, old_exts, new_exts) != old_exts) { in alloc_slab_obj_exts()
2172 /* Retry if a racing thread changed slab->obj_exts from under us. */ in alloc_slab_obj_exts()
2188 * If obj_exts allocation failed, slab->obj_exts is set to in free_slab_obj_exts()
2192 slab->obj_exts = 0; in free_slab_obj_exts()
2204 if (unlikely(READ_ONCE(slab->obj_exts) & OBJEXTS_NOSPIN_ALLOC)) in free_slab_obj_exts()
2208 slab->obj_exts = 0; in free_slab_obj_exts()
2240 __func__, s->name); in prepare_slab_obj_exts_hook()
2256 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) in __alloc_tagging_slab_alloc_hook()
2269 alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size); in __alloc_tagging_slab_alloc_hook()
2271 alloc_tag_set_inaccurate(current->alloc_tag); in __alloc_tagging_slab_alloc_hook()
2289 /* slab->obj_exts might not be NULL if it was created for MEMCG accounting. */ in __alloc_tagging_slab_free_hook()
2290 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE)) in __alloc_tagging_slab_free_hook()
2300 alloc_tag_sub(&obj_exts[off].ref, s->size); in __alloc_tagging_slab_free_hook()
2339 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT))) in memcg_slab_post_alloc_hook()
2397 node_stat_mod_folio(folio, NR_SLAB_UNRECLAIMABLE_B, -size); in memcg_slab_post_charge()
2403 s = slab->slab_cache; in memcg_slab_post_charge()
2466 bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay; in slab_free_hook()
2468 kmemleak_free_recursive(x, s->flags); in slab_free_hook()
2471 debug_check_no_locks_freed(x, s->object_size); in slab_free_hook()
2473 if (!(s->flags & SLAB_DEBUG_OBJECTS)) in slab_free_hook()
2474 debug_check_no_obj_freed(x, s->object_size); in slab_free_hook()
2476 /* Use KCSAN to help debug racy use-after-free. */ in slab_free_hook()
2478 __kcsan_check_access(x, s->object_size, in slab_free_hook()
2506 delayed_free->object = x; in slab_free_hook()
2507 call_rcu(&delayed_free->head, slab_free_after_rcu_debug); in slab_free_hook()
2532 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0; in slab_free_hook()
2534 s->size - inuse - rsize); in slab_free_hook()
2583 --(*cnt); in slab_free_freelist_hook()
2594 if (unlikely(s->ctor)) { in setup_object()
2596 s->ctor(object); in setup_object()
2605 s->sheaf_capacity), gfp); in alloc_empty_sheaf()
2610 sheaf->cache = s; in alloc_empty_sheaf()
2631 int to_fill = s->sheaf_capacity - sheaf->size; in refill_sheaf()
2638 &sheaf->objects[sheaf->size]); in refill_sheaf()
2640 sheaf->size += filled; in refill_sheaf()
2645 return -ENOMEM; in refill_sheaf()
2667 * Maximum number of objects freed during a single flush of main pcs sheaf.
2668 * Translates directly to an on-stack array size.
2676 * __kmem_cache_free_bulk() outside of cpu_sheaves->lock, work in batches where
2677 * object pointers are moved to a on-stack array under the lock. To bound the
2684 struct slub_percpu_sheaves *pcs; in sheaf_flush_main() local
2691 if (!local_trylock(&s->cpu_sheaves->lock)) in sheaf_flush_main()
2694 pcs = this_cpu_ptr(s->cpu_sheaves); in sheaf_flush_main()
2695 sheaf = pcs->main; in sheaf_flush_main()
2697 batch = min(PCS_BATCH_MAX, sheaf->size); in sheaf_flush_main()
2699 sheaf->size -= batch; in sheaf_flush_main()
2700 memcpy(objects, sheaf->objects + sheaf->size, batch * sizeof(void *)); in sheaf_flush_main()
2702 remaining = sheaf->size; in sheaf_flush_main()
2704 local_unlock(&s->cpu_sheaves->lock); in sheaf_flush_main()
2726 if (!sheaf->size) in sheaf_flush_unused()
2729 stat_add(s, SHEAF_FLUSH, sheaf->size); in sheaf_flush_unused()
2731 __kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]); in sheaf_flush_unused()
2733 sheaf->size = 0; in sheaf_flush_unused()
2740 void **p = &sheaf->objects[0]; in __rcu_free_sheaf_prepare()
2743 while (i < sheaf->size) { in __rcu_free_sheaf_prepare()
2750 p[i] = p[--sheaf->size]; in __rcu_free_sheaf_prepare()
2764 s = sheaf->cache; in rcu_free_sheaf_nobarn()
2784 struct slub_percpu_sheaves *pcs; in pcs_flush_all() local
2787 local_lock(&s->cpu_sheaves->lock); in pcs_flush_all()
2788 pcs = this_cpu_ptr(s->cpu_sheaves); in pcs_flush_all()
2790 spare = pcs->spare; in pcs_flush_all()
2791 pcs->spare = NULL; in pcs_flush_all()
2793 rcu_free = pcs->rcu_free; in pcs_flush_all()
2794 pcs->rcu_free = NULL; in pcs_flush_all()
2796 local_unlock(&s->cpu_sheaves->lock); in pcs_flush_all()
2804 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); in pcs_flush_all()
2811 struct slub_percpu_sheaves *pcs; in __pcs_flush_all_cpu() local
2813 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); in __pcs_flush_all_cpu()
2815 /* The cpu is not executing anymore so we don't need pcs->lock */ in __pcs_flush_all_cpu()
2816 sheaf_flush_unused(s, pcs->main); in __pcs_flush_all_cpu()
2817 if (pcs->spare) { in __pcs_flush_all_cpu()
2818 sheaf_flush_unused(s, pcs->spare); in __pcs_flush_all_cpu()
2819 free_empty_sheaf(s, pcs->spare); in __pcs_flush_all_cpu()
2820 pcs->spare = NULL; in __pcs_flush_all_cpu()
2823 if (pcs->rcu_free) { in __pcs_flush_all_cpu()
2824 call_rcu(&pcs->rcu_free->rcu_head, rcu_free_sheaf_nobarn); in __pcs_flush_all_cpu()
2825 pcs->rcu_free = NULL; in __pcs_flush_all_cpu()
2834 struct slub_percpu_sheaves *pcs; in pcs_destroy() local
2836 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); in pcs_destroy()
2839 if (!pcs->main) in pcs_destroy()
2851 WARN_ON(pcs->spare); in pcs_destroy()
2852 WARN_ON(pcs->rcu_free); in pcs_destroy()
2854 if (!WARN_ON(pcs->main->size)) { in pcs_destroy()
2855 free_empty_sheaf(s, pcs->main); in pcs_destroy()
2856 pcs->main = NULL; in pcs_destroy()
2860 free_percpu(s->cpu_sheaves); in pcs_destroy()
2861 s->cpu_sheaves = NULL; in pcs_destroy()
2869 if (!data_race(barn->nr_empty)) in barn_get_empty_sheaf()
2872 spin_lock_irqsave(&barn->lock, flags); in barn_get_empty_sheaf()
2874 if (likely(barn->nr_empty)) { in barn_get_empty_sheaf()
2875 empty = list_first_entry(&barn->sheaves_empty, in barn_get_empty_sheaf()
2877 list_del(&empty->barn_list); in barn_get_empty_sheaf()
2878 barn->nr_empty--; in barn_get_empty_sheaf()
2881 spin_unlock_irqrestore(&barn->lock, flags); in barn_get_empty_sheaf()
2896 spin_lock_irqsave(&barn->lock, flags); in barn_put_empty_sheaf()
2898 list_add(&sheaf->barn_list, &barn->sheaves_empty); in barn_put_empty_sheaf()
2899 barn->nr_empty++; in barn_put_empty_sheaf()
2901 spin_unlock_irqrestore(&barn->lock, flags); in barn_put_empty_sheaf()
2908 spin_lock_irqsave(&barn->lock, flags); in barn_put_full_sheaf()
2910 list_add(&sheaf->barn_list, &barn->sheaves_full); in barn_put_full_sheaf()
2911 barn->nr_full++; in barn_put_full_sheaf()
2913 spin_unlock_irqrestore(&barn->lock, flags); in barn_put_full_sheaf()
2921 if (!data_race(barn->nr_full) && !data_race(barn->nr_empty)) in barn_get_full_or_empty_sheaf()
2924 spin_lock_irqsave(&barn->lock, flags); in barn_get_full_or_empty_sheaf()
2926 if (barn->nr_full) { in barn_get_full_or_empty_sheaf()
2927 sheaf = list_first_entry(&barn->sheaves_full, struct slab_sheaf, in barn_get_full_or_empty_sheaf()
2929 list_del(&sheaf->barn_list); in barn_get_full_or_empty_sheaf()
2930 barn->nr_full--; in barn_get_full_or_empty_sheaf()
2931 } else if (barn->nr_empty) { in barn_get_full_or_empty_sheaf()
2932 sheaf = list_first_entry(&barn->sheaves_empty, in barn_get_full_or_empty_sheaf()
2934 list_del(&sheaf->barn_list); in barn_get_full_or_empty_sheaf()
2935 barn->nr_empty--; in barn_get_full_or_empty_sheaf()
2938 spin_unlock_irqrestore(&barn->lock, flags); in barn_get_full_or_empty_sheaf()
2954 if (!data_race(barn->nr_full)) in barn_replace_empty_sheaf()
2957 spin_lock_irqsave(&barn->lock, flags); in barn_replace_empty_sheaf()
2959 if (likely(barn->nr_full)) { in barn_replace_empty_sheaf()
2960 full = list_first_entry(&barn->sheaves_full, struct slab_sheaf, in barn_replace_empty_sheaf()
2962 list_del(&full->barn_list); in barn_replace_empty_sheaf()
2963 list_add(&empty->barn_list, &barn->sheaves_empty); in barn_replace_empty_sheaf()
2964 barn->nr_full--; in barn_replace_empty_sheaf()
2965 barn->nr_empty++; in barn_replace_empty_sheaf()
2968 spin_unlock_irqrestore(&barn->lock, flags); in barn_replace_empty_sheaf()
2975 * barn. But if there are too many full sheaves, reject this with -E2BIG.
2983 /* we don't repeat this check under barn->lock as it's not critical */ in barn_replace_full_sheaf()
2984 if (data_race(barn->nr_full) >= MAX_FULL_SHEAVES) in barn_replace_full_sheaf()
2985 return ERR_PTR(-E2BIG); in barn_replace_full_sheaf()
2986 if (!data_race(barn->nr_empty)) in barn_replace_full_sheaf()
2987 return ERR_PTR(-ENOMEM); in barn_replace_full_sheaf()
2989 spin_lock_irqsave(&barn->lock, flags); in barn_replace_full_sheaf()
2991 if (likely(barn->nr_empty)) { in barn_replace_full_sheaf()
2992 empty = list_first_entry(&barn->sheaves_empty, struct slab_sheaf, in barn_replace_full_sheaf()
2994 list_del(&empty->barn_list); in barn_replace_full_sheaf()
2995 list_add(&full->barn_list, &barn->sheaves_full); in barn_replace_full_sheaf()
2996 barn->nr_empty--; in barn_replace_full_sheaf()
2997 barn->nr_full++; in barn_replace_full_sheaf()
2999 empty = ERR_PTR(-ENOMEM); in barn_replace_full_sheaf()
3002 spin_unlock_irqrestore(&barn->lock, flags); in barn_replace_full_sheaf()
3009 spin_lock_init(&barn->lock); in barn_init()
3010 INIT_LIST_HEAD(&barn->sheaves_full); in barn_init()
3011 INIT_LIST_HEAD(&barn->sheaves_empty); in barn_init()
3012 barn->nr_full = 0; in barn_init()
3013 barn->nr_empty = 0; in barn_init()
3026 spin_lock_irqsave(&barn->lock, flags); in barn_shrink()
3028 list_splice_init(&barn->sheaves_full, &full_list); in barn_shrink()
3029 barn->nr_full = 0; in barn_shrink()
3030 list_splice_init(&barn->sheaves_empty, &empty_list); in barn_shrink()
3031 barn->nr_empty = 0; in barn_shrink()
3033 spin_unlock_irqrestore(&barn->lock, flags); in barn_shrink()
3075 /* Pre-initialize the random sequence cache */
3078 unsigned int count = oo_objects(s->oo); in init_cache_random_seq()
3082 if (s->random_seq) in init_cache_random_seq()
3088 s->name); in init_cache_random_seq()
3093 if (s->random_seq) { in init_cache_random_seq()
3097 s->random_seq[i] *= s->size; in init_cache_random_seq()
3115 /* Get the next entry on the pre-computed freelist randomized */
3128 idx = s->random_seq[*pos]; in next_freelist_entry()
3137 /* Shuffle the single linked freelist based on a random pre-computed sequence */
3145 if (slab->objects < 2 || !s->random_seq) in shuffle_freelist()
3148 freelist_count = oo_objects(s->oo); in shuffle_freelist()
3151 page_limit = slab->objects * s->size; in shuffle_freelist()
3157 slab->freelist = cur; in shuffle_freelist()
3159 for (idx = 1; idx < slab->objects; idx++) { in shuffle_freelist()
3185 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT)) in account_slab()
3198 * might have been disabled after slab->obj_exts got allocated. in unaccount_slab()
3203 -(PAGE_SIZE << order)); in unaccount_slab()
3210 struct kmem_cache_order_objects oo = s->oo; in allocate_slab()
3218 flags |= s->allocflags; in allocate_slab()
3221 * Let the initial higher-order allocation fail under memory pressure in allocate_slab()
3222 * so we fall-back to the minimum order allocation. in allocate_slab()
3225 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min)) in allocate_slab()
3234 oo = s->min; in allocate_slab()
3246 slab->objects = oo_objects(oo); in allocate_slab()
3247 slab->inuse = 0; in allocate_slab()
3248 slab->frozen = 0; in allocate_slab()
3253 slab->slab_cache = s; in allocate_slab()
3266 slab->freelist = start; in allocate_slab()
3267 for (idx = 0, p = start; idx < slab->objects - 1; idx++) { in allocate_slab()
3268 next = p + s->size; in allocate_slab()
3284 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO)); in new_slab()
3297 folio->mapping = NULL; in __free_slab()
3301 free_frozen_pages(&folio->page, order); in __free_slab()
3308 __free_slab(slab->slab_cache, slab); in rcu_free_slab()
3317 for_each_object(p, s, slab_address(slab), slab->objects) in free_slab()
3321 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) in free_slab()
3322 call_rcu(&slab->rcu_head, rcu_free_slab); in free_slab()
3329 dec_slabs_node(s, slab_nid(slab), slab->objects); in discard_slab()
3335 return test_bit(SL_partial, &slab->flags.f); in slab_test_node_partial()
3340 set_bit(SL_partial, &slab->flags.f); in slab_set_node_partial()
3345 clear_bit(SL_partial, &slab->flags.f); in slab_clear_node_partial()
3354 n->nr_partial++; in __add_partial()
3356 list_add_tail(&slab->slab_list, &n->partial); in __add_partial()
3358 list_add(&slab->slab_list, &n->partial); in __add_partial()
3365 lockdep_assert_held(&n->list_lock); in add_partial()
3372 lockdep_assert_held(&n->list_lock); in remove_partial()
3373 list_del(&slab->slab_list); in remove_partial()
3375 n->nr_partial--; in remove_partial()
3380 * slab from the n->partial list. Remove only a single object from the slab, do
3389 lockdep_assert_held(&n->list_lock); in alloc_single_from_partial()
3392 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in alloc_single_from_partial()
3400 object = slab->freelist; in alloc_single_from_partial()
3401 slab->freelist = get_freepointer(s, object); in alloc_single_from_partial()
3402 slab->inuse++; in alloc_single_from_partial()
3409 if (slab->inuse == slab->objects) { in alloc_single_from_partial()
3433 if (!allow_spin && !spin_trylock_irqsave(&n->list_lock, flags)) { in alloc_single_from_new_slab()
3439 object = slab->freelist; in alloc_single_from_new_slab()
3440 slab->freelist = get_freepointer(s, object); in alloc_single_from_new_slab()
3441 slab->inuse = 1; in alloc_single_from_new_slab()
3451 spin_unlock_irqrestore(&n->list_lock, flags); in alloc_single_from_new_slab()
3456 spin_lock_irqsave(&n->list_lock, flags); in alloc_single_from_new_slab()
3458 if (slab->inuse == slab->objects) in alloc_single_from_new_slab()
3463 inc_slabs_node(s, nid, slab->objects); in alloc_single_from_new_slab()
3464 spin_unlock_irqrestore(&n->list_lock, flags); in alloc_single_from_new_slab()
3494 if (!n || !n->nr_partial) in get_partial_node()
3497 if (gfpflags_allow_spinning(pc->flags)) in get_partial_node()
3498 spin_lock_irqsave(&n->list_lock, flags); in get_partial_node()
3499 else if (!spin_trylock_irqsave(&n->list_lock, flags)) in get_partial_node()
3501 list_for_each_entry_safe(slab, slab2, &n->partial, slab_list) { in get_partial_node()
3502 if (!pfmemalloc_match(slab, pc->flags)) in get_partial_node()
3507 pc->orig_size); in get_partial_node()
3510 pc->object = object; in get_partial_node()
3534 spin_unlock_irqrestore(&n->list_lock, flags); in get_partial_node()
3548 enum zone_type highest_zoneidx = gfp_zone(pc->flags); in get_any_partial()
3570 if (!s->remote_node_defrag_ratio || in get_any_partial()
3571 get_cycles() % 1024 > s->remote_node_defrag_ratio) in get_any_partial()
3576 zonelist = node_zonelist(mempolicy_slab_node(), pc->flags); in get_any_partial()
3582 if (n && cpuset_zone_allowed(zone, pc->flags) && in get_any_partial()
3583 n->nr_partial > s->min_partial) { in get_any_partial()
3588 * here - if mems_allowed was updated in in get_any_partial()
3615 if (slab || (node != NUMA_NO_NODE && (pc->flags & __GFP_THISNODE))) in get_partial()
3664 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid); in note_cmpxchg_failure()
3666 pr_info("%s %s: cmpxchg redo ", n, s->name); in note_cmpxchg_failure()
3670 pr_warn("due to cpu change %d -> %d\n", in note_cmpxchg_failure()
3673 pr_warn("due to cpu running other code. Event %ld->%ld\n", in note_cmpxchg_failure()
3687 * Register lockdep key for non-boot kmem caches to avoid in init_kmem_cache_cpus()
3702 lockdep_register_key(&s->lock_key); in init_kmem_cache_cpus()
3704 c = per_cpu_ptr(s->cpu_slab, cpu); in init_kmem_cache_cpus()
3705 local_trylock_init(&c->lock); in init_kmem_cache_cpus()
3707 lockdep_set_class(&c->lock, &s->lock_key); in init_kmem_cache_cpus()
3708 c->tid = init_tid(cpu); in init_kmem_cache_cpus()
3729 if (READ_ONCE(slab->freelist)) { in deactivate_slab()
3758 * Stage two: Unfreeze the slab while splicing the per-cpu in deactivate_slab()
3762 old.freelist = READ_ONCE(slab->freelist); in deactivate_slab()
3763 old.counters = READ_ONCE(slab->counters); in deactivate_slab()
3770 new.inuse -= free_delta; in deactivate_slab()
3784 if (!new.inuse && n->nr_partial >= s->min_partial) { in deactivate_slab()
3789 spin_lock_irqsave(&n->list_lock, flags); in deactivate_slab()
3791 spin_unlock_irqrestore(&n->list_lock, flags); in deactivate_slab()
3810 * kmalloc() -> ___slab_alloc() -> irqsave -> NMI -> bpf -> kmalloc_nolock()
3813 * kmalloc() -> ___slab_alloc() -> irqsave -> tracepoint/kprobe -> bpf -> kmalloc_nolock()
3815 * On PREEMPT_RT an invocation is not possible from IRQ-off or preempt
3820 * kmalloc() -> ___slab_alloc() -> rt_spin_lock(kmem_cache_A) ->
3821 * tracepoint/kprobe -> bpf -> kmalloc_nolock() -> rt_spin_lock(kmem_cache_B)
3827 local_lock_irqsave(&(s)->cpu_slab->lock, flags)
3831 bool __l = local_trylock_irqsave(&(s)->cpu_slab->lock, flags); \
3837 local_unlock_irqrestore(&(s)->cpu_slab->lock, flags)
3848 partial_slab = slab->next; in __put_partials()
3853 spin_unlock_irqrestore(&n->list_lock, flags); in __put_partials()
3856 spin_lock_irqsave(&n->list_lock, flags); in __put_partials()
3859 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) { in __put_partials()
3860 slab->next = slab_to_discard; in __put_partials()
3869 spin_unlock_irqrestore(&n->list_lock, flags); in __put_partials()
3873 slab_to_discard = slab_to_discard->next; in __put_partials()
3889 local_lock_irqsave(&s->cpu_slab->lock, flags); in put_partials()
3890 partial_slab = this_cpu_read(s->cpu_slab->partial); in put_partials()
3891 this_cpu_write(s->cpu_slab->partial, NULL); in put_partials()
3892 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in put_partials()
3904 c->partial = NULL; in put_partials_cpu()
3925 oldslab = this_cpu_read(s->cpu_slab->partial); in put_cpu_partial()
3928 if (drain && oldslab->slabs >= s->cpu_partial_slabs) { in put_cpu_partial()
3937 slabs = oldslab->slabs; in put_cpu_partial()
3943 slab->slabs = slabs; in put_cpu_partial()
3944 slab->next = oldslab; in put_cpu_partial()
3946 this_cpu_write(s->cpu_slab->partial, slab); in put_cpu_partial()
3970 local_lock_irqsave(&s->cpu_slab->lock, flags); in flush_slab()
3972 slab = c->slab; in flush_slab()
3973 freelist = c->freelist; in flush_slab()
3975 c->slab = NULL; in flush_slab()
3976 c->freelist = NULL; in flush_slab()
3977 c->tid = next_tid(c->tid); in flush_slab()
3979 local_unlock_irqrestore(&s->cpu_slab->lock, flags); in flush_slab()
3989 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in __flush_cpu_slab()
3990 void *freelist = c->freelist; in __flush_cpu_slab()
3991 struct slab *slab = c->slab; in __flush_cpu_slab()
3993 c->slab = NULL; in __flush_cpu_slab()
3994 c->freelist = NULL; in __flush_cpu_slab()
3995 c->tid = next_tid(c->tid); in __flush_cpu_slab()
4007 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab); in flush_this_cpu_slab()
4009 if (c->slab) in flush_this_cpu_slab()
4017 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu); in has_cpu_slab()
4019 return c->slab || slub_percpu_partial(c); in has_cpu_slab()
4030 struct slub_percpu_sheaves *pcs; in has_pcs_used() local
4032 if (!s->cpu_sheaves) in has_pcs_used()
4035 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); in has_pcs_used()
4037 return (pcs->spare || pcs->rcu_free || pcs->main->size); in has_pcs_used()
4052 s = sfw->s; in flush_cpu_slab()
4054 if (s->cpu_sheaves) in flush_cpu_slab()
4071 sfw->skip = true; in flush_all_cpus_locked()
4074 INIT_WORK(&sfw->work, flush_cpu_slab); in flush_all_cpus_locked()
4075 sfw->skip = false; in flush_all_cpus_locked()
4076 sfw->s = s; in flush_all_cpus_locked()
4077 queue_work_on(cpu, flushwq, &sfw->work); in flush_all_cpus_locked()
4082 if (sfw->skip) in flush_all_cpus_locked()
4084 flush_work(&sfw->work); in flush_all_cpus_locked()
4099 struct slub_percpu_sheaves *pcs; in flush_rcu_sheaf() local
4105 s = sfw->s; in flush_rcu_sheaf()
4107 local_lock(&s->cpu_sheaves->lock); in flush_rcu_sheaf()
4108 pcs = this_cpu_ptr(s->cpu_sheaves); in flush_rcu_sheaf()
4110 rcu_free = pcs->rcu_free; in flush_rcu_sheaf()
4111 pcs->rcu_free = NULL; in flush_rcu_sheaf()
4113 local_unlock(&s->cpu_sheaves->lock); in flush_rcu_sheaf()
4116 call_rcu(&rcu_free->rcu_head, rcu_free_sheaf_nobarn); in flush_rcu_sheaf()
4131 if (!s->cpu_sheaves) in flush_all_rcu_sheaves()
4140 * we don't check if rcu_free sheaf exists - racing in flush_all_rcu_sheaves()
4146 INIT_WORK(&sfw->work, flush_rcu_sheaf); in flush_all_rcu_sheaves()
4147 sfw->s = s; in flush_all_rcu_sheaves()
4148 queue_work_on(cpu, flushwq, &sfw->work); in flush_all_rcu_sheaves()
4153 flush_work(&sfw->work); in flush_all_rcu_sheaves()
4176 if (s->cpu_sheaves) in slub_cpu_dead()
4199 return slab->objects - slab->inuse; in count_free()
4204 return atomic_long_read(&n->total_objects); in node_nr_objs()
4210 unsigned long addr, depot_stack_handle_t handle) in free_debug_processing() argument
4216 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in free_debug_processing()
4221 if (slab->inuse < *bulk_cnt) { in free_debug_processing()
4223 slab->inuse, *bulk_cnt); in free_debug_processing()
4232 if (s->flags & SLAB_CONSISTENCY_CHECKS) { in free_debug_processing()
4237 if (s->flags & SLAB_STORE_USER) in free_debug_processing()
4238 set_track_update(s, object, TRACK_FREE, addr, handle); in free_debug_processing()
4274 spin_lock_irqsave(&n->list_lock, flags); in count_partial()
4275 list_for_each_entry(slab, &n->partial, slab_list) in count_partial()
4277 spin_unlock_irqrestore(&n->list_lock, flags); in count_partial()
4291 spin_lock_irqsave(&n->list_lock, flags); in count_partial_free_approx()
4292 if (n->nr_partial <= MAX_PARTIAL_TO_SCAN) { in count_partial_free_approx()
4293 list_for_each_entry(slab, &n->partial, slab_list) in count_partial_free_approx()
4294 x += slab->objects - slab->inuse; in count_partial_free_approx()
4303 list_for_each_entry(slab, &n->partial, slab_list) { in count_partial_free_approx()
4304 x += slab->objects - slab->inuse; in count_partial_free_approx()
4308 list_for_each_entry_reverse(slab, &n->partial, slab_list) { in count_partial_free_approx()
4309 x += slab->objects - slab->inuse; in count_partial_free_approx()
4313 x = mult_frac(x, n->nr_partial, scanned); in count_partial_free_approx()
4316 spin_unlock_irqrestore(&n->list_lock, flags); in count_partial_free_approx()
4335 s->name, s->object_size, s->size, oo_order(s->oo), in slab_out_of_memory()
4336 oo_order(s->min)); in slab_out_of_memory()
4338 if (oo_order(s->min) > get_order(s->object_size)) in slab_out_of_memory()
4340 s->name); in slab_out_of_memory()
4377 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid.full, in __update_cpu_freelist_fast()
4382 * Check the slab->freelist and either transfer the freelist to the
4395 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in get_freelist()
4398 freelist = slab->freelist; in get_freelist()
4399 counters = slab->counters; in get_freelist()
4403 new.inuse = slab->objects; in get_freelist()
4424 freelist = slab->freelist; in freeze_slab()
4425 counters = slab->counters; in freeze_slab()
4430 new.inuse = slab->objects; in freeze_slab()
4474 slab = READ_ONCE(c->slab); in ___slab_alloc()
4499 * unnecessary slab allocations even when n->partial list in ___slab_alloc()
4514 * information when the page leaves the per-cpu allocator in ___slab_alloc()
4519 /* must check again c->slab in case we got preempted and it changed */ in ___slab_alloc()
4522 if (unlikely(slab != c->slab)) { in ___slab_alloc()
4526 freelist = c->freelist; in ___slab_alloc()
4533 c->slab = NULL; in ___slab_alloc()
4534 c->tid = next_tid(c->tid); in ___slab_alloc()
4544 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock)); in ___slab_alloc()
4551 VM_BUG_ON(!c->slab->frozen); in ___slab_alloc()
4552 c->freelist = get_freepointer(s, freelist); in ___slab_alloc()
4553 c->tid = next_tid(c->tid); in ___slab_alloc()
4560 if (slab != c->slab) { in ___slab_alloc()
4564 freelist = c->freelist; in ___slab_alloc()
4565 c->slab = NULL; in ___slab_alloc()
4566 c->freelist = NULL; in ___slab_alloc()
4567 c->tid = next_tid(c->tid); in ___slab_alloc()
4576 if (unlikely(c->slab)) { in ___slab_alloc()
4592 c->slab = slab; in ___slab_alloc()
4601 slab->next = NULL; in ___slab_alloc()
4643 if (s->flags & SLAB_STORE_USER) in ___slab_alloc()
4654 slub_put_cpu_ptr(s->cpu_slab); in ___slab_alloc()
4656 c = slub_get_cpu_ptr(s->cpu_slab); in ___slab_alloc()
4680 if (s->flags & SLAB_STORE_USER) in ___slab_alloc()
4691 freelist = slab->freelist; in ___slab_alloc()
4692 slab->freelist = NULL; in ___slab_alloc()
4693 slab->inuse = slab->objects; in ___slab_alloc()
4694 slab->frozen = 1; in ___slab_alloc()
4696 inc_slabs_node(s, slab_nid(slab), slab->objects); in ___slab_alloc()
4710 if (unlikely(c->slab)) { in ___slab_alloc()
4711 void *flush_freelist = c->freelist; in ___slab_alloc()
4712 struct slab *flush_slab = c->slab; in ___slab_alloc()
4714 c->slab = NULL; in ___slab_alloc()
4715 c->freelist = NULL; in ___slab_alloc()
4716 c->tid = next_tid(c->tid); in ___slab_alloc()
4731 c->slab = slab; in ___slab_alloc()
4738 * kmalloc() -> ___slab_alloc() -> local_lock_cpu_slab() protected part of
4739 * ___slab_alloc() manipulating c->freelist -> kprobe -> bpf ->
4740 * kmalloc_nolock() or kfree_nolock() -> __update_cpu_freelist_fast()
4741 * manipulating c->freelist without lock.
4745 * c->freelist manipulation in ___slab_alloc() itself.
4765 c = slub_get_cpu_ptr(s->cpu_slab); in __slab_alloc()
4768 if (local_lock_is_locked(&s->cpu_slab->lock)) { in __slab_alloc()
4774 p = ERR_PTR(-EBUSY); in __slab_alloc()
4781 slub_put_cpu_ptr(s->cpu_slab); in __slab_alloc()
4807 c = raw_cpu_ptr(s->cpu_slab); in __slab_alloc_node()
4808 tid = READ_ONCE(c->tid); in __slab_alloc_node()
4827 object = c->freelist; in __slab_alloc_node()
4828 slab = c->slab; in __slab_alloc_node()
4834 struct mempolicy *mpol = current->mempolicy; in __slab_alloc_node()
4844 if (mpol->mode != MPOL_BIND || !slab || in __slab_alloc_node()
4845 !node_isset(slab_nid(slab), mpol->nodes)) in __slab_alloc_node()
4920 memset((void *)((char *)kasan_reset_tag(obj) + s->offset), in maybe_wipe_obj_freeptr()
4942 unsigned int zero_size = s->object_size; in slab_post_alloc_hook()
4956 (s->flags & SLAB_KMALLOC)) in slab_post_alloc_hook()
4963 * cause false-positive reports. This does not lead to a performance in slab_post_alloc_hook()
4983 kmemleak_alloc_recursive(p[i], s->object_size, 1, in slab_post_alloc_hook()
4984 s->flags, init_flags); in slab_post_alloc_hook()
4996 * the pcs pointer and the local lock locked (possibly on a different cpu than
5001 __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp) in __pcs_replace_empty_main() argument
5008 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); in __pcs_replace_empty_main()
5010 if (pcs->spare && pcs->spare->size > 0) { in __pcs_replace_empty_main()
5011 swap(pcs->main, pcs->spare); in __pcs_replace_empty_main()
5012 return pcs; in __pcs_replace_empty_main()
5017 local_unlock(&s->cpu_sheaves->lock); in __pcs_replace_empty_main()
5021 full = barn_replace_empty_sheaf(barn, pcs->main); in __pcs_replace_empty_main()
5025 pcs->main = full; in __pcs_replace_empty_main()
5026 return pcs; in __pcs_replace_empty_main()
5034 if (pcs->spare) { in __pcs_replace_empty_main()
5035 empty = pcs->spare; in __pcs_replace_empty_main()
5036 pcs->spare = NULL; in __pcs_replace_empty_main()
5042 local_unlock(&s->cpu_sheaves->lock); in __pcs_replace_empty_main()
5068 local_lock(&s->cpu_sheaves->lock); in __pcs_replace_empty_main()
5069 pcs = this_cpu_ptr(s->cpu_sheaves); in __pcs_replace_empty_main()
5079 if (pcs->main->size == 0) { in __pcs_replace_empty_main()
5080 barn_put_empty_sheaf(barn, pcs->main); in __pcs_replace_empty_main()
5081 pcs->main = full; in __pcs_replace_empty_main()
5082 return pcs; in __pcs_replace_empty_main()
5085 if (!pcs->spare) { in __pcs_replace_empty_main()
5086 pcs->spare = full; in __pcs_replace_empty_main()
5087 return pcs; in __pcs_replace_empty_main()
5090 if (pcs->spare->size == 0) { in __pcs_replace_empty_main()
5091 barn_put_empty_sheaf(barn, pcs->spare); in __pcs_replace_empty_main()
5092 pcs->spare = full; in __pcs_replace_empty_main()
5093 return pcs; in __pcs_replace_empty_main()
5099 return pcs; in __pcs_replace_empty_main()
5105 struct slub_percpu_sheaves *pcs; in alloc_from_pcs() local
5113 struct mempolicy *mpol = current->mempolicy; in alloc_from_pcs()
5123 if (mpol->mode != MPOL_BIND || in alloc_from_pcs()
5124 !node_isset(numa_mem_id(), mpol->nodes)) in alloc_from_pcs()
5140 if (!local_trylock(&s->cpu_sheaves->lock)) in alloc_from_pcs()
5143 pcs = this_cpu_ptr(s->cpu_sheaves); in alloc_from_pcs()
5145 if (unlikely(pcs->main->size == 0)) { in alloc_from_pcs()
5146 pcs = __pcs_replace_empty_main(s, pcs, gfp); in alloc_from_pcs()
5147 if (unlikely(!pcs)) in alloc_from_pcs()
5151 object = pcs->main->objects[pcs->main->size - 1]; in alloc_from_pcs()
5160 local_unlock(&s->cpu_sheaves->lock); in alloc_from_pcs()
5165 pcs->main->size--; in alloc_from_pcs()
5167 local_unlock(&s->cpu_sheaves->lock); in alloc_from_pcs()
5177 struct slub_percpu_sheaves *pcs; in alloc_from_pcs_bulk() local
5183 if (!local_trylock(&s->cpu_sheaves->lock)) in alloc_from_pcs_bulk()
5186 pcs = this_cpu_ptr(s->cpu_sheaves); in alloc_from_pcs_bulk()
5188 if (unlikely(pcs->main->size == 0)) { in alloc_from_pcs_bulk()
5193 if (pcs->spare && pcs->spare->size > 0) { in alloc_from_pcs_bulk()
5194 swap(pcs->main, pcs->spare); in alloc_from_pcs_bulk()
5200 local_unlock(&s->cpu_sheaves->lock); in alloc_from_pcs_bulk()
5204 full = barn_replace_empty_sheaf(barn, pcs->main); in alloc_from_pcs_bulk()
5208 pcs->main = full; in alloc_from_pcs_bulk()
5214 local_unlock(&s->cpu_sheaves->lock); in alloc_from_pcs_bulk()
5226 main = pcs->main; in alloc_from_pcs_bulk()
5227 batch = min(size, main->size); in alloc_from_pcs_bulk()
5229 main->size -= batch; in alloc_from_pcs_bulk()
5230 memcpy(p, main->objects + main->size, batch * sizeof(void *)); in alloc_from_pcs_bulk()
5232 local_unlock(&s->cpu_sheaves->lock); in alloc_from_pcs_bulk()
5240 size -= batch; in alloc_from_pcs_bulk()
5272 if (s->cpu_sheaves) in slab_alloc_node()
5284 * @orig_size bytes might be zeroed instead of s->object_size in slab_alloc_node()
5296 s->object_size); in kmem_cache_alloc_noprof()
5308 s->object_size); in kmem_cache_alloc_lru_noprof()
5326 * kmem_cache_alloc_node - Allocate an object on the specified node
5340 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size); in kmem_cache_alloc_node_noprof()
5357 struct slub_percpu_sheaves *pcs; in kmem_cache_prefill_sheaf() local
5361 if (unlikely(size > s->sheaf_capacity)) { in kmem_cache_prefill_sheaf()
5370 WARN_ON_ONCE(s->sheaf_capacity == 0 && in kmem_cache_prefill_sheaf()
5372 !(s->flags & SLAB_DEBUG_FLAGS)); in kmem_cache_prefill_sheaf()
5379 sheaf->cache = s; in kmem_cache_prefill_sheaf()
5380 sheaf->capacity = size; in kmem_cache_prefill_sheaf()
5383 &sheaf->objects[0])) { in kmem_cache_prefill_sheaf()
5388 sheaf->size = size; in kmem_cache_prefill_sheaf()
5393 local_lock(&s->cpu_sheaves->lock); in kmem_cache_prefill_sheaf()
5394 pcs = this_cpu_ptr(s->cpu_sheaves); in kmem_cache_prefill_sheaf()
5396 if (pcs->spare) { in kmem_cache_prefill_sheaf()
5397 sheaf = pcs->spare; in kmem_cache_prefill_sheaf()
5398 pcs->spare = NULL; in kmem_cache_prefill_sheaf()
5406 if (sheaf && sheaf->size) in kmem_cache_prefill_sheaf()
5412 local_unlock(&s->cpu_sheaves->lock); in kmem_cache_prefill_sheaf()
5418 if (sheaf && sheaf->size < size) { in kmem_cache_prefill_sheaf()
5427 sheaf->capacity = s->sheaf_capacity; in kmem_cache_prefill_sheaf()
5445 struct slub_percpu_sheaves *pcs; in kmem_cache_return_sheaf() local
5448 if (unlikely(sheaf->capacity != s->sheaf_capacity)) { in kmem_cache_return_sheaf()
5454 local_lock(&s->cpu_sheaves->lock); in kmem_cache_return_sheaf()
5455 pcs = this_cpu_ptr(s->cpu_sheaves); in kmem_cache_return_sheaf()
5458 if (!pcs->spare) { in kmem_cache_return_sheaf()
5459 pcs->spare = sheaf; in kmem_cache_return_sheaf()
5464 local_unlock(&s->cpu_sheaves->lock); in kmem_cache_return_sheaf()
5475 if (!barn || data_race(barn->nr_full) >= MAX_FULL_SHEAVES || in kmem_cache_return_sheaf()
5491 * s->sheaf_capacity objects if such replacement is necessary, but the refill
5492 * fails (returning -ENOMEM), the existing sheaf is left intact
5506 return -EINVAL; in kmem_cache_refill_sheaf()
5509 if (sheaf->size >= size) in kmem_cache_refill_sheaf()
5512 if (likely(sheaf->capacity >= size)) { in kmem_cache_refill_sheaf()
5513 if (likely(sheaf->capacity == s->sheaf_capacity)) in kmem_cache_refill_sheaf()
5516 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size, in kmem_cache_refill_sheaf()
5517 &sheaf->objects[sheaf->size])) { in kmem_cache_refill_sheaf()
5518 return -ENOMEM; in kmem_cache_refill_sheaf()
5520 sheaf->size = sheaf->capacity; in kmem_cache_refill_sheaf()
5532 return -ENOMEM; in kmem_cache_refill_sheaf()
5543 * After the sheaf is emptied, it fails - no fallback to the slab cache itself.
5555 if (sheaf->size == 0) in kmem_cache_alloc_from_sheaf_noprof()
5558 ret = sheaf->objects[--sheaf->size]; in kmem_cache_alloc_from_sheaf_noprof()
5563 slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size); in kmem_cache_alloc_from_sheaf_noprof()
5572 return sheaf->size; in kmem_cache_sheaf_size()
5651 trace_kmalloc(caller, ret, size, s->size, flags, node); in __do_kmalloc_node()
5667 * kmalloc_nolock - Allocate an object of given size from any context.
5682 void *ret = ERR_PTR(-EBUSY); in kmalloc_nolock_noprof()
5698 if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s)) in kmalloc_nolock_noprof()
5702 * per-cpu slab and per-cpu partial slabs. They rely on in kmalloc_nolock_noprof()
5703 * kmem_cache_node->list_lock, so kmalloc_nolock() can in kmalloc_nolock_noprof()
5705 * spin_trylock_irqsave(&n->list_lock, ...) in kmalloc_nolock_noprof()
5721 * If in_nmi() && local_lock_is_locked(s->cpu_slab) then it means that in kmalloc_nolock_noprof()
5723 * it did local_lock_irqsave(&s->cpu_slab->lock, flags). in kmalloc_nolock_noprof()
5727 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock)) in kmalloc_nolock_noprof()
5731 if (PTR_ERR(ret) == -EBUSY) { in kmalloc_nolock_noprof()
5734 size = s->object_size + 1; in kmalloc_nolock_noprof()
5769 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE); in __kmalloc_cache_noprof()
5781 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node); in __kmalloc_cache_node_noprof()
5797 depot_stack_handle_t handle = 0; in free_to_partial_list() local
5803 if (s->flags & SLAB_STORE_USER) in free_to_partial_list()
5804 handle = set_track_prepare(__GFP_NOWARN); in free_to_partial_list()
5806 spin_lock_irqsave(&n->list_lock, flags); in free_to_partial_list()
5808 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) { in free_to_partial_list()
5809 void *prior = slab->freelist; in free_to_partial_list()
5812 slab->inuse -= cnt; in free_to_partial_list()
5814 slab->freelist = head; in free_to_partial_list()
5821 if (slab->inuse == 0 && n->nr_partial >= s->min_partial) in free_to_partial_list()
5839 * Update the counters while still holding n->list_lock to in free_to_partial_list()
5842 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects); in free_to_partial_list()
5845 spin_unlock_irqrestore(&n->list_lock, flags); in free_to_partial_list()
5883 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
5886 prior = slab->freelist; in __slab_free()
5887 counters = slab->counters; in __slab_free()
5891 new.inuse -= cnt; in __slab_free()
5905 spin_lock_irqsave(&n->list_lock, flags); in __slab_free()
5937 * This slab was partially empty but not on the per-node partial list, in __slab_free()
5941 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
5945 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial)) in __slab_free()
5956 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
5968 spin_unlock_irqrestore(&n->list_lock, flags); in __slab_free()
5974 * pcs is locked. We should have get rid of the spare sheaf and obtained an
5979 * the empty sheaf, we need to handle some unlikely but possible cases.
5992 struct slub_percpu_sheaves *pcs, struct slab_sheaf *empty, in __pcs_install_empty_sheaf() argument
5995 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); in __pcs_install_empty_sheaf()
5998 if (likely(!pcs->spare)) { in __pcs_install_empty_sheaf()
5999 pcs->spare = pcs->main; in __pcs_install_empty_sheaf()
6000 pcs->main = empty; in __pcs_install_empty_sheaf()
6008 if (pcs->main->size < s->sheaf_capacity) { in __pcs_install_empty_sheaf()
6014 if (pcs->spare->size < s->sheaf_capacity) { in __pcs_install_empty_sheaf()
6015 swap(pcs->main, pcs->spare); in __pcs_install_empty_sheaf()
6024 barn_put_full_sheaf(barn, pcs->main); in __pcs_install_empty_sheaf()
6026 pcs->main = empty; in __pcs_install_empty_sheaf()
6033 * the pcs pointer and the local lock locked (possibly on a different cpu than
6038 __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs) in __pcs_replace_full_main() argument
6045 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock)); in __pcs_replace_full_main()
6049 local_unlock(&s->cpu_sheaves->lock); in __pcs_replace_full_main()
6055 if (!pcs->spare) { in __pcs_replace_full_main()
6058 pcs->spare = pcs->main; in __pcs_replace_full_main()
6059 pcs->main = empty; in __pcs_replace_full_main()
6060 return pcs; in __pcs_replace_full_main()
6065 if (pcs->spare->size < s->sheaf_capacity) { in __pcs_replace_full_main()
6066 swap(pcs->main, pcs->spare); in __pcs_replace_full_main()
6067 return pcs; in __pcs_replace_full_main()
6070 empty = barn_replace_full_sheaf(barn, pcs->main); in __pcs_replace_full_main()
6074 pcs->main = empty; in __pcs_replace_full_main()
6075 return pcs; in __pcs_replace_full_main()
6078 if (PTR_ERR(empty) == -E2BIG) { in __pcs_replace_full_main()
6080 struct slab_sheaf *to_flush = pcs->spare; in __pcs_replace_full_main()
6084 pcs->spare = NULL; in __pcs_replace_full_main()
6085 local_unlock(&s->cpu_sheaves->lock); in __pcs_replace_full_main()
6101 local_unlock(&s->cpu_sheaves->lock); in __pcs_replace_full_main()
6113 if (!local_trylock(&s->cpu_sheaves->lock)) in __pcs_replace_full_main()
6116 pcs = this_cpu_ptr(s->cpu_sheaves); in __pcs_replace_full_main()
6123 if (pcs->main->size == s->sheaf_capacity) in __pcs_replace_full_main()
6126 return pcs; in __pcs_replace_full_main()
6129 if (!local_trylock(&s->cpu_sheaves->lock)) { in __pcs_replace_full_main()
6134 pcs = this_cpu_ptr(s->cpu_sheaves); in __pcs_replace_full_main()
6135 __pcs_install_empty_sheaf(s, pcs, empty, barn); in __pcs_replace_full_main()
6137 return pcs; in __pcs_replace_full_main()
6147 struct slub_percpu_sheaves *pcs; in free_to_pcs() local
6149 if (!local_trylock(&s->cpu_sheaves->lock)) in free_to_pcs()
6152 pcs = this_cpu_ptr(s->cpu_sheaves); in free_to_pcs()
6154 if (unlikely(pcs->main->size == s->sheaf_capacity)) { in free_to_pcs()
6156 pcs = __pcs_replace_full_main(s, pcs); in free_to_pcs()
6157 if (unlikely(!pcs)) in free_to_pcs()
6161 pcs->main->objects[pcs->main->size++] = object; in free_to_pcs()
6163 local_unlock(&s->cpu_sheaves->lock); in free_to_pcs()
6179 s = sheaf->cache; in rcu_free_sheaf()
6184 * to handle it as full (unless it became completely empty), as the code in rcu_free_sheaf()
6191 n = get_node(s, sheaf->node); in rcu_free_sheaf()
6195 barn = n->barn; in rcu_free_sheaf()
6198 if (unlikely(sheaf->size == 0)) in rcu_free_sheaf()
6207 if (data_race(barn->nr_full) < MAX_FULL_SHEAVES) { in rcu_free_sheaf()
6218 if (barn && data_race(barn->nr_empty) < MAX_EMPTY_SHEAVES) { in rcu_free_sheaf()
6228 struct slub_percpu_sheaves *pcs; in __kfree_rcu_sheaf() local
6231 if (!local_trylock(&s->cpu_sheaves->lock)) in __kfree_rcu_sheaf()
6234 pcs = this_cpu_ptr(s->cpu_sheaves); in __kfree_rcu_sheaf()
6236 if (unlikely(!pcs->rcu_free)) { in __kfree_rcu_sheaf()
6241 if (pcs->spare && pcs->spare->size == 0) { in __kfree_rcu_sheaf()
6242 pcs->rcu_free = pcs->spare; in __kfree_rcu_sheaf()
6243 pcs->spare = NULL; in __kfree_rcu_sheaf()
6249 local_unlock(&s->cpu_sheaves->lock); in __kfree_rcu_sheaf()
6256 pcs->rcu_free = empty; in __kfree_rcu_sheaf()
6260 local_unlock(&s->cpu_sheaves->lock); in __kfree_rcu_sheaf()
6267 if (!local_trylock(&s->cpu_sheaves->lock)) { in __kfree_rcu_sheaf()
6272 pcs = this_cpu_ptr(s->cpu_sheaves); in __kfree_rcu_sheaf()
6274 if (unlikely(pcs->rcu_free)) in __kfree_rcu_sheaf()
6277 pcs->rcu_free = empty; in __kfree_rcu_sheaf()
6282 rcu_sheaf = pcs->rcu_free; in __kfree_rcu_sheaf()
6288 rcu_sheaf->objects[rcu_sheaf->size++] = obj; in __kfree_rcu_sheaf()
6290 if (likely(rcu_sheaf->size < s->sheaf_capacity)) { in __kfree_rcu_sheaf()
6293 pcs->rcu_free = NULL; in __kfree_rcu_sheaf()
6294 rcu_sheaf->node = numa_mem_id(); in __kfree_rcu_sheaf()
6302 call_rcu(&rcu_sheaf->rcu_head, rcu_free_sheaf); in __kfree_rcu_sheaf()
6304 local_unlock(&s->cpu_sheaves->lock); in __kfree_rcu_sheaf()
6321 struct slub_percpu_sheaves *pcs; in free_to_pcs_bulk() local
6338 p[i] = p[--size]; in free_to_pcs_bulk()
6344 p[i] = p[--size]; in free_to_pcs_bulk()
6357 if (!local_trylock(&s->cpu_sheaves->lock)) in free_to_pcs_bulk()
6360 pcs = this_cpu_ptr(s->cpu_sheaves); in free_to_pcs_bulk()
6362 if (likely(pcs->main->size < s->sheaf_capacity)) in free_to_pcs_bulk()
6369 if (!pcs->spare) { in free_to_pcs_bulk()
6374 pcs->spare = pcs->main; in free_to_pcs_bulk()
6375 pcs->main = empty; in free_to_pcs_bulk()
6379 if (pcs->spare->size < s->sheaf_capacity) { in free_to_pcs_bulk()
6380 swap(pcs->main, pcs->spare); in free_to_pcs_bulk()
6384 empty = barn_replace_full_sheaf(barn, pcs->main); in free_to_pcs_bulk()
6391 pcs->main = empty; in free_to_pcs_bulk()
6394 main = pcs->main; in free_to_pcs_bulk()
6395 batch = min(size, s->sheaf_capacity - main->size); in free_to_pcs_bulk()
6397 memcpy(main->objects + main->size, p, batch * sizeof(void *)); in free_to_pcs_bulk()
6398 main->size += batch; in free_to_pcs_bulk()
6400 local_unlock(&s->cpu_sheaves->lock); in free_to_pcs_bulk()
6406 size -= batch; in free_to_pcs_bulk()
6416 local_unlock(&s->cpu_sheaves->lock); in free_to_pcs_bulk()
6450 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe
6457 struct llist_head *objs = &df->objects; in free_deferred_objects()
6458 struct llist_head *slabs = &df->slabs; in free_deferred_objects()
6471 s = slab->slab_cache; in free_deferred_objects()
6474 x -= s->offset; in free_deferred_objects()
6477 * We used freepointer in 'x' to link 'x' into df->objects. in free_deferred_objects()
6491 free_slab(slab->slab_cache, slab); in free_deferred_objects()
6493 if (slab->frozen) in free_deferred_objects()
6494 deactivate_slab(slab->slab_cache, slab, slab->flush_freelist); in free_deferred_objects()
6496 free_slab(slab->slab_cache, slab); in free_deferred_objects()
6508 if (llist_add(head + s->offset, &df->objects)) in defer_free()
6509 irq_work_queue(&df->work); in defer_free()
6516 slab->flush_freelist = flush_freelist; in defer_deactivate_slab()
6521 if (llist_add(&slab->llnode, &df->slabs)) in defer_deactivate_slab()
6522 irq_work_queue(&df->work); in defer_deactivate_slab()
6530 irq_work_sync(&per_cpu_ptr(&defer_free_objects, cpu)->work); in defer_free_barrier()
6566 c = raw_cpu_ptr(s->cpu_slab); in do_slab_free()
6567 tid = READ_ONCE(c->tid); in do_slab_free()
6572 if (unlikely(slab != c->slab)) { in do_slab_free()
6589 local_lock_is_locked(&s->cpu_slab->lock)) { in do_slab_free()
6597 freelist = READ_ONCE(c->freelist); in do_slab_free()
6610 c = this_cpu_ptr(s->cpu_slab); in do_slab_free()
6611 if (unlikely(slab != c->slab)) { in do_slab_free()
6615 tid = c->tid; in do_slab_free()
6616 freelist = c->freelist; in do_slab_free()
6619 c->freelist = head; in do_slab_free()
6620 c->tid = next_tid(tid); in do_slab_free()
6645 if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) || in slab_free()
6683 void *object = delayed_free->object; in slab_free_after_rcu_debug()
6695 s = slab->slab_cache; in slab_free_after_rcu_debug()
6696 if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU))) in slab_free_after_rcu_debug()
6719 return slab->slab_cache; in virt_to_cache()
6733 __func__, s->name, cachep->name)) in cache_from_obj()
6739 * kmem_cache_free - Deallocate an object
6761 dump_page(&folio->page, "Not a kmalloc allocation"); in free_large_kmalloc()
6773 -(PAGE_SIZE << order)); in free_large_kmalloc()
6775 free_frozen_pages(&folio->page, order); in free_large_kmalloc()
6808 s = slab->slab_cache; in kvfree_rcu_cb()
6816 obj = slab_addr + s->size * idx; in kvfree_rcu_cb()
6824 * kfree - free previously allocated memory
6848 s = slab->slab_cache; in kfree()
6879 s = slab->slab_cache; in kfree_nolock()
6885 * kmemleak_free_recursive(x, s->flags); in kfree_nolock()
6886 * debug_check_no_locks_freed(x, s->object_size); in kfree_nolock()
6887 * debug_check_no_obj_freed(x, s->object_size); in kfree_nolock()
6888 * __kcsan_check_access(x, s->object_size, ..); in kfree_nolock()
6898 * kasan_report_invalid_free() -> raw_spin_lock_irqsave() -> NMI in kfree_nolock()
6899 * -> kfree_nolock() -> kasan_report_invalid_free() on the same CPU in kfree_nolock()
6929 /* Check for double-free. */ in __do_krealloc()
6955 s = folio_slab(folio)->slab_cache; in __do_krealloc()
6957 ks = s->object_size; in __do_krealloc()
6973 memset(kasan_reset_tag(p) + orig_size, 0, new_size - orig_size); in __do_krealloc()
6975 memset(kasan_reset_tag(p) + new_size, 0, ks - new_size); in __do_krealloc()
6982 if (s->flags & SLAB_RED_ZONE && new_size < ks) in __do_krealloc()
6984 SLUB_RED_ACTIVE, ks - new_size); in __do_krealloc()
7003 * krealloc_node_align - reallocate memory. The contents will remain unchanged.
7014 * Documentation/core-api/memory-allocation.rst for more details.
7028 * |--------|----------------|
7064 * However make sure that larger requests are not too disruptive - i.e. in kmalloc_gfp_adjust()
7083 * __kvmalloc_node - attempt to allocate physically contiguous memory, but upon
7084 * failure, fall back to non-contiguous (vmalloc) allocation.
7088 * @flags: gfp mask for the allocation - must be compatible (superset) with GFP_KERNEL.
7092 * Documentation/core-api/memory-allocation.rst for more details.
7118 /* non-sleeping allocations are not supported by vmalloc */ in __kvmalloc_node_noprof()
7141 * kvfree() - Free memory.
7148 * Context: Either preemptible task context or not-NMI interrupt.
7160 * kvfree_sensitive - Free a data object containing sensitive information.
7178 * kvrealloc_node_align - reallocate memory; contents remain unchanged
7189 * Documentation/core-api/memory-allocation.rst for more details.
7262 object = p[--size]; in build_detached_freelist()
7265 /* Handle kalloc'ed objects */ in build_detached_freelist()
7268 df->slab = NULL; in build_detached_freelist()
7272 df->slab = folio_slab(folio); in build_detached_freelist()
7273 df->s = df->slab->slab_cache; in build_detached_freelist()
7275 df->slab = folio_slab(folio); in build_detached_freelist()
7276 df->s = cache_from_obj(s, object); /* Support for memcg */ in build_detached_freelist()
7280 df->tail = object; in build_detached_freelist()
7281 df->freelist = object; in build_detached_freelist()
7282 df->cnt = 1; in build_detached_freelist()
7287 set_freepointer(df->s, object, NULL); in build_detached_freelist()
7291 object = p[--size]; in build_detached_freelist()
7292 /* df->slab is always set at this point */ in build_detached_freelist()
7293 if (df->slab == virt_to_slab(object)) { in build_detached_freelist()
7295 set_freepointer(df->s, object, df->freelist); in build_detached_freelist()
7296 df->freelist = object; in build_detached_freelist()
7297 df->cnt++; in build_detached_freelist()
7298 same--; in build_detached_freelist()
7305 if (!--lookahead) in build_detached_freelist()
7346 if (s && s->cpu_sheaves) { in kmem_cache_free_bulk()
7378 c = slub_get_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
7379 local_lock_irqsave(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
7382 void *object = kfence_alloc(s, s->object_size, flags); in __kmem_cache_alloc_bulk()
7389 object = c->freelist; in __kmem_cache_alloc_bulk()
7392 * We may have removed an object from c->freelist using in __kmem_cache_alloc_bulk()
7394 * c->tid has not been bumped yet. in __kmem_cache_alloc_bulk()
7396 * allocating memory, we should bump c->tid now. in __kmem_cache_alloc_bulk()
7398 c->tid = next_tid(c->tid); in __kmem_cache_alloc_bulk()
7400 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
7403 * Invoking slow path likely have side-effect in __kmem_cache_alloc_bulk()
7404 * of re-populating per CPU c->freelist in __kmem_cache_alloc_bulk()
7407 _RET_IP_, c, s->object_size); in __kmem_cache_alloc_bulk()
7411 c = this_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
7414 local_lock_irqsave(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
7416 continue; /* goto for-loop */ in __kmem_cache_alloc_bulk()
7418 c->freelist = get_freepointer(s, object); in __kmem_cache_alloc_bulk()
7423 c->tid = next_tid(c->tid); in __kmem_cache_alloc_bulk()
7424 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags); in __kmem_cache_alloc_bulk()
7425 slub_put_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
7430 slub_put_cpu_ptr(s->cpu_slab); in __kmem_cache_alloc_bulk()
7442 void *object = kfence_alloc(s, s->object_size, flags); in __kmem_cache_alloc_bulk()
7450 _RET_IP_, s->object_size); in __kmem_cache_alloc_bulk()
7478 if (s->cpu_sheaves) in kmem_cache_alloc_bulk_noprof()
7486 if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) { in kmem_cache_alloc_bulk_noprof()
7498 slab_want_init_on_alloc(flags, s), s->object_size))) { in kmem_cache_alloc_bulk_noprof()
7605 return get_order(size * MAX_OBJS_PER_PAGE) - 1; in calculate_order()
7613 * smallest order from min_objects-derived/slab_min_order up to in calculate_order()
7635 return -ENOSYS; in calculate_order()
7641 n->nr_partial = 0; in init_kmem_cache_node()
7642 spin_lock_init(&n->list_lock); in init_kmem_cache_node()
7643 INIT_LIST_HEAD(&n->partial); in init_kmem_cache_node()
7645 atomic_long_set(&n->nr_slabs, 0); in init_kmem_cache_node()
7646 atomic_long_set(&n->total_objects, 0); in init_kmem_cache_node()
7647 INIT_LIST_HEAD(&n->full); in init_kmem_cache_node()
7649 n->barn = barn; in init_kmem_cache_node()
7665 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu), in alloc_kmem_cache_cpus()
7668 if (!s->cpu_slab) in alloc_kmem_cache_cpus()
7687 struct slub_percpu_sheaves *pcs; in init_percpu_sheaves() local
7689 pcs = per_cpu_ptr(s->cpu_sheaves, cpu); in init_percpu_sheaves()
7691 local_trylock_init(&pcs->lock); in init_percpu_sheaves()
7693 pcs->main = alloc_empty_sheaf(s, GFP_KERNEL); in init_percpu_sheaves()
7695 if (!pcs->main) in init_percpu_sheaves()
7696 return -ENOMEM; in init_percpu_sheaves()
7718 BUG_ON(kmem_cache_node->size < sizeof(struct kmem_cache_node)); in early_kmem_cache_node_alloc()
7728 n = slab->freelist; in early_kmem_cache_node_alloc()
7734 slab->freelist = get_freepointer(kmem_cache_node, n); in early_kmem_cache_node_alloc()
7735 slab->inuse = 1; in early_kmem_cache_node_alloc()
7736 kmem_cache_node->node[node] = n; in early_kmem_cache_node_alloc()
7738 inc_slabs_node(kmem_cache_node, node, slab->objects); in early_kmem_cache_node_alloc()
7753 if (n->barn) { in free_kmem_cache_nodes()
7754 WARN_ON(n->barn->nr_full); in free_kmem_cache_nodes()
7755 WARN_ON(n->barn->nr_empty); in free_kmem_cache_nodes()
7756 kfree(n->barn); in free_kmem_cache_nodes()
7757 n->barn = NULL; in free_kmem_cache_nodes()
7760 s->node[node] = NULL; in free_kmem_cache_nodes()
7768 if (s->cpu_sheaves) in __kmem_cache_release()
7772 if (s->cpu_slab) in __kmem_cache_release()
7773 lockdep_unregister_key(&s->lock_key); in __kmem_cache_release()
7775 free_percpu(s->cpu_slab); in __kmem_cache_release()
7793 if (s->cpu_sheaves) { in init_kmem_cache_nodes()
7809 s->node[node] = n; in init_kmem_cache_nodes()
7834 else if (s->size >= PAGE_SIZE) in set_cpu_partial()
7836 else if (s->size >= 1024) in set_cpu_partial()
7838 else if (s->size >= 256) in set_cpu_partial()
7853 slab_flags_t flags = s->flags; in calculate_sizes()
7854 unsigned int size = s->object_size; in calculate_sizes()
7871 !s->ctor) in calculate_sizes()
7872 s->flags |= __OBJECT_POISON; in calculate_sizes()
7874 s->flags &= ~__OBJECT_POISON; in calculate_sizes()
7882 if ((flags & SLAB_RED_ZONE) && size == s->object_size) in calculate_sizes()
7890 s->inuse = size; in calculate_sizes()
7892 if (((flags & SLAB_TYPESAFE_BY_RCU) && !args->use_freeptr_offset) || in calculate_sizes()
7893 (flags & SLAB_POISON) || s->ctor || in calculate_sizes()
7895 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) { in calculate_sizes()
7907 * The assumption that s->offset >= s->inuse means free in calculate_sizes()
7912 s->offset = size; in calculate_sizes()
7914 } else if ((flags & SLAB_TYPESAFE_BY_RCU) && args->use_freeptr_offset) { in calculate_sizes()
7915 s->offset = args->freeptr_offset; in calculate_sizes()
7922 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *)); in calculate_sizes()
7939 kasan_cache_create(s, &size, &s->flags); in calculate_sizes()
7951 s->red_left_pad = sizeof(void *); in calculate_sizes()
7952 s->red_left_pad = ALIGN(s->red_left_pad, s->align); in calculate_sizes()
7953 size += s->red_left_pad; in calculate_sizes()
7962 size = ALIGN(size, s->align); in calculate_sizes()
7963 s->size = size; in calculate_sizes()
7964 s->reciprocal_size = reciprocal_value(size); in calculate_sizes()
7970 s->allocflags = __GFP_COMP; in calculate_sizes()
7972 if (s->flags & SLAB_CACHE_DMA) in calculate_sizes()
7973 s->allocflags |= GFP_DMA; in calculate_sizes()
7975 if (s->flags & SLAB_CACHE_DMA32) in calculate_sizes()
7976 s->allocflags |= GFP_DMA32; in calculate_sizes()
7978 if (s->flags & SLAB_RECLAIM_ACCOUNT) in calculate_sizes()
7979 s->allocflags |= __GFP_RECLAIMABLE; in calculate_sizes()
7984 s->oo = oo_make(order, size); in calculate_sizes()
7985 s->min = oo_make(get_order(size), size); in calculate_sizes()
7987 return !!oo_objects(s->oo); in calculate_sizes()
8002 for_each_object(p, s, addr, slab->objects) { in list_slab_objects()
8007 pr_err("Object 0x%p @offset=%tu\n", p, p - addr); in list_slab_objects()
8028 spin_lock_irq(&n->list_lock); in free_partial()
8029 list_for_each_entry_safe(slab, h, &n->partial, slab_list) { in free_partial()
8030 if (!slab->inuse) { in free_partial()
8032 list_add(&slab->slab_list, &discard); in free_partial()
8037 spin_unlock_irq(&n->list_lock); in free_partial()
8049 if (n->nr_partial || node_nr_slabs(n)) in __kmem_cache_empty()
8065 if (s->cpu_sheaves) in __kmem_cache_shutdown()
8070 if (n->barn) in __kmem_cache_shutdown()
8071 barn_shrink(s, n->barn); in __kmem_cache_shutdown()
8073 if (n->nr_partial || node_nr_slabs(n)) in __kmem_cache_shutdown()
8087 struct kmem_cache *s = slab->slab_cache; in __kmem_obj_info()
8090 kpp->kp_ptr = object; in __kmem_obj_info()
8091 kpp->kp_slab = slab; in __kmem_obj_info()
8092 kpp->kp_slab_cache = s; in __kmem_obj_info()
8101 kpp->kp_data_offset = (unsigned long)((char *)objp0 - (char *)objp); in __kmem_obj_info()
8102 objp = base + s->size * objnr; in __kmem_obj_info()
8103 kpp->kp_objp = objp; in __kmem_obj_info()
8104 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size in __kmem_obj_info()
8105 || (objp - base) % s->size) || in __kmem_obj_info()
8106 !(s->flags & SLAB_STORE_USER)) in __kmem_obj_info()
8111 kpp->kp_ret = (void *)trackp->addr; in __kmem_obj_info()
8114 depot_stack_handle_t handle; in __kmem_obj_info() local
8118 handle = READ_ONCE(trackp->handle); in __kmem_obj_info()
8119 if (handle) { in __kmem_obj_info()
8120 nr_entries = stack_depot_fetch(handle, &entries); in __kmem_obj_info()
8122 kpp->kp_stack[i] = (void *)entries[i]; in __kmem_obj_info()
8126 handle = READ_ONCE(trackp->handle); in __kmem_obj_info()
8127 if (handle) { in __kmem_obj_info()
8128 nr_entries = stack_depot_fetch(handle, &entries); in __kmem_obj_info()
8130 kpp->kp_free_stack[i] = (void *)entries[i]; in __kmem_obj_info()
8216 s = slab->slab_cache; in __check_heap_object()
8225 offset = ptr - kfence_object_start(ptr); in __check_heap_object()
8227 offset = (ptr - slab_address(slab)) % s->size; in __check_heap_object()
8231 if (offset < s->red_left_pad) in __check_heap_object()
8233 s->name, to_user, offset, n); in __check_heap_object()
8234 offset -= s->red_left_pad; in __check_heap_object()
8238 if (offset >= s->useroffset && in __check_heap_object()
8239 offset - s->useroffset <= s->usersize && in __check_heap_object()
8240 n <= s->useroffset - offset + s->usersize) in __check_heap_object()
8243 usercopy_abort("SLUB object", s->name, to_user, offset, n); in __check_heap_object()
8275 if (n->barn) in __kmem_cache_do_shrink()
8276 barn_shrink(s, n->barn); in __kmem_cache_do_shrink()
8278 spin_lock_irqsave(&n->list_lock, flags); in __kmem_cache_do_shrink()
8284 * list_lock. slab->inuse here is the upper limit. in __kmem_cache_do_shrink()
8286 list_for_each_entry_safe(slab, t, &n->partial, slab_list) { in __kmem_cache_do_shrink()
8287 int free = slab->objects - slab->inuse; in __kmem_cache_do_shrink()
8289 /* Do not reread slab->inuse */ in __kmem_cache_do_shrink()
8295 if (free == slab->objects) { in __kmem_cache_do_shrink()
8296 list_move(&slab->slab_list, &discard); in __kmem_cache_do_shrink()
8298 n->nr_partial--; in __kmem_cache_do_shrink()
8299 dec_slabs_node(s, node, slab->objects); in __kmem_cache_do_shrink()
8301 list_move(&slab->slab_list, promote + free - 1); in __kmem_cache_do_shrink()
8308 for (i = SHRINK_PROMOTE_MAX - 1; i >= 0; i--) in __kmem_cache_do_shrink()
8309 list_splice(promote + i, &n->partial); in __kmem_cache_do_shrink()
8311 spin_unlock_irqrestore(&n->list_lock, flags); in __kmem_cache_do_shrink()
8366 if (s->cpu_sheaves) { in slab_mem_going_online_callback()
8370 ret = -ENOMEM; in slab_mem_going_online_callback()
8383 ret = -ENOMEM; in slab_mem_going_online_callback()
8389 s->node[nid] = n; in slab_mem_going_online_callback()
8405 int nid = nn->nid; in slab_memory_callback()
8439 memcpy(s, static_cache, kmem_cache->object_size); in bootstrap()
8450 list_for_each_entry(p, &n->partial, slab_list) in bootstrap()
8451 p->slab_cache = s; in bootstrap()
8454 list_for_each_entry(p, &n->full, slab_list) in bootstrap()
8455 p->slab_cache = s; in bootstrap()
8458 list_add(&s->list, &slab_caches); in bootstrap()
8511 pr_info("SLUB: HWalign=%d, Order=%u-%u, MinObjects=%u, CPUs=%u, Nodes=%u\n", in kmem_cache_init()
8537 s->refcount++; in __kmem_cache_alias()
8543 s->object_size = max(s->object_size, size); in __kmem_cache_alias()
8544 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *))); in __kmem_cache_alias()
8554 int err = -EINVAL; in do_kmem_cache_create()
8556 s->name = name; in do_kmem_cache_create()
8557 s->size = s->object_size = size; in do_kmem_cache_create()
8559 s->flags = kmem_cache_flags(flags, s->name); in do_kmem_cache_create()
8561 s->random = get_random_long(); in do_kmem_cache_create()
8563 s->align = args->align; in do_kmem_cache_create()
8564 s->ctor = args->ctor; in do_kmem_cache_create()
8566 s->useroffset = args->useroffset; in do_kmem_cache_create()
8567 s->usersize = args->usersize; in do_kmem_cache_create()
8577 if (get_order(s->size) > get_order(s->object_size)) { in do_kmem_cache_create()
8578 s->flags &= ~DEBUG_METADATA_FLAGS; in do_kmem_cache_create()
8579 s->offset = 0; in do_kmem_cache_create()
8586 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) { in do_kmem_cache_create()
8588 s->flags |= __CMPXCHG_DOUBLE; in do_kmem_cache_create()
8596 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2); in do_kmem_cache_create()
8597 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial); in do_kmem_cache_create()
8601 if (args->sheaf_capacity && !IS_ENABLED(CONFIG_SLUB_TINY) in do_kmem_cache_create()
8602 && !(s->flags & SLAB_DEBUG_FLAGS)) { in do_kmem_cache_create()
8603 s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves); in do_kmem_cache_create()
8604 if (!s->cpu_sheaves) { in do_kmem_cache_create()
8605 err = -ENOMEM; in do_kmem_cache_create()
8609 s->sheaf_capacity = args->sheaf_capacity; in do_kmem_cache_create()
8613 s->remote_node_defrag_ratio = 1000; in do_kmem_cache_create()
8616 /* Initialize the pre-computed randomized freelist if slab is up */ in do_kmem_cache_create()
8628 if (s->cpu_sheaves) { in do_kmem_cache_create()
8645 pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name); in do_kmem_cache_create()
8647 if (s->flags & SLAB_STORE_USER) in do_kmem_cache_create()
8659 return slab->inuse; in count_inuse()
8664 return slab->objects; in count_total()
8685 for_each_object(p, s, addr, slab->objects) { in validate_slab()
8701 spin_lock_irqsave(&n->list_lock, flags); in validate_slab_node()
8703 list_for_each_entry(slab, &n->partial, slab_list) { in validate_slab_node()
8707 if (count != n->nr_partial) { in validate_slab_node()
8709 s->name, count, n->nr_partial); in validate_slab_node()
8713 if (!(s->flags & SLAB_STORE_USER)) in validate_slab_node()
8716 list_for_each_entry(slab, &n->full, slab_list) { in validate_slab_node()
8722 s->name, count, node_nr_slabs(n)); in validate_slab_node()
8727 spin_unlock_irqrestore(&n->list_lock, flags); in validate_slab_node()
8738 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); in validate_slab_cache()
8740 return -ENOMEM; in validate_slab_cache()
8759 depot_stack_handle_t handle; member
8783 if (t->max) in free_loc_track()
8784 free_pages((unsigned long)t->loc, in free_loc_track()
8785 get_order(sizeof(struct location) * t->max)); in free_loc_track()
8799 if (t->count) { in alloc_loc_track()
8800 memcpy(l, t->loc, sizeof(struct location) * t->count); in alloc_loc_track()
8803 t->max = max; in alloc_loc_track()
8804 t->loc = l; in alloc_loc_track()
8815 unsigned long age = jiffies - track->when; in add_location()
8816 depot_stack_handle_t handle = 0; in add_location() local
8817 unsigned int waste = s->object_size - orig_size; in add_location()
8820 handle = READ_ONCE(track->handle); in add_location()
8822 start = -1; in add_location()
8823 end = t->count; in add_location()
8826 pos = start + (end - start + 1) / 2; in add_location()
8835 l = &t->loc[pos]; in add_location()
8836 caddr = l->addr; in add_location()
8837 chandle = l->handle; in add_location()
8838 cwaste = l->waste; in add_location()
8839 if ((track->addr == caddr) && (handle == chandle) && in add_location()
8842 l->count++; in add_location()
8843 if (track->when) { in add_location()
8844 l->sum_time += age; in add_location()
8845 if (age < l->min_time) in add_location()
8846 l->min_time = age; in add_location()
8847 if (age > l->max_time) in add_location()
8848 l->max_time = age; in add_location()
8850 if (track->pid < l->min_pid) in add_location()
8851 l->min_pid = track->pid; in add_location()
8852 if (track->pid > l->max_pid) in add_location()
8853 l->max_pid = track->pid; in add_location()
8855 cpumask_set_cpu(track->cpu, in add_location()
8856 to_cpumask(l->cpus)); in add_location()
8858 node_set(page_to_nid(virt_to_page(track)), l->nodes); in add_location()
8862 if (track->addr < caddr) in add_location()
8864 else if (track->addr == caddr && handle < chandle) in add_location()
8866 else if (track->addr == caddr && handle == chandle && in add_location()
8876 if (t->count >= t->max && !alloc_loc_track(t, 2 * t->max, GFP_ATOMIC)) in add_location()
8879 l = t->loc + pos; in add_location()
8880 if (pos < t->count) in add_location()
8882 (t->count - pos) * sizeof(struct location)); in add_location()
8883 t->count++; in add_location()
8884 l->count = 1; in add_location()
8885 l->addr = track->addr; in add_location()
8886 l->sum_time = age; in add_location()
8887 l->min_time = age; in add_location()
8888 l->max_time = age; in add_location()
8889 l->min_pid = track->pid; in add_location()
8890 l->max_pid = track->pid; in add_location()
8891 l->handle = handle; in add_location()
8892 l->waste = waste; in add_location()
8893 cpumask_clear(to_cpumask(l->cpus)); in add_location()
8894 cpumask_set_cpu(track->cpu, to_cpumask(l->cpus)); in add_location()
8895 nodes_clear(l->nodes); in add_location()
8896 node_set(page_to_nid(virt_to_page(track)), l->nodes); in add_location()
8910 for_each_object(p, s, addr, slab->objects) in process_slab()
8914 s->object_size); in process_slab()
8945 return -ENOMEM; in show_slab_objects()
8951 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, in show_slab_objects()
8956 slab = READ_ONCE(c->slab); in show_slab_objects()
8962 x = slab->objects; in show_slab_objects()
8964 x = slab->inuse; in show_slab_objects()
8980 x = data_race(slab->slabs); in show_slab_objects()
8992 * mem_hotplug_lock->slab_mutex->kernfs_mutex in show_slab_objects()
8996 * unplug code doesn't destroy the kmem_cache->node[] data. in show_slab_objects()
9008 x = node_nr_objs(n) - count_partial(n, count_free); in show_slab_objects()
9026 x = n->nr_partial; in show_slab_objects()
9063 return sysfs_emit(buf, "%u\n", s->size); in slab_size_show()
9069 return sysfs_emit(buf, "%u\n", s->align); in align_show()
9075 return sysfs_emit(buf, "%u\n", s->object_size); in object_size_show()
9081 return sysfs_emit(buf, "%u\n", oo_objects(s->oo)); in objs_per_slab_show()
9087 return sysfs_emit(buf, "%u\n", oo_order(s->oo)); in order_show()
9093 return sysfs_emit(buf, "%u\n", s->sheaf_capacity); in sheaf_capacity_show()
9099 return sysfs_emit(buf, "%lu\n", s->min_partial); in min_partial_show()
9112 s->min_partial = min; in min_partial_store()
9121 nr_partial = s->cpu_partial; in cpu_partial_show()
9137 return -EINVAL; in cpu_partial_store()
9147 if (!s->ctor) in ctor_show()
9149 return sysfs_emit(buf, "%pS\n", s->ctor); in ctor_show()
9155 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1); in aliases_show()
9188 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
9191 slabs += data_race(slab->slabs); in slabs_cpu_partial_show()
9195 /* Approximate half-full slabs, see slub_set_cpu_partial() */ in slabs_cpu_partial_show()
9196 objects = (slabs * oo_objects(s->oo)) / 2; in slabs_cpu_partial_show()
9203 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu)); in slabs_cpu_partial_show()
9205 slabs = data_race(slab->slabs); in slabs_cpu_partial_show()
9206 objects = (slabs * oo_objects(s->oo)) / 2; in slabs_cpu_partial_show()
9220 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT)); in reclaim_account_show()
9226 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN)); in hwcache_align_show()
9233 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA)); in cache_dma_show()
9241 return sysfs_emit(buf, "%u\n", s->usersize); in usersize_show()
9248 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU)); in destroy_by_rcu_show()
9273 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS)); in sanity_checks_show()
9279 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE)); in trace_show()
9285 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE)); in red_zone_show()
9292 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON)); in poison_show()
9299 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER)); in store_user_show()
9312 int ret = -EINVAL; in validate_store()
9328 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB)); in failslab_show()
9334 if (s->refcount > 1) in failslab_store()
9335 return -EINVAL; in failslab_store()
9338 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB); in failslab_store()
9340 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB); in failslab_store()
9358 return -EINVAL; in shrink_store()
9366 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10); in remote_node_defrag_ratio_show()
9379 return -ERANGE; in remote_node_defrag_ratio_store()
9381 s->remote_node_defrag_ratio = ratio * 10; in remote_node_defrag_ratio_store()
9397 return -ENOMEM; in show_stat()
9400 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si]; in show_stat()
9426 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0; in clear_stat()
9438 return -EINVAL; \
9492 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE)); in skip_kfence_show()
9501 s->flags &= ~SLAB_SKIP_KFENCE; in skip_kfence_store()
9503 s->flags |= SLAB_SKIP_KFENCE; in skip_kfence_store()
9505 ret = -EINVAL; in skip_kfence_store()
9620 if (!attribute->show) in slab_attr_show()
9621 return -EIO; in slab_attr_show()
9623 return attribute->show(s, buf); in slab_attr_show()
9636 if (!attribute->store) in slab_attr_store()
9637 return -EIO; in slab_attr_store()
9639 return attribute->store(s, buf, len); in slab_attr_store()
9668 * Format :[flags-]size
9676 return ERR_PTR(-ENOMEM); in create_unique_id()
9686 if (s->flags & SLAB_CACHE_DMA) in create_unique_id()
9688 if (s->flags & SLAB_CACHE_DMA32) in create_unique_id()
9690 if (s->flags & SLAB_RECLAIM_ACCOUNT) in create_unique_id()
9692 if (s->flags & SLAB_CONSISTENCY_CHECKS) in create_unique_id()
9694 if (s->flags & SLAB_ACCOUNT) in create_unique_id()
9697 *p++ = '-'; in create_unique_id()
9698 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size); in create_unique_id()
9700 if (WARN_ON(p > name + ID_STR_LENGTH - 1)) { in create_unique_id()
9702 return ERR_PTR(-EINVAL); in create_unique_id()
9704 kmsan_unpoison_memory(name, p - name); in create_unique_id()
9725 sysfs_remove_link(&slab_kset->kobj, s->name); in sysfs_slab_add()
9726 name = s->name; in sysfs_slab_add()
9737 s->kobj.kset = kset; in sysfs_slab_add()
9738 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name); in sysfs_slab_add()
9742 err = sysfs_create_group(&s->kobj, &slab_attr_group); in sysfs_slab_add()
9748 sysfs_slab_alias(s, s->name); in sysfs_slab_add()
9755 kobject_del(&s->kobj); in sysfs_slab_add()
9761 if (s->kobj.state_in_sysfs) in sysfs_slab_unlink()
9762 kobject_del(&s->kobj); in sysfs_slab_unlink()
9767 kobject_put(&s->kobj); in sysfs_slab_release()
9790 sysfs_remove_link(&slab_kset->kobj, name); in sysfs_slab_alias()
9793 * In that case, sysfs_create_link() returns -ENOENT and in sysfs_slab_alias()
9796 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name); in sysfs_slab_alias()
9801 return -ENOMEM; in sysfs_slab_alias()
9803 al->s = s; in sysfs_slab_alias()
9804 al->name = name; in sysfs_slab_alias()
9805 al->next = alias_list; in sysfs_slab_alias()
9822 return -ENOMEM; in slab_sysfs_init()
9831 s->name); in slab_sysfs_init()
9837 alias_list = alias_list->next; in slab_sysfs_init()
9838 err = sysfs_slab_alias(al->s, al->name); in slab_sysfs_init()
9841 al->name); in slab_sysfs_init()
9854 struct loc_track *t = seq->private; in slab_debugfs_show()
9858 idx = (unsigned long) t->idx; in slab_debugfs_show()
9859 if (idx < t->count) { in slab_debugfs_show()
9860 l = &t->loc[idx]; in slab_debugfs_show()
9862 seq_printf(seq, "%7ld ", l->count); in slab_debugfs_show()
9864 if (l->addr) in slab_debugfs_show()
9865 seq_printf(seq, "%pS", (void *)l->addr); in slab_debugfs_show()
9867 seq_puts(seq, "<not-available>"); in slab_debugfs_show()
9869 if (l->waste) in slab_debugfs_show()
9871 l->count * l->waste, l->waste); in slab_debugfs_show()
9873 if (l->sum_time != l->min_time) { in slab_debugfs_show()
9875 l->min_time, div_u64(l->sum_time, l->count), in slab_debugfs_show()
9876 l->max_time); in slab_debugfs_show()
9878 seq_printf(seq, " age=%ld", l->min_time); in slab_debugfs_show()
9880 if (l->min_pid != l->max_pid) in slab_debugfs_show()
9881 seq_printf(seq, " pid=%ld-%ld", l->min_pid, l->max_pid); in slab_debugfs_show()
9884 l->min_pid); in slab_debugfs_show()
9886 if (num_online_cpus() > 1 && !cpumask_empty(to_cpumask(l->cpus))) in slab_debugfs_show()
9888 cpumask_pr_args(to_cpumask(l->cpus))); in slab_debugfs_show()
9890 if (nr_online_nodes > 1 && !nodes_empty(l->nodes)) in slab_debugfs_show()
9892 nodemask_pr_args(&l->nodes)); in slab_debugfs_show()
9896 depot_stack_handle_t handle; in slab_debugfs_show() local
9900 handle = READ_ONCE(l->handle); in slab_debugfs_show()
9901 if (handle) { in slab_debugfs_show()
9902 nr_entries = stack_depot_fetch(handle, &entries); in slab_debugfs_show()
9912 if (!idx && !t->count) in slab_debugfs_show()
9924 struct loc_track *t = seq->private; in slab_debugfs_next()
9926 t->idx = ++(*ppos); in slab_debugfs_next()
9927 if (*ppos <= t->count) in slab_debugfs_next()
9938 return cmp_int(loc2->count, loc1->count); in cmp_loc_by_count()
9943 struct loc_track *t = seq->private; in slab_debugfs_start()
9945 t->idx = *ppos; in slab_debugfs_start()
9964 struct kmem_cache *s = file_inode(filep)->i_private; in slab_debug_trace_open()
9968 return -ENOMEM; in slab_debug_trace_open()
9970 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL); in slab_debug_trace_open()
9973 return -ENOMEM; in slab_debug_trace_open()
9981 return -ENOMEM; in slab_debug_trace_open()
9991 spin_lock_irqsave(&n->list_lock, flags); in slab_debug_trace_open()
9992 list_for_each_entry(slab, &n->partial, slab_list) in slab_debug_trace_open()
9994 list_for_each_entry(slab, &n->full, slab_list) in slab_debug_trace_open()
9996 spin_unlock_irqrestore(&n->list_lock, flags); in slab_debug_trace_open()
10000 sort(t->loc, t->count, sizeof(struct location), in slab_debug_trace_open()
10009 struct seq_file *seq = file->private_data; in slab_debug_trace_release()
10010 struct loc_track *t = seq->private; in slab_debug_trace_release()
10030 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root); in debugfs_slab_add()
10041 debugfs_lookup_and_remove(s->name, slab_debugfs_root); in debugfs_slab_release()
10051 if (s->flags & SLAB_STORE_USER) in slab_debugfs_init()
10077 sinfo->active_objs = nr_objs - nr_free; in get_slabinfo()
10078 sinfo->num_objs = nr_objs; in get_slabinfo()
10079 sinfo->active_slabs = nr_slabs; in get_slabinfo()
10080 sinfo->num_slabs = nr_slabs; in get_slabinfo()
10081 sinfo->objects_per_slab = oo_objects(s->oo); in get_slabinfo()
10082 sinfo->cache_order = oo_order(s->oo); in get_slabinfo()