Lines Matching refs:s
89 * slab's freelist.
205 * We could simply use migrate_disable()/enable() but as long as it's a
251 static inline bool kmem_cache_debug(struct kmem_cache *s)
253 return kmem_cache_debug_flags(s, SLAB_DEBUG_FLAGS);
256 void *fixup_red_left(struct kmem_cache *s, void *p)
258 if (kmem_cache_debug_flags(s, SLAB_RED_ZONE))
259 p += s->red_left_pad;
264 static inline bool kmem_cache_has_cpu_partial(struct kmem_cache *s)
267 return !kmem_cache_debug(s);
315 * disabled when slab_debug=O is used and a cache's min order increases with
355 static inline int sysfs_slab_add(struct kmem_cache *s) { return 0; }
356 static inline int sysfs_slab_alias(struct kmem_cache *s, const char *p)
363 static inline void debugfs_slab_add(struct kmem_cache *s) { }
439 static inline void stat(const struct kmem_cache *s, enum stat_item si)
444 * avoid this_cpu_add()'s irq-disable overhead.
446 raw_cpu_inc(s->cpu_slab->stat[si]);
451 void stat_add(const struct kmem_cache *s, enum stat_item si, int v)
454 raw_cpu_add(s->cpu_slab->stat[si], v);
507 static inline struct kmem_cache_node *get_node(struct kmem_cache *s, int node)
509 return s->node[node];
513 * Get the barn of the current cpu's closest memory node. It may not exist on
516 static inline struct node_barn *get_barn(struct kmem_cache *s)
518 struct kmem_cache_node *n = get_node(s, numa_mem_id());
549 struct kmem_cache *s;
565 static inline freeptr_t freelist_ptr_encode(const struct kmem_cache *s,
571 encoded = (unsigned long)ptr ^ s->random ^ swab(ptr_addr);
578 static inline void *freelist_ptr_decode(const struct kmem_cache *s,
584 decoded = (void *)(ptr.v ^ s->random ^ swab(ptr_addr));
591 static inline void *get_freepointer(struct kmem_cache *s, void *object)
597 ptr_addr = (unsigned long)object + s->offset;
599 return freelist_ptr_decode(s, p, ptr_addr);
602 static void prefetch_freepointer(const struct kmem_cache *s, void *object)
604 prefetchw(object + s->offset);
618 static inline void *get_freepointer_safe(struct kmem_cache *s, void *object)
624 return get_freepointer(s, object);
627 freepointer_addr = (unsigned long)object + s->offset;
629 return freelist_ptr_decode(s, p, freepointer_addr);
632 static inline void set_freepointer(struct kmem_cache *s, void *object, void *fp)
634 unsigned long freeptr_addr = (unsigned long)object + s->offset;
641 *(freeptr_t *)freeptr_addr = freelist_ptr_encode(s, fp, freeptr_addr);
647 static inline bool freeptr_outside_object(struct kmem_cache *s)
649 return s->offset >= s->inuse;
656 static inline unsigned int get_info_end(struct kmem_cache *s)
658 if (freeptr_outside_object(s))
659 return s->inuse + sizeof(void *);
661 return s->inuse;
696 static void slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
700 s->cpu_partial = nr_objects;
708 nr_slabs = DIV_ROUND_UP(nr_objects * 2, oo_objects(s->oo));
709 s->cpu_partial_slabs = nr_slabs;
712 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
714 return s->cpu_partial_slabs;
719 slub_set_cpu_partial(struct kmem_cache *s, unsigned int nr_objects)
724 static inline unsigned int slub_get_cpu_partial(struct kmem_cache *s)
800 static inline bool __slab_update_freelist(struct kmem_cache *s, struct slab *slab,
808 if (s->flags & __CMPXCHG_DOUBLE)
817 stat(s, CMPXCHG_DOUBLE_FAIL);
820 pr_info("%s %s: cmpxchg double redo ", n, s->name);
826 static inline bool slab_update_freelist(struct kmem_cache *s, struct slab *slab,
831 if (s->flags & __CMPXCHG_DOUBLE) {
844 stat(s, CMPXCHG_DOUBLE_FAIL);
847 pr_info("%s %s: cmpxchg double redo ", n, s->name);
859 static inline void set_orig_size(struct kmem_cache *s,
864 if (!slub_debug_orig_size(s))
867 p += get_info_end(s);
873 static inline unsigned int get_orig_size(struct kmem_cache *s, void *object)
880 if (!slub_debug_orig_size(s))
881 return s->object_size;
883 p += get_info_end(s);
903 static void __fill_map(unsigned long *obj_map, struct kmem_cache *s,
911 for (p = slab->freelist; p; p = get_freepointer(s, p))
912 set_bit(__obj_to_index(s, addr, p), obj_map);
950 static inline unsigned int size_from_object(struct kmem_cache *s)
952 if (s->flags & SLAB_RED_ZONE)
953 return s->size - s->red_left_pad;
955 return s->size;
958 static inline void *restore_red_left(struct kmem_cache *s, void *p)
960 if (s->flags & SLAB_RED_ZONE)
961 p -= s->red_left_pad;
1001 static inline int check_valid_pointer(struct kmem_cache *s,
1011 object = restore_red_left(s, object);
1012 if (object < base || object >= base + slab->objects * s->size ||
1013 (object - base) % s->size) {
1029 static struct track *get_track(struct kmem_cache *s, void *object,
1034 p = object + get_info_end(s);
1058 static void set_track_update(struct kmem_cache *s, void *object,
1062 struct track *p = get_track(s, object, alloc);
1073 static __always_inline void set_track(struct kmem_cache *s, void *object,
1078 set_track_update(s, object, alloc, addr, handle);
1081 static void init_tracking(struct kmem_cache *s, void *object)
1085 if (!(s->flags & SLAB_STORE_USER))
1088 p = get_track(s, object, TRACK_ALLOC);
1092 static void print_track(const char *s, struct track *t, unsigned long pr_time)
1099 pr_err("%s in %pS age=%lu cpu=%u pid=%d\n",
1100 s, (void *)t->addr, pr_time - t->when, t->cpu, t->pid);
1110 void print_tracking(struct kmem_cache *s, void *object)
1113 if (!(s->flags & SLAB_STORE_USER))
1116 print_track("Allocated", get_track(s, object, TRACK_ALLOC), pr_time);
1117 print_track("Freed", get_track(s, object, TRACK_FREE), pr_time);
1127 void skip_orig_size_check(struct kmem_cache *s, const void *object)
1129 set_orig_size(s, (void *)object, s->object_size);
1132 static void __slab_bug(struct kmem_cache *s, const char *fmt, va_list argsp)
1141 pr_err("BUG %s (%s): %pV\n", s ? s->name : "<unknown>", print_tainted(), &vaf);
1146 static void slab_bug(struct kmem_cache *s, const char *fmt, ...)
1151 __slab_bug(s, fmt, args);
1156 static void slab_fix(struct kmem_cache *s, const char *fmt, ...)
1167 pr_err("FIX %s: %pV\n", s->name, &vaf);
1171 static void print_trailer(struct kmem_cache *s, struct slab *slab, u8 *p)
1176 print_tracking(s, p);
1181 p, p - addr, get_freepointer(s, p));
1183 if (s->flags & SLAB_RED_ZONE)
1184 print_section(KERN_ERR, "Redzone ", p - s->red_left_pad,
1185 s->red_left_pad);
1190 min_t(unsigned int, s->object_size, PAGE_SIZE));
1191 if (s->flags & SLAB_RED_ZONE)
1192 print_section(KERN_ERR, "Redzone ", p + s->object_size,
1193 s->inuse - s->object_size);
1195 off = get_info_end(s);
1197 if (s->flags & SLAB_STORE_USER)
1200 if (slub_debug_orig_size(s))
1203 off += kasan_metadata_size(s, false);
1205 if (off != size_from_object(s))
1208 size_from_object(s) - off);
1211 static void object_err(struct kmem_cache *s, struct slab *slab,
1217 slab_bug(s, reason);
1218 if (!object || !check_valid_pointer(s, slab, object)) {
1222 print_trailer(s, slab, object);
1229 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
1232 if ((s->flags & SLAB_CONSISTENCY_CHECKS) &&
1233 !check_valid_pointer(s, slab, nextfree) && freelist) {
1234 object_err(s, slab, *freelist, "Freechain corrupt");
1236 slab_fix(s, "Isolate corrupted freechain");
1254 static __printf(3, 4) void slab_err(struct kmem_cache *s, struct slab *slab,
1263 __slab_bug(s, fmt, args);
1269 static void init_object(struct kmem_cache *s, void *object, u8 val)
1272 unsigned int poison_size = s->object_size;
1274 if (s->flags & SLAB_RED_ZONE) {
1280 memset_no_sanitize_memory(p - s->red_left_pad, val,
1281 s->red_left_pad);
1283 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1289 poison_size = get_orig_size(s, object);
1293 if (s->flags & __OBJECT_POISON) {
1298 if (s->flags & SLAB_RED_ZONE)
1300 s->inuse - poison_size);
1303 static void restore_bytes(struct kmem_cache *s, const char *message, u8 data,
1306 slab_fix(s, "Restoring %s 0x%p-0x%p=0x%x", message, from, to - 1, data);
1317 check_bytes_and_report(struct kmem_cache *s, struct slab *slab,
1338 pr_err("[%s overwritten] 0x%p-0x%p @offset=%tu. First byte 0x%x instead of 0x%x\n",
1342 object_err(s, slab, object, "Object corrupt");
1345 restore_bytes(s, what, value, fault, end);
1360 * object + s->object_size
1368 * object + s->inuse
1380 * object + s->size
1381 * Nothing is used beyond s->size.
1388 static int check_pad_bytes(struct kmem_cache *s, struct slab *slab, u8 *p)
1390 unsigned long off = get_info_end(s); /* The end of info */
1392 if (s->flags & SLAB_STORE_USER) {
1396 if (s->flags & SLAB_KMALLOC)
1400 off += kasan_metadata_size(s, false);
1402 if (size_from_object(s) == off)
1405 return check_bytes_and_report(s, slab, p, "Object padding",
1406 p + off, POISON_INUSE, size_from_object(s) - off, true);
1411 slab_pad_check(struct kmem_cache *s, struct slab *slab)
1420 if (!(s->flags & SLAB_POISON))
1426 remainder = length % s->size;
1439 slab_bug(s, "Padding overwritten. 0x%p-0x%p @offset=%tu",
1444 restore_bytes(s, "slab padding", POISON_INUSE, fault, end);
1447 static int check_object(struct kmem_cache *s, struct slab *slab,
1451 u8 *endobject = object + s->object_size;
1455 if (s->flags & SLAB_RED_ZONE) {
1456 if (!check_bytes_and_report(s, slab, object, "Left Redzone",
1457 object - s->red_left_pad, val, s->red_left_pad, ret))
1460 if (!check_bytes_and_report(s, slab, object, "Right Redzone",
1461 endobject, val, s->inuse - s->object_size, ret))
1464 if (slub_debug_orig_size(s) && val == SLUB_RED_ACTIVE) {
1465 orig_size = get_orig_size(s, object);
1467 if (s->object_size > orig_size &&
1468 !check_bytes_and_report(s, slab, object,
1470 val, s->object_size - orig_size, ret)) {
1475 if ((s->flags & SLAB_POISON) && s->object_size < s->inuse) {
1476 if (!check_bytes_and_report(s, slab, p, "Alignment padding",
1478 s->inuse - s->object_size, ret))
1483 if (s->flags & SLAB_POISON) {
1484 if (val != SLUB_RED_ACTIVE && (s->flags & __OBJECT_POISON)) {
1490 kasan_meta_size = kasan_metadata_size(s, true);
1491 if (kasan_meta_size < s->object_size - 1 &&
1492 !check_bytes_and_report(s, slab, p, "Poison",
1494 s->object_size - kasan_meta_size - 1, ret))
1496 if (kasan_meta_size < s->object_size &&
1497 !check_bytes_and_report(s, slab, p, "End Poison",
1498 p + s->object_size - 1, POISON_END, 1, ret))
1504 if (!check_pad_bytes(s, slab, p))
1512 if ((freeptr_outside_object(s) || val != SLUB_RED_ACTIVE) &&
1513 !check_valid_pointer(s, slab, get_freepointer(s, p))) {
1514 object_err(s, slab, p, "Freepointer corrupt");
1520 set_freepointer(s, p, NULL);
1529 * was either obtained in a way that ensures it's valid, or validated
1532 static int check_slab(struct kmem_cache *s, struct slab *slab)
1536 maxobj = order_objects(slab_order(slab), s->size);
1538 slab_err(s, slab, "objects %u > max %u",
1543 slab_err(s, slab, "inuse %u > max %u",
1548 slab_err(s, slab, "Slab disabled since SLUB metadata consistency check failed");
1553 slab_pad_check(s, slab);
1561 static bool on_freelist(struct kmem_cache *s, struct slab *slab, void *search)
1572 if (!check_valid_pointer(s, slab, fp)) {
1574 object_err(s, slab, object,
1576 set_freepointer(s, object, NULL);
1579 slab_err(s, slab, "Freepointer corrupt");
1582 slab_fix(s, "Freelist cleared");
1587 fp = get_freepointer(s, object);
1592 slab_err(s, slab, "Freelist cycle detected");
1595 slab_fix(s, "Freelist cleared");
1599 max_objects = order_objects(slab_order(slab), s->size);
1604 slab_err(s, slab, "Wrong number of objects. Found %d but should be %d",
1607 slab_fix(s, "Number of objects adjusted");
1610 slab_err(s, slab, "Wrong object count. Counter is %d but counted were %d",
1613 slab_fix(s, "Object count adjusted");
1618 static void trace(struct kmem_cache *s, struct slab *slab, void *object,
1621 if (s->flags & SLAB_TRACE) {
1622 pr_info("TRACE %s %s 0x%p inuse=%d fp=0x%p\n",
1623 s->name,
1630 s->object_size);
1639 static void add_full(struct kmem_cache *s,
1642 if (!(s->flags & SLAB_STORE_USER))
1649 static void remove_full(struct kmem_cache *s, struct kmem_cache_node *n, struct slab *slab)
1651 if (!(s->flags & SLAB_STORE_USER))
1663 static inline void inc_slabs_node(struct kmem_cache *s, int node, int objects)
1665 struct kmem_cache_node *n = get_node(s, node);
1670 static inline void dec_slabs_node(struct kmem_cache *s, int node, int objects)
1672 struct kmem_cache_node *n = get_node(s, node);
1679 static void setup_object_debug(struct kmem_cache *s, void *object)
1681 if (!kmem_cache_debug_flags(s, SLAB_STORE_USER|SLAB_RED_ZONE|__OBJECT_POISON))
1684 init_object(s, object, SLUB_RED_INACTIVE);
1685 init_tracking(s, object);
1689 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr)
1691 if (!kmem_cache_debug_flags(s, SLAB_POISON))
1699 static inline int alloc_consistency_checks(struct kmem_cache *s,
1702 if (!check_slab(s, slab))
1705 if (!check_valid_pointer(s, slab, object)) {
1706 object_err(s, slab, object, "Freelist Pointer check fails");
1710 if (!check_object(s, slab, object, SLUB_RED_INACTIVE))
1716 static noinline bool alloc_debug_processing(struct kmem_cache *s,
1719 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
1720 if (!alloc_consistency_checks(s, slab, object))
1725 trace(s, slab, object, 1);
1726 set_orig_size(s, object, orig_size);
1727 init_object(s, object, SLUB_RED_ACTIVE);
1732 * Let's do the best we can to avoid issues in the future. Marking all
1735 slab_fix(s, "Marking all objects used");
1743 static inline int free_consistency_checks(struct kmem_cache *s,
1746 if (!check_valid_pointer(s, slab, object)) {
1747 slab_err(s, slab, "Invalid object pointer 0x%p", object);
1751 if (on_freelist(s, slab, object)) {
1752 object_err(s, slab, object, "Object already free");
1756 if (!check_object(s, slab, object, SLUB_RED_ACTIVE))
1759 if (unlikely(s != slab->slab_cache)) {
1764 object_err(s, slab, object,
1777 * @slabs: return start of list of slabs, or NULL when there's no list
1780 * returns the start of next block if there's any, or NULL
1929 * Debug option(s) are applied to @flags. In addition to the debug
1930 * option(s), if a slab name (or multiple) is specified i.e.
1932 * then only the select slabs will receive the debug option(s).
1955 /* Go through all blocks of debug options, see if any matches our slab's name */
1989 static inline void setup_object_debug(struct kmem_cache *s, void *object) {}
1991 void setup_slab_debug(struct kmem_cache *s, struct slab *slab, void *addr) {}
1993 static inline bool alloc_debug_processing(struct kmem_cache *s,
1996 static inline bool free_debug_processing(struct kmem_cache *s,
2000 static inline void slab_pad_check(struct kmem_cache *s, struct slab *slab) {}
2001 static inline int check_object(struct kmem_cache *s, struct slab *slab,
2004 static inline void set_track(struct kmem_cache *s, void *object,
2006 static inline void add_full(struct kmem_cache *s, struct kmem_cache_node *n,
2008 static inline void remove_full(struct kmem_cache *s, struct kmem_cache_node *n,
2020 static inline void inc_slabs_node(struct kmem_cache *s, int node,
2022 static inline void dec_slabs_node(struct kmem_cache *s, int node,
2024 static bool freelist_corrupted(struct kmem_cache *s, struct slab *slab,
2098 int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
2102 unsigned int objects = objs_per_slab(s, slab);
2213 static int alloc_slab_obj_exts(struct slab *slab, struct kmem_cache *s,
2228 prepare_slab_obj_exts_hook(struct kmem_cache *s, gfp_t flags, void *p)
2234 alloc_slab_obj_exts(slab, s, flags, false)) {
2235 pr_warn_once("%s, %s: Failed to create slab extension vector!\n",
2236 __func__, s->name);
2240 return slab_obj_exts(slab) + obj_to_index(s, slab, p);
2245 __alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
2252 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2258 obj_exts = prepare_slab_obj_exts_hook(s, flags, object);
2265 alloc_tag_add(&obj_exts->ref, current->alloc_tag, s->size);
2271 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
2274 __alloc_tagging_slab_alloc_hook(s, object, flags);
2279 __alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2286 if (s->flags & (SLAB_NO_OBJ_EXT | SLAB_NOLEAKTRACE))
2294 unsigned int off = obj_to_index(s, slab, p[i]);
2296 alloc_tag_sub(&obj_exts[off].ref, s->size);
2301 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2305 __alloc_tagging_slab_free_hook(s, slab, p, objects);
2311 alloc_tagging_slab_alloc_hook(struct kmem_cache *s, void *object, gfp_t flags)
2316 alloc_tagging_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2326 static void memcg_alloc_abort_single(struct kmem_cache *s, void *object);
2329 bool memcg_slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
2335 if (likely(!(flags & __GFP_ACCOUNT) && !(s->flags & SLAB_ACCOUNT)))
2338 if (likely(__memcg_slab_post_alloc_hook(s, lru, flags, size, p)))
2342 memcg_alloc_abort_single(s, *p);
2345 kmem_cache_free_bulk(s, size, p);
2352 void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab, void **p,
2364 __memcg_slab_free_hook(s, slab, p, objects, obj_exts);
2371 struct kmem_cache *s;
2400 s = slab->slab_cache;
2407 if (is_kmalloc_normal(s))
2413 off = obj_to_index(s, slab, p);
2418 return __memcg_slab_post_alloc_hook(s, NULL, flags, 1, &p);
2422 static inline bool memcg_slab_post_alloc_hook(struct kmem_cache *s,
2430 static inline void memcg_slab_free_hook(struct kmem_cache *s, struct slab *slab,
2459 bool slab_free_hook(struct kmem_cache *s, void *x, bool init,
2463 bool still_accessible = (s->flags & SLAB_TYPESAFE_BY_RCU) && !after_rcu_delay;
2465 kmemleak_free_recursive(x, s->flags);
2466 kmsan_slab_free(s, x);
2468 debug_check_no_locks_freed(x, s->object_size);
2470 if (!(s->flags & SLAB_DEBUG_OBJECTS))
2471 debug_check_no_obj_freed(x, s->object_size);
2475 __kcsan_check_access(x, s->object_size,
2485 if (kasan_slab_pre_free(s, x))
2512 * kasan_slab_free and initialization memset's must be
2515 * The initialization memset's clear the object and the metadata,
2518 * The object's freepointer is also avoided if stored outside the
2525 inuse = get_info_end(s);
2526 orig_size = get_orig_size(s, x);
2529 rsize = (s->flags & SLAB_RED_ZONE) ? s->red_left_pad : 0;
2531 s->size - inuse - rsize);
2536 set_orig_size(s, x, orig_size);
2540 return !kasan_slab_free(s, x, init, still_accessible, false);
2544 bool slab_free_freelist_hook(struct kmem_cache *s, void **head, void **tail,
2554 slab_free_hook(s, next, false, false);
2562 init = slab_want_init_on_free(s);
2566 next = get_freepointer(s, object);
2568 /* If object's reuse doesn't have to be delayed */
2569 if (likely(slab_free_hook(s, object, init, false))) {
2571 set_freepointer(s, object, *head);
2578 * accordingly if object's reuse is delayed.
2587 static void *setup_object(struct kmem_cache *s, void *object)
2589 setup_object_debug(s, object);
2590 object = kasan_init_slab_obj(s, object);
2591 if (unlikely(s->ctor)) {
2592 kasan_unpoison_new_object(s, object);
2593 s->ctor(object);
2594 kasan_poison_new_object(s, object);
2599 static struct slab_sheaf *alloc_empty_sheaf(struct kmem_cache *s, gfp_t gfp)
2614 if (s->flags & SLAB_KMALLOC)
2617 sheaf_size = struct_size(sheaf, objects, s->sheaf_capacity);
2623 sheaf->cache = s;
2625 stat(s, SHEAF_ALLOC);
2630 static void free_empty_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf)
2634 stat(s, SHEAF_FREE);
2637 static int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags,
2641 static int refill_sheaf(struct kmem_cache *s, struct slab_sheaf *sheaf,
2644 int to_fill = s->sheaf_capacity - sheaf->size;
2650 filled = __kmem_cache_alloc_bulk(s, gfp, to_fill,
2655 stat_add(s, SHEAF_REFILL, filled);
2664 static struct slab_sheaf *alloc_full_sheaf(struct kmem_cache *s, gfp_t gfp)
2666 struct slab_sheaf *sheaf = alloc_empty_sheaf(s, gfp);
2671 if (refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC)) {
2672 free_empty_sheaf(s, sheaf);
2685 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p);
2695 static bool sheaf_flush_main(struct kmem_cache *s)
2704 if (!local_trylock(&s->cpu_sheaves->lock))
2707 pcs = this_cpu_ptr(s->cpu_sheaves);
2717 local_unlock(&s->cpu_sheaves->lock);
2719 __kmem_cache_free_bulk(s, batch, &objects[0]);
2721 stat_add(s, SHEAF_FLUSH, batch);
2732 * Free all objects from a sheaf that's unused, i.e. not linked to any
2734 * necessary when flushing cpu's sheaves (both spare and main) during cpu
2737 static void sheaf_flush_unused(struct kmem_cache *s, struct slab_sheaf *sheaf)
2742 stat_add(s, SHEAF_FLUSH, sheaf->size);
2744 __kmem_cache_free_bulk(s, sheaf->size, &sheaf->objects[0]);
2749 static bool __rcu_free_sheaf_prepare(struct kmem_cache *s,
2752 bool init = slab_want_init_on_free(s);
2760 memcg_slab_free_hook(s, slab, p + i, 1);
2761 alloc_tagging_slab_free_hook(s, slab, p + i, 1);
2763 if (unlikely(!slab_free_hook(s, p[i], init, true))) {
2780 struct kmem_cache *s;
2783 s = sheaf->cache;
2785 __rcu_free_sheaf_prepare(s, sheaf);
2787 sheaf_flush_unused(s, sheaf);
2789 free_empty_sheaf(s, sheaf);
2794 * single cpu's sheaves
2798 * flushing operations are rare so let's keep it simple and flush to slabs
2801 static void pcs_flush_all(struct kmem_cache *s)
2806 local_lock(&s->cpu_sheaves->lock);
2807 pcs = this_cpu_ptr(s->cpu_sheaves);
2815 local_unlock(&s->cpu_sheaves->lock);
2818 sheaf_flush_unused(s, spare);
2819 free_empty_sheaf(s, spare);
2825 sheaf_flush_main(s);
2828 static void __pcs_flush_all_cpu(struct kmem_cache *s, unsigned int cpu)
2832 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
2835 sheaf_flush_unused(s, pcs->main);
2837 sheaf_flush_unused(s, pcs->spare);
2838 free_empty_sheaf(s, pcs->spare);
2848 static void pcs_destroy(struct kmem_cache *s)
2855 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
2874 free_empty_sheaf(s, pcs->main);
2879 free_percpu(s->cpu_sheaves);
2880 s->cpu_sheaves = NULL;
3002 /* we don't repeat this check under barn->lock as it's not critical */
3035 static void barn_shrink(struct kmem_cache *s, struct node_barn *barn)
3052 sheaf_flush_unused(s, sheaf);
3053 free_empty_sheaf(s, sheaf);
3057 free_empty_sheaf(s, sheaf);
3092 static int init_cache_random_seq(struct kmem_cache *s)
3094 unsigned int count = oo_objects(s->oo);
3098 if (s->random_seq)
3101 err = cache_random_seq_create(s, count, GFP_KERNEL);
3103 pr_err("SLUB: Unable to initialize free list for %s\n",
3104 s->name);
3109 if (s->random_seq) {
3113 s->random_seq[i] *= s->size;
3121 struct kmem_cache *s;
3125 list_for_each_entry(s, &slab_caches, list)
3126 init_cache_random_seq(s);
3132 static void *next_freelist_entry(struct kmem_cache *s,
3144 idx = s->random_seq[*pos];
3154 static bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
3161 if (slab->objects < 2 || !s->random_seq)
3164 freelist_count = oo_objects(s->oo);
3167 page_limit = slab->objects * s->size;
3168 start = fixup_red_left(s, slab_address(slab));
3171 cur = next_freelist_entry(s, &pos, start, page_limit, freelist_count);
3172 cur = setup_object(s, cur);
3176 next = next_freelist_entry(s, &pos, start, page_limit,
3178 next = setup_object(s, next);
3179 set_freepointer(s, cur, next);
3182 set_freepointer(s, cur, NULL);
3187 static inline int init_cache_random_seq(struct kmem_cache *s)
3192 static inline bool shuffle_freelist(struct kmem_cache *s, struct slab *slab)
3199 struct kmem_cache *s, gfp_t gfp)
3201 if (memcg_kmem_online() && (s->flags & SLAB_ACCOUNT))
3202 alloc_slab_obj_exts(slab, s, gfp, true);
3204 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
3209 struct kmem_cache *s)
3218 mod_node_page_state(slab_pgdat(slab), cache_vmstat_idx(s),
3222 static struct slab *allocate_slab(struct kmem_cache *s, gfp_t flags, int node)
3226 struct kmem_cache_order_objects oo = s->oo;
3234 flags |= s->allocflags;
3241 if ((alloc_gfp & __GFP_DIRECT_RECLAIM) && oo_order(oo) > oo_order(s->min))
3250 oo = s->min;
3259 stat(s, ORDER_FALLBACK);
3267 account_slab(slab, oo_order(oo), s, flags);
3269 slab->slab_cache = s;
3275 setup_slab_debug(s, slab, start);
3277 shuffle = shuffle_freelist(s, slab);
3280 start = fixup_red_left(s, start);
3281 start = setup_object(s, start);
3284 next = p + s->size;
3285 next = setup_object(s, next);
3286 set_freepointer(s, p, next);
3289 set_freepointer(s, p, NULL);
3295 static struct slab *new_slab(struct kmem_cache *s, gfp_t flags, int node)
3300 WARN_ON_ONCE(s->ctor && (flags & __GFP_ZERO));
3302 return allocate_slab(s,
3306 static void __free_slab(struct kmem_cache *s, struct slab *slab)
3316 unaccount_slab(slab, order, s);
3327 static void free_slab(struct kmem_cache *s, struct slab *slab)
3329 if (kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS)) {
3332 slab_pad_check(s, slab);
3333 for_each_object(p, s, slab_address(slab), slab->objects)
3334 check_object(s, slab, p, SLUB_RED_INACTIVE);
3337 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU))
3340 __free_slab(s, slab);
3343 static void discard_slab(struct kmem_cache *s, struct slab *slab)
3345 dec_slabs_node(s, slab_nid(slab), slab->objects);
3346 free_slab(s, slab);
3400 static void *alloc_single_from_partial(struct kmem_cache *s,
3408 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
3410 slab_err(s, slab, "Not a valid slab page");
3417 slab->freelist = get_freepointer(s, object);
3420 if (!alloc_debug_processing(s, slab, object, orig_size)) {
3427 add_full(s, n, slab);
3440 static void *alloc_single_from_new_slab(struct kmem_cache *s, struct slab *slab,
3445 struct kmem_cache_node *n = get_node(s, nid);
3456 slab->freelist = get_freepointer(s, object);
3459 if (!alloc_debug_processing(s, slab, object, orig_size)) {
3461 * It's not really expected that this would fail on a
3475 add_full(s, n, slab);
3479 inc_slabs_node(s, nid, slab->objects);
3486 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain);
3488 static inline void put_cpu_partial(struct kmem_cache *s, struct slab *slab,
3496 static struct slab *get_partial_node(struct kmem_cache *s,
3521 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
3522 void *object = alloc_single_from_partial(s, n, slab,
3536 stat(s, ALLOC_FROM_PARTIAL);
3538 if ((slub_get_cpu_partial(s) == 0)) {
3542 put_cpu_partial(s, slab, 0);
3543 stat(s, CPU_PARTIAL_NODE);
3545 if (++partial_slabs > slub_get_cpu_partial(s) / 2) {
3557 static struct slab *get_any_partial(struct kmem_cache *s,
3586 if (!s->remote_node_defrag_ratio ||
3587 get_cycles() % 1024 > s->remote_node_defrag_ratio)
3596 n = get_node(s, zone_to_nid(zone));
3599 n->nr_partial > s->min_partial) {
3600 slab = get_partial_node(s, n, pc);
3621 static struct slab *get_partial(struct kmem_cache *s, int node,
3630 slab = get_partial_node(s, get_node(s, searchnode), pc);
3634 return get_any_partial(s, pc);
3675 const struct kmem_cache *s, unsigned long tid)
3678 unsigned long actual_tid = __this_cpu_read(s->cpu_slab->tid);
3680 pr_info("%s %s: cmpxchg redo ", n, s->name);
3694 stat(s, CMPXCHG_DOUBLE_CPU_FAIL);
3697 static void init_kmem_cache_cpus(struct kmem_cache *s)
3704 bool finegrain_lockdep = !init_section_contains(s, 1);
3716 lockdep_register_key(&s->lock_key);
3718 c = per_cpu_ptr(s->cpu_slab, cpu);
3721 lockdep_set_class(&c->lock, &s->lock_key);
3727 * Finishes removing the cpu slab. Merges cpu's freelist with slab's freelist,
3732 static void deactivate_slab(struct kmem_cache *s, struct slab *slab,
3735 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
3743 stat(s, DEACTIVATE_REMOTE_FREES);
3748 * Stage one: Count the objects on cpu's freelist as free_delta and
3754 nextfree = get_freepointer(s, freelist_iter);
3761 if (freelist_corrupted(s, slab, &freelist_iter, nextfree))
3772 * freelist to the head of slab's freelist.
3784 set_freepointer(s, freelist_tail, old.freelist);
3789 } while (!slab_update_freelist(s, slab, &old, &new, "unfreezing slab"));
3794 if (!new.inuse && n->nr_partial >= s->min_partial) {
3795 stat(s, DEACTIVATE_EMPTY);
3796 discard_slab(s, slab);
3797 stat(s, FREE_SLAB);
3802 stat(s, tail);
3804 stat(s, DEACTIVATE_FULL);
3809 * ___slab_alloc()'s caller is supposed to check if kmem_cache::kmem_cache_cpu::lock
3836 #define local_lock_cpu_slab(s, flags) \
3837 local_lock_irqsave(&(s)->cpu_slab->lock, flags)
3839 #define local_lock_cpu_slab(s, flags) \
3841 bool __l = local_trylock_irqsave(&(s)->cpu_slab->lock, flags); \
3846 #define local_unlock_cpu_slab(s, flags) \
3847 local_unlock_irqrestore(&(s)->cpu_slab->lock, flags)
3850 static void __put_partials(struct kmem_cache *s, struct slab *partial_slab)
3860 n2 = get_node(s, slab_nid(slab));
3869 if (unlikely(!slab->inuse && n->nr_partial >= s->min_partial)) {
3874 stat(s, FREE_ADD_PARTIAL);
3885 stat(s, DEACTIVATE_EMPTY);
3886 discard_slab(s, slab);
3887 stat(s, FREE_SLAB);
3894 static void put_partials(struct kmem_cache *s)
3899 local_lock_irqsave(&s->cpu_slab->lock, flags);
3900 partial_slab = this_cpu_read(s->cpu_slab->partial);
3901 this_cpu_write(s->cpu_slab->partial, NULL);
3902 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3905 __put_partials(s, partial_slab);
3908 static void put_partials_cpu(struct kmem_cache *s,
3917 __put_partials(s, partial_slab);
3926 static void put_cpu_partial(struct kmem_cache *s, struct slab *slab, int drain)
3933 local_lock_cpu_slab(s, flags);
3935 oldslab = this_cpu_read(s->cpu_slab->partial);
3938 if (drain && oldslab->slabs >= s->cpu_partial_slabs) {
3956 this_cpu_write(s->cpu_slab->partial, slab);
3958 local_unlock_cpu_slab(s, flags);
3961 __put_partials(s, slab_to_put);
3962 stat(s, CPU_PARTIAL_DRAIN);
3968 static inline void put_partials(struct kmem_cache *s) { }
3969 static inline void put_partials_cpu(struct kmem_cache *s,
3974 static inline void flush_slab(struct kmem_cache *s, struct kmem_cache_cpu *c)
3980 local_lock_irqsave(&s->cpu_slab->lock, flags);
3989 local_unlock_irqrestore(&s->cpu_slab->lock, flags);
3992 deactivate_slab(s, slab, freelist);
3993 stat(s, CPUSLAB_FLUSH);
3997 static inline void __flush_cpu_slab(struct kmem_cache *s, int cpu)
3999 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4008 deactivate_slab(s, slab, freelist);
4009 stat(s, CPUSLAB_FLUSH);
4012 put_partials_cpu(s, c);
4015 static inline void flush_this_cpu_slab(struct kmem_cache *s)
4017 struct kmem_cache_cpu *c = this_cpu_ptr(s->cpu_slab);
4020 flush_slab(s, c);
4022 put_partials(s);
4025 static bool has_cpu_slab(int cpu, struct kmem_cache *s)
4027 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab, cpu);
4032 static bool has_pcs_used(int cpu, struct kmem_cache *s)
4036 if (!s->cpu_sheaves)
4039 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
4051 struct kmem_cache *s;
4056 s = sfw->s;
4058 if (s->cpu_sheaves)
4059 pcs_flush_all(s);
4061 flush_this_cpu_slab(s);
4064 static void flush_all_cpus_locked(struct kmem_cache *s)
4074 if (!has_cpu_slab(cpu, s) && !has_pcs_used(cpu, s)) {
4080 sfw->s = s;
4094 static void flush_all(struct kmem_cache *s)
4097 flush_all_cpus_locked(s);
4106 struct kmem_cache *s;
4109 s = sfw->s;
4111 local_lock(&s->cpu_sheaves->lock);
4112 pcs = this_cpu_ptr(s->cpu_sheaves);
4117 local_unlock(&s->cpu_sheaves->lock);
4125 void flush_rcu_sheaves_on_cache(struct kmem_cache *s)
4143 sfw->s = s;
4157 struct kmem_cache *s;
4162 list_for_each_entry(s, &slab_caches, list) {
4163 if (!s->cpu_sheaves)
4165 flush_rcu_sheaves_on_cache(s);
4180 struct kmem_cache *s;
4183 list_for_each_entry(s, &slab_caches, list) {
4184 __flush_cpu_slab(s, cpu);
4185 if (s->cpu_sheaves)
4186 __pcs_flush_all_cpu(s, cpu);
4217 static inline bool free_debug_processing(struct kmem_cache *s,
4225 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
4226 if (!check_slab(s, slab))
4231 slab_err(s, slab, "Slab has %d allocated objects but %d are to be freed\n",
4241 if (s->flags & SLAB_CONSISTENCY_CHECKS) {
4242 if (!free_consistency_checks(s, slab, object, addr))
4246 if (s->flags & SLAB_STORE_USER)
4247 set_track_update(s, object, TRACK_FREE, addr, handle);
4248 trace(s, slab, object, 0);
4250 init_object(s, object, SLUB_RED_INACTIVE);
4254 object = get_freepointer(s, object);
4261 slab_err(s, slab, "Bulk free expected %d objects but found %d\n",
4269 slab_fix(s, "Object at 0x%p not freed", object);
4308 * Scan from both the list's head and tail for better accuracy.
4330 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid)
4343 pr_warn(" cache: %s, object size: %u, buffer size: %u, default order: %u, min order: %u\n",
4344 s->name, s->object_size, s->size, oo_order(s->oo),
4345 oo_order(s->min));
4347 if (oo_order(s->min) > get_order(s->object_size))
4348 pr_warn(" %s debugging increased min order, use slab_debug=O to disable.\n",
4349 s->name);
4351 for_each_kmem_cache_node(s, node, n) {
4366 slab_out_of_memory(struct kmem_cache *s, gfp_t gfpflags, int nid) { }
4378 __update_cpu_freelist_fast(struct kmem_cache *s,
4385 return this_cpu_try_cmpxchg_freelist(s->cpu_slab->freelist_tid,
4397 static inline void *get_freelist(struct kmem_cache *s, struct slab *slab)
4401 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
4414 } while (!__slab_update_freelist(s, slab, &old, &new, "get_freelist"));
4422 static inline void *freeze_slab(struct kmem_cache *s, struct slab *slab)
4437 } while (!slab_update_freelist(s, slab, &old, &new, "freeze_slab"));
4461 static void *___slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
4471 stat(s, ALLOC_SLOWPATH);
4507 stat(s, ALLOC_NODE_MISMATCH);
4521 local_lock_cpu_slab(s, flags);
4524 local_unlock_cpu_slab(s, flags);
4531 freelist = get_freelist(s, slab);
4536 local_unlock_cpu_slab(s, flags);
4537 stat(s, DEACTIVATE_BYPASS);
4541 stat(s, ALLOC_REFILL);
4545 lockdep_assert_held(this_cpu_ptr(&s->cpu_slab->lock));
4553 c->freelist = get_freepointer(s, freelist);
4555 local_unlock_cpu_slab(s, flags);
4560 local_lock_cpu_slab(s, flags);
4562 local_unlock_cpu_slab(s, flags);
4569 local_unlock_cpu_slab(s, flags);
4570 deactivate_slab(s, slab, freelist);
4576 local_lock_cpu_slab(s, flags);
4578 local_unlock_cpu_slab(s, flags);
4582 local_unlock_cpu_slab(s, flags);
4594 freelist = get_freelist(s, slab);
4596 stat(s, CPU_PARTIAL_ALLOC);
4600 local_unlock_cpu_slab(s, flags);
4603 __put_partials(s, slab);
4631 slab = get_partial(s, node, &pc);
4633 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4644 if (s->flags & SLAB_STORE_USER)
4645 set_track(s, freelist, TRACK_ALLOC, addr,
4651 freelist = freeze_slab(s, slab);
4655 slub_put_cpu_ptr(s->cpu_slab);
4656 slab = new_slab(s, pc.flags, node);
4657 c = slub_get_cpu_ptr(s->cpu_slab);
4665 slab_out_of_memory(s, gfpflags, node);
4669 stat(s, ALLOC_SLAB);
4671 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
4672 freelist = alloc_single_from_new_slab(s, slab, orig_size, gfpflags);
4681 if (s->flags & SLAB_STORE_USER)
4682 set_track(s, freelist, TRACK_ALLOC, addr,
4697 inc_slabs_node(s, slab_nid(slab), slab->objects);
4704 deactivate_slab(s, slab, get_freepointer(s, freelist));
4710 local_lock_cpu_slab(s, flags);
4719 local_unlock_cpu_slab(s, flags);
4725 deactivate_slab(s, flush_slab, flush_freelist);
4728 stat(s, CPUSLAB_FLUSH);
4755 static void *__slab_alloc(struct kmem_cache *s, gfp_t gfpflags, int node,
4766 c = slub_get_cpu_ptr(s->cpu_slab);
4769 if (local_lock_is_locked(&s->cpu_slab->lock)) {
4772 * retry a different bucket. It's not propagated
4779 p = ___slab_alloc(s, gfpflags, node, addr, c, orig_size);
4782 slub_put_cpu_ptr(s->cpu_slab);
4787 static __always_inline void *__slab_alloc_node(struct kmem_cache *s,
4805 * two reads, it's OK as the two are still associated with the same cpu
4808 c = raw_cpu_ptr(s->cpu_slab);
4813 * of fetching cpu_slab's data. tid should be fetched before anything
4855 object = __slab_alloc(s, gfpflags, node, addr, c, orig_size);
4857 void *next_object = get_freepointer_safe(s, object);
4873 if (unlikely(!__update_cpu_freelist_fast(s, object, next_object, tid))) {
4874 note_cmpxchg_failure("slab_alloc", s, tid);
4877 prefetch_freepointer(s, next_object);
4878 stat(s, ALLOC_FASTPATH);
4885 * If the object has been wiped upon free, make sure it's fully initialized by
4890 static __always_inline void maybe_wipe_obj_freeptr(struct kmem_cache *s,
4893 if (unlikely(slab_want_init_on_free(s)) && obj &&
4894 !freeptr_outside_object(s))
4895 memset((void *)((char *)kasan_reset_tag(obj) + s->offset),
4900 struct kmem_cache *slab_pre_alloc_hook(struct kmem_cache *s, gfp_t flags)
4906 if (unlikely(should_failslab(s, flags)))
4909 return s;
4913 bool slab_post_alloc_hook(struct kmem_cache *s, struct list_lru *lru,
4917 unsigned int zero_size = s->object_size;
4930 if (kmem_cache_debug_flags(s, SLAB_STORE_USER | SLAB_RED_ZONE) &&
4931 (s->flags & SLAB_KMALLOC))
4953 p[i] = kasan_slab_alloc(s, p[i], init_flags, kasan_init);
4958 kmemleak_alloc_recursive(p[i], s->object_size, 1,
4959 s->flags, init_flags);
4960 kmsan_slab_alloc(s, p[i], init_flags);
4961 alloc_tagging_slab_alloc_hook(s, p[i], flags);
4964 return memcg_slab_post_alloc_hook(s, lru, flags, size, p);
4976 __pcs_replace_empty_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs, gfp_t gfp)
4983 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
4990 barn = get_barn(s);
4992 local_unlock(&s->cpu_sheaves->lock);
4999 stat(s, BARN_GET);
5004 stat(s, BARN_GET_FAIL);
5017 local_unlock(&s->cpu_sheaves->lock);
5023 if (!refill_sheaf(s, empty, gfp | __GFP_NOMEMALLOC)) {
5030 free_empty_sheaf(s, empty);
5033 full = alloc_full_sheaf(s, gfp);
5043 local_lock(&s->cpu_sheaves->lock);
5044 pcs = this_cpu_ptr(s->cpu_sheaves);
5049 * sheaf, it's due to racing or being migrated to a different
5050 * cpu. Breaching the barn's sheaf limits should be thus rare
5072 stat(s, BARN_PUT);
5078 void *alloc_from_pcs(struct kmem_cache *s, gfp_t gfp, int node)
5109 * We assume the percpu sheaves contain only local objects although it's
5115 if (!local_trylock(&s->cpu_sheaves->lock))
5118 pcs = this_cpu_ptr(s->cpu_sheaves);
5121 pcs = __pcs_replace_empty_main(s, pcs, gfp);
5135 local_unlock(&s->cpu_sheaves->lock);
5142 local_unlock(&s->cpu_sheaves->lock);
5144 stat(s, ALLOC_PCS);
5150 unsigned int alloc_from_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
5158 if (!local_trylock(&s->cpu_sheaves->lock))
5161 pcs = this_cpu_ptr(s->cpu_sheaves);
5173 barn = get_barn(s);
5175 local_unlock(&s->cpu_sheaves->lock);
5182 stat(s, BARN_GET);
5187 stat(s, BARN_GET_FAIL);
5189 local_unlock(&s->cpu_sheaves->lock);
5207 local_unlock(&s->cpu_sheaves->lock);
5209 stat_add(s, ALLOC_PCS, batch);
5233 static __fastpath_inline void *slab_alloc_node(struct kmem_cache *s, struct list_lru *lru,
5239 s = slab_pre_alloc_hook(s, gfpflags);
5240 if (unlikely(!s))
5243 object = kfence_alloc(s, orig_size, gfpflags);
5247 if (s->cpu_sheaves)
5248 object = alloc_from_pcs(s, gfpflags, node);
5251 object = __slab_alloc_node(s, gfpflags, node, addr, orig_size);
5253 maybe_wipe_obj_freeptr(s, object);
5254 init = slab_want_init_on_alloc(gfpflags, s);
5259 * @orig_size bytes might be zeroed instead of s->object_size
5263 slab_post_alloc_hook(s, lru, gfpflags, 1, &object, init, orig_size);
5268 void *kmem_cache_alloc_noprof(struct kmem_cache *s, gfp_t gfpflags)
5270 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE, _RET_IP_,
5271 s->object_size);
5273 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
5279 void *kmem_cache_alloc_lru_noprof(struct kmem_cache *s, struct list_lru *lru,
5282 void *ret = slab_alloc_node(s, lru, gfpflags, NUMA_NO_NODE, _RET_IP_,
5283 s->object_size);
5285 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, NUMA_NO_NODE);
5302 * @s: The cache to allocate from.
5313 void *kmem_cache_alloc_node_noprof(struct kmem_cache *s, gfp_t gfpflags, int node)
5315 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, s->object_size);
5317 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfpflags, node);
5323 static int __prefill_sheaf_pfmemalloc(struct kmem_cache *s,
5328 ret = refill_sheaf(s, sheaf, gfp | __GFP_NOMEMALLOC);
5335 * it for when it's returned
5337 ret = refill_sheaf(s, sheaf, gfp);
5350 kmem_cache_prefill_sheaf(struct kmem_cache *s, gfp_t gfp, unsigned int size)
5356 if (unlikely(size > s->sheaf_capacity)) {
5365 WARN_ON_ONCE(s->sheaf_capacity == 0 &&
5367 !(s->flags & SLAB_DEBUG_FLAGS));
5373 stat(s, SHEAF_PREFILL_OVERSIZE);
5374 sheaf->cache = s;
5381 if (!__kmem_cache_alloc_bulk(s, gfp, size,
5392 local_lock(&s->cpu_sheaves->lock);
5393 pcs = this_cpu_ptr(s->cpu_sheaves);
5398 stat(s, SHEAF_PREFILL_FAST);
5400 barn = get_barn(s);
5402 stat(s, SHEAF_PREFILL_SLOW);
5406 stat(s, BARN_GET);
5408 stat(s, BARN_GET_FAIL);
5411 local_unlock(&s->cpu_sheaves->lock);
5415 sheaf = alloc_empty_sheaf(s, gfp);
5418 sheaf->capacity = s->sheaf_capacity;
5422 __prefill_sheaf_pfmemalloc(s, sheaf, gfp)) {
5423 sheaf_flush_unused(s, sheaf);
5424 free_empty_sheaf(s, sheaf);
5435 * If the sheaf cannot simply become the percpu spare sheaf, but there's space
5436 * for a full sheaf in the barn, we try to refill the sheaf back to the cache's
5442 void kmem_cache_return_sheaf(struct kmem_cache *s, gfp_t gfp,
5448 if (unlikely((sheaf->capacity != s->sheaf_capacity)
5450 sheaf_flush_unused(s, sheaf);
5455 local_lock(&s->cpu_sheaves->lock);
5456 pcs = this_cpu_ptr(s->cpu_sheaves);
5457 barn = get_barn(s);
5462 stat(s, SHEAF_RETURN_FAST);
5465 local_unlock(&s->cpu_sheaves->lock);
5470 stat(s, SHEAF_RETURN_SLOW);
5477 refill_sheaf(s, sheaf, gfp)) {
5478 sheaf_flush_unused(s, sheaf);
5479 free_empty_sheaf(s, sheaf);
5484 stat(s, BARN_PUT);
5492 * s->sheaf_capacity objects if such replacement is necessary, but the refill
5495 * In practice we always refill to full sheaf's capacity.
5497 int kmem_cache_refill_sheaf(struct kmem_cache *s, gfp_t gfp,
5514 if (likely(sheaf->capacity == s->sheaf_capacity))
5515 return __prefill_sheaf_pfmemalloc(s, sheaf, gfp);
5517 if (!__kmem_cache_alloc_bulk(s, gfp, sheaf->capacity - sheaf->size,
5529 * This should be a very rare path so let's not complicate it.
5531 sheaf = kmem_cache_prefill_sheaf(s, gfp, size);
5535 kmem_cache_return_sheaf(s, gfp, *sheafp);
5553 kmem_cache_alloc_from_sheaf_noprof(struct kmem_cache *s, gfp_t gfp,
5562 ret = kfence_alloc(s, s->object_size, gfp);
5567 init = slab_want_init_on_alloc(gfp, s);
5570 slab_post_alloc_hook(s, NULL, gfp | __GFP_NOFAIL, 1, &ret, init, s->object_size);
5572 trace_kmem_cache_alloc(_RET_IP_, ret, s, gfp, NUMA_NO_NODE);
5641 struct kmem_cache *s;
5654 s = kmalloc_slab(size, b, flags, caller);
5656 ret = slab_alloc_node(s, NULL, flags, node, caller, size);
5657 ret = kasan_kmalloc(s, ret, size, flags);
5658 trace_kmalloc(caller, ret, size, s->size, flags, node);
5687 struct kmem_cache *s;
5703 s = kmalloc_slab(size, NULL, alloc_gfp, _RET_IP_);
5705 if (!(s->flags & __CMPXCHG_DOUBLE) && !kmem_cache_debug(s))
5728 * If in_nmi() && local_lock_is_locked(s->cpu_slab) then it means that
5730 * it did local_lock_irqsave(&s->cpu_slab->lock, flags).
5733 if (!in_nmi() || !local_lock_is_locked(&s->cpu_slab->lock))
5734 ret = __slab_alloc_node(s, alloc_gfp, node, _RET_IP_, size);
5739 size = s->object_size + 1;
5752 maybe_wipe_obj_freeptr(s, ret);
5753 slab_post_alloc_hook(s, NULL, alloc_gfp, 1, &ret,
5754 slab_want_init_on_alloc(alloc_gfp, s), size);
5756 ret = kasan_kmalloc(s, ret, size, alloc_gfp);
5769 void *__kmalloc_cache_noprof(struct kmem_cache *s, gfp_t gfpflags, size_t size)
5771 void *ret = slab_alloc_node(s, NULL, gfpflags, NUMA_NO_NODE,
5774 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, NUMA_NO_NODE);
5776 ret = kasan_kmalloc(s, ret, size, gfpflags);
5781 void *__kmalloc_cache_node_noprof(struct kmem_cache *s, gfp_t gfpflags,
5784 void *ret = slab_alloc_node(s, NULL, gfpflags, node, _RET_IP_, size);
5786 trace_kmalloc(_RET_IP_, ret, size, s->size, gfpflags, node);
5788 ret = kasan_kmalloc(s, ret, size, gfpflags);
5794 struct kmem_cache *s, struct slab *slab,
5798 struct kmem_cache_node *n = get_node(s, slab_nid(slab));
5808 if (s->flags & SLAB_STORE_USER)
5813 if (free_debug_processing(s, slab, head, tail, &cnt, addr, handle)) {
5818 set_freepointer(s, tail, prior);
5822 * If the slab is empty, and node's partial list is full,
5823 * it should be discarded anyway no matter it's on full or
5826 if (slab->inuse == 0 && n->nr_partial >= s->min_partial)
5831 remove_full(s, n, slab);
5834 stat(s, FREE_ADD_PARTIAL);
5838 stat(s, FREE_REMOVE_PARTIAL);
5847 dec_slabs_node(s, slab_nid(slab_free), slab_free->objects);
5853 stat(s, FREE_SLAB);
5854 free_slab(s, slab_free);
5866 static void __slab_free(struct kmem_cache *s, struct slab *slab,
5877 stat(s, FREE_SLOWPATH);
5879 if (IS_ENABLED(CONFIG_SLUB_TINY) || kmem_cache_debug(s)) {
5880 free_to_partial_list(s, slab, head, tail, cnt, addr);
5886 * instead of kmem_cache_has_cpu_partial(s), because kmem_cache_debug(s)
5903 set_freepointer(s, tail, old.freelist);
5912 * Unless it's frozen.
5922 n = get_node(s, slab_nid(slab));
5937 } while (!slab_update_freelist(s, slab, &old, &new, "__slab_free"));
5946 stat(s, FREE_FROZEN);
5952 put_cpu_partial(s, slab, 1);
5953 stat(s, CPU_PARTIAL_FREE);
5977 if (unlikely(!new.inuse && n->nr_partial >= s->min_partial))
5987 stat(s, FREE_ADD_PARTIAL);
5999 stat(s, FREE_REMOVE_PARTIAL);
6003 stat(s, FREE_SLAB);
6004 discard_slab(s, slab);
6015 * If we put any sheaf to barn here, it's because we were interrupted or have
6017 * the barn's limits to simplify the handling.
6020 * barn_replace_full_sheaf(), because there's no empty sheaf available in the
6025 static void __pcs_install_empty_sheaf(struct kmem_cache *s,
6029 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
6042 if (pcs->main->size < s->sheaf_capacity) {
6048 if (pcs->spare->size < s->sheaf_capacity) {
6059 stat(s, BARN_PUT);
6072 __pcs_replace_full_main(struct kmem_cache *s, struct slub_percpu_sheaves *pcs)
6079 lockdep_assert_held(this_cpu_ptr(&s->cpu_sheaves->lock));
6081 barn = get_barn(s);
6083 local_unlock(&s->cpu_sheaves->lock);
6099 if (pcs->spare->size < s->sheaf_capacity) {
6107 stat(s, BARN_PUT);
6116 stat(s, BARN_PUT_FAIL);
6119 local_unlock(&s->cpu_sheaves->lock);
6121 sheaf_flush_unused(s, to_flush);
6135 local_unlock(&s->cpu_sheaves->lock);
6137 empty = alloc_empty_sheaf(s, GFP_NOWAIT);
6142 stat(s, BARN_PUT_FAIL);
6144 if (!sheaf_flush_main(s))
6147 if (!local_trylock(&s->cpu_sheaves->lock))
6150 pcs = this_cpu_ptr(s->cpu_sheaves);
6157 if (pcs->main->size == s->sheaf_capacity)
6163 if (!local_trylock(&s->cpu_sheaves->lock)) {
6168 pcs = this_cpu_ptr(s->cpu_sheaves);
6169 __pcs_install_empty_sheaf(s, pcs, empty, barn);
6179 bool free_to_pcs(struct kmem_cache *s, void *object)
6183 if (!local_trylock(&s->cpu_sheaves->lock))
6186 pcs = this_cpu_ptr(s->cpu_sheaves);
6188 if (unlikely(pcs->main->size == s->sheaf_capacity)) {
6190 pcs = __pcs_replace_full_main(s, pcs);
6197 local_unlock(&s->cpu_sheaves->lock);
6199 stat(s, FREE_PCS);
6209 struct kmem_cache *s;
6213 s = sheaf->cache;
6217 * so that the sheaf might no longer be completely full. But it's easier
6226 if (__rcu_free_sheaf_prepare(s, sheaf))
6229 n = get_node(s, sheaf->node);
6246 stat(s, BARN_PUT);
6252 stat(s, BARN_PUT_FAIL);
6253 sheaf_flush_unused(s, sheaf);
6261 free_empty_sheaf(s, sheaf);
6264 bool __kfree_rcu_sheaf(struct kmem_cache *s, void *obj)
6269 if (!local_trylock(&s->cpu_sheaves->lock))
6272 pcs = this_cpu_ptr(s->cpu_sheaves);
6285 barn = get_barn(s);
6287 local_unlock(&s->cpu_sheaves->lock);
6298 local_unlock(&s->cpu_sheaves->lock);
6300 empty = alloc_empty_sheaf(s, GFP_NOWAIT);
6305 if (!local_trylock(&s->cpu_sheaves->lock)) {
6310 pcs = this_cpu_ptr(s->cpu_sheaves);
6328 if (likely(rcu_sheaf->size < s->sheaf_capacity)) {
6342 local_unlock(&s->cpu_sheaves->lock);
6344 stat(s, FREE_RCU_SHEAF);
6348 stat(s, FREE_RCU_SHEAF_FAIL);
6357 static void free_to_pcs_bulk(struct kmem_cache *s, size_t size, void **p)
6361 bool init = slab_want_init_on_free(s);
6372 memcg_slab_free_hook(s, slab, p + i, 1);
6373 alloc_tagging_slab_free_hook(s, slab, p + i, 1);
6375 if (unlikely(!slab_free_hook(s, p[i], init, false))) {
6396 if (!local_trylock(&s->cpu_sheaves->lock))
6399 pcs = this_cpu_ptr(s->cpu_sheaves);
6401 if (likely(pcs->main->size < s->sheaf_capacity))
6404 barn = get_barn(s);
6418 if (pcs->spare->size < s->sheaf_capacity) {
6425 stat(s, BARN_PUT_FAIL);
6429 stat(s, BARN_PUT);
6434 batch = min(size, s->sheaf_capacity - main->size);
6439 local_unlock(&s->cpu_sheaves->lock);
6441 stat_add(s, FREE_PCS, batch);
6455 local_unlock(&s->cpu_sheaves->lock);
6462 __kmem_cache_free_bulk(s, size, p);
6466 __kmem_cache_free_bulk(s, remote_nr, &remote_objects[0]);
6489 * In PREEMPT_RT irq_work runs in per-cpu kthread, so it's safe
6505 struct kmem_cache *s;
6510 s = slab->slab_cache;
6513 x -= s->offset;
6520 set_freepointer(s, x, NULL);
6522 __slab_free(s, slab, x, x, 1, _THIS_IP_);
6536 static void defer_free(struct kmem_cache *s, void *head)
6545 if (llist_add(head + s->offset, &df->objects))
6585 static __always_inline void do_slab_free(struct kmem_cache *s,
6589 /* cnt == 0 signals that it's called from kfree_nolock() */
6602 c = raw_cpu_ptr(s->cpu_slab);
6616 defer_free(s, head);
6618 __slab_free(s, slab, head, tail, cnt, addr);
6625 local_lock_is_locked(&s->cpu_slab->lock)) {
6626 defer_free(s, head);
6635 set_freepointer(s, tail, freelist);
6637 if (unlikely(!__update_cpu_freelist_fast(s, freelist, head, tid))) {
6638 note_cmpxchg_failure("slab_free", s, tid);
6645 local_lock_cpu_slab(s, flags);
6646 c = this_cpu_ptr(s->cpu_slab);
6648 local_unlock_cpu_slab(s, flags);
6654 set_freepointer(s, tail, freelist);
6658 local_unlock_cpu_slab(s, flags);
6660 stat_add(s, FREE_FASTPATH, cnt);
6664 void slab_free(struct kmem_cache *s, struct slab *slab, void *object,
6667 memcg_slab_free_hook(s, slab, &object, 1);
6668 alloc_tagging_slab_free_hook(s, slab, &object, 1);
6670 if (unlikely(!slab_free_hook(s, object, slab_want_init_on_free(s), false)))
6673 if (s->cpu_sheaves && likely(!IS_ENABLED(CONFIG_NUMA) ||
6676 if (likely(free_to_pcs(s, object)))
6680 do_slab_free(s, slab, object, object, 1, addr);
6686 void memcg_alloc_abort_single(struct kmem_cache *s, void *object)
6688 if (likely(slab_free_hook(s, object, slab_want_init_on_free(s), false)))
6689 do_slab_free(s, virt_to_slab(object), object, object, 1, _RET_IP_);
6694 void slab_free_bulk(struct kmem_cache *s, struct slab *slab, void *head,
6697 memcg_slab_free_hook(s, slab, p, cnt);
6698 alloc_tagging_slab_free_hook(s, slab, p, cnt);
6703 if (likely(slab_free_freelist_hook(s, &head, &tail, &cnt)))
6704 do_slab_free(s, slab, head, tail, cnt, addr);
6714 struct kmem_cache *s;
6724 s = slab->slab_cache;
6725 if (WARN_ON(!(s->flags & SLAB_TYPESAFE_BY_RCU)))
6729 if (slab_free_hook(s, object, slab_want_init_on_free(s), true))
6730 do_slab_free(s, slab, object, object, 1, _THIS_IP_);
6746 if (WARN_ONCE(!slab, "%s: Object is not a Slab page!\n", __func__))
6751 static inline struct kmem_cache *cache_from_obj(struct kmem_cache *s, void *x)
6756 !kmem_cache_debug_flags(s, SLAB_CONSISTENCY_CHECKS))
6757 return s;
6760 if (WARN(cachep && cachep != s,
6761 "%s: Wrong slab cache. %s but object is from %s\n",
6762 __func__, s->name, cachep->name))
6769 * @s: The cache the allocation was from.
6775 void kmem_cache_free(struct kmem_cache *s, void *x)
6777 s = cache_from_obj(s, x);
6778 if (!s)
6780 trace_kmem_cache_free(_RET_IP_, x, s);
6781 slab_free(s, virt_to_slab(x), x, _RET_IP_);
6816 struct kmem_cache *s;
6837 s = slab->slab_cache;
6843 unsigned int idx = __obj_to_index(s, slab_addr, obj);
6845 obj = slab_addr + s->size * idx;
6846 obj = fixup_red_left(s, obj);
6849 slab_free(s, slab, obj, _RET_IP_);
6862 struct kmem_cache *s;
6877 s = slab->slab_cache;
6878 slab_free(s, slab, x, _RET_IP_);
6894 struct kmem_cache *s;
6906 s = slab->slab_cache;
6908 memcg_slab_free_hook(s, slab, &x, 1);
6909 alloc_tagging_slab_free_hook(s, slab, &x, 1);
6912 * kmemleak_free_recursive(x, s->flags);
6913 * debug_check_no_locks_freed(x, s->object_size);
6914 * debug_check_no_obj_freed(x, s->object_size);
6915 * __kcsan_check_access(x, s->object_size, ..);
6919 kmsan_slab_free(s, x);
6929 if (kasan_slab_pre_free(s, x))
6936 kasan_slab_free(s, x, false, false, /* skip quarantine */true);
6937 do_slab_free(s, slab, x, x, 0, _RET_IP_);
6947 struct kmem_cache *s = NULL;
6978 s = slab->slab_cache;
6979 orig_size = get_orig_size(s, (void *)p);
6980 ks = s->object_size;
7003 if (s && slub_debug_orig_size(s)) {
7004 set_orig_size(s, (void *)p, new_size);
7005 if (s->flags & SLAB_RED_ZONE && new_size < ks)
7016 /* Disable KASAN checks as the object's redzone is accessed. */
7268 struct kmem_cache *s;
7279 * transferred to the real freelist(s), but only requiring a single
7284 int build_detached_freelist(struct kmem_cache *s, size_t size,
7296 if (!s) {
7305 df->s = slab->slab_cache;
7308 df->s = cache_from_obj(s, object); /* Support for memcg */
7319 set_freepointer(df->s, object, NULL);
7327 set_freepointer(df->s, object, df->freelist);
7348 static void __kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
7356 size = build_detached_freelist(s, size, p, &df);
7363 do_slab_free(df.s, df.slab, df.freelist, df.tail, df.cnt,
7369 void kmem_cache_free_bulk(struct kmem_cache *s, size_t size, void **p)
7378 if (s && s->cpu_sheaves) {
7379 free_to_pcs_bulk(s, size, p);
7386 size = build_detached_freelist(s, size, p, &df);
7390 slab_free_bulk(df.s, df.slab, df.freelist, df.tail, &p[size],
7397 int __kmem_cache_alloc_bulk(struct kmem_cache *s, gfp_t flags, size_t size,
7409 c = slub_get_cpu_ptr(s->cpu_slab);
7410 local_lock_irqsave(&s->cpu_slab->lock, irqflags);
7425 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
7431 p[i] = ___slab_alloc(s, flags, NUMA_NO_NODE,
7432 _RET_IP_, c, s->object_size);
7436 c = this_cpu_ptr(s->cpu_slab);
7437 maybe_wipe_obj_freeptr(s, p[i]);
7439 local_lock_irqsave(&s->cpu_slab->lock, irqflags);
7443 c->freelist = get_freepointer(s, object);
7445 maybe_wipe_obj_freeptr(s, p[i]);
7446 stat(s, ALLOC_FASTPATH);
7449 local_unlock_irqrestore(&s->cpu_slab->lock, irqflags);
7450 slub_put_cpu_ptr(s->cpu_slab);
7455 slub_put_cpu_ptr(s->cpu_slab);
7456 __kmem_cache_free_bulk(s, i, p);
7462 int kmem_cache_alloc_bulk_noprof(struct kmem_cache *s, gfp_t flags, size_t size,
7471 s = slab_pre_alloc_hook(s, flags);
7472 if (unlikely(!s))
7479 kfence_obj = kfence_alloc(s, s->object_size, flags);
7489 if (s->cpu_sheaves)
7490 i = alloc_from_pcs_bulk(s, size, p);
7497 if (unlikely(__kmem_cache_alloc_bulk(s, flags, size - i, p + i) == 0)) {
7499 __kmem_cache_free_bulk(s, i, p);
7521 if (unlikely(!slab_post_alloc_hook(s, NULL, flags, size, p,
7522 slab_want_init_on_alloc(flags, s), s->object_size))) {
7610 * onlining them, so don't trust the number if it's just 1. But
7678 static inline int alloc_kmem_cache_cpus(struct kmem_cache *s)
7688 s->cpu_slab = __alloc_percpu(sizeof(struct kmem_cache_cpu),
7691 if (!s->cpu_slab)
7694 init_kmem_cache_cpus(s);
7699 static int init_percpu_sheaves(struct kmem_cache *s)
7706 pcs = per_cpu_ptr(s->cpu_sheaves, cpu);
7710 pcs->main = alloc_empty_sheaf(s, GFP_KERNEL);
7764 static void free_kmem_cache_nodes(struct kmem_cache *s)
7769 for_each_kmem_cache_node(s, node, n) {
7777 s->node[node] = NULL;
7782 void __kmem_cache_release(struct kmem_cache *s)
7784 cache_random_seq_destroy(s);
7785 if (s->cpu_sheaves)
7786 pcs_destroy(s);
7788 if (s->cpu_slab)
7789 lockdep_unregister_key(&s->lock_key);
7791 free_percpu(s->cpu_slab);
7792 free_kmem_cache_nodes(s);
7795 static int init_kmem_cache_nodes(struct kmem_cache *s)
7808 if (s->cpu_sheaves) {
7824 s->node[node] = n;
7829 static void set_cpu_partial(struct kmem_cache *s)
7847 if (!kmem_cache_has_cpu_partial(s))
7849 else if (s->size >= PAGE_SIZE)
7851 else if (s->size >= 1024)
7853 else if (s->size >= 256)
7858 slub_set_cpu_partial(s, nr_objects);
7866 static int calculate_sizes(struct kmem_cache_args *args, struct kmem_cache *s)
7868 slab_flags_t flags = s->flags;
7869 unsigned int size = s->object_size;
7886 !s->ctor)
7887 s->flags |= __OBJECT_POISON;
7889 s->flags &= ~__OBJECT_POISON;
7897 if ((flags & SLAB_RED_ZONE) && size == s->object_size)
7905 s->inuse = size;
7908 (flags & SLAB_POISON) || s->ctor ||
7910 (s->object_size < sizeof(void *) || slub_debug_orig_size(s)))) {
7922 * The assumption that s->offset >= s->inuse means free
7927 s->offset = size;
7930 s->offset = args->freeptr_offset;
7937 s->offset = ALIGN_DOWN(s->object_size / 2, sizeof(void *));
7954 kasan_cache_create(s, &size, &s->flags);
7966 s->red_left_pad = sizeof(void *);
7967 s->red_left_pad = ALIGN(s->red_left_pad, s->align);
7968 size += s->red_left_pad;
7977 size = ALIGN(size, s->align);
7978 s->size = size;
7979 s->reciprocal_size = reciprocal_value(size);
7985 s->allocflags = __GFP_COMP;
7987 if (s->flags & SLAB_CACHE_DMA)
7988 s->allocflags |= GFP_DMA;
7990 if (s->flags & SLAB_CACHE_DMA32)
7991 s->allocflags |= GFP_DMA32;
7993 if (s->flags & SLAB_RECLAIM_ACCOUNT)
7994 s->allocflags |= __GFP_RECLAIMABLE;
7999 s->oo = oo_make(order, size);
8000 s->min = oo_make(get_order(size), size);
8002 return !!oo_objects(s->oo);
8005 static void list_slab_objects(struct kmem_cache *s, struct slab *slab)
8012 slab_bug(s, "Objects remaining on __kmem_cache_shutdown()");
8015 __fill_map(object_map, s, slab);
8017 for_each_object(p, s, addr, slab->objects) {
8019 if (!test_bit(__obj_to_index(s, addr, p), object_map)) {
8023 print_tracking(s, p);
8037 static void free_partial(struct kmem_cache *s, struct kmem_cache_node *n)
8049 list_slab_objects(s, slab);
8055 discard_slab(s, slab);
8058 bool __kmem_cache_empty(struct kmem_cache *s)
8063 for_each_kmem_cache_node(s, node, n)
8072 int __kmem_cache_shutdown(struct kmem_cache *s)
8077 flush_all_cpus_locked(s);
8080 if (s->cpu_sheaves)
8084 for_each_kmem_cache_node(s, node, n) {
8086 barn_shrink(s, n->barn);
8087 free_partial(s, n);
8102 struct kmem_cache *s = slab->slab_cache;
8107 kpp->kp_slab_cache = s;
8111 objp = restore_red_left(s, objp0);
8115 objnr = obj_to_index(s, slab, objp);
8117 objp = base + s->size * objnr;
8119 if (WARN_ON_ONCE(objp < base || objp >= base + slab->objects * s->size
8120 || (objp - base) % s->size) ||
8121 !(s->flags & SLAB_STORE_USER))
8124 objp = fixup_red_left(s, objp);
8125 trackp = get_track(s, objp, TRACK_ALLOC);
8140 trackp = get_track(s, objp, TRACK_FREE);
8227 * cache's usercopy region.
8235 struct kmem_cache *s;
8242 s = slab->slab_cache;
8253 offset = (ptr - slab_address(slab)) % s->size;
8256 if (!is_kfence && kmem_cache_debug_flags(s, SLAB_RED_ZONE)) {
8257 if (offset < s->red_left_pad)
8259 s->name, to_user, offset, n);
8260 offset -= s->red_left_pad;
8264 if (offset >= s->useroffset &&
8265 offset - s->useroffset <= s->usersize &&
8266 n <= s->useroffset - offset + s->usersize)
8269 usercopy_abort("SLUB object", s->name, to_user, offset, n);
8284 static int __kmem_cache_do_shrink(struct kmem_cache *s)
8296 for_each_kmem_cache_node(s, node, n) {
8302 barn_shrink(s, n->barn);
8325 dec_slabs_node(s, node, slab->objects);
8341 free_slab(s, slab);
8350 int __kmem_cache_shrink(struct kmem_cache *s)
8352 flush_all(s);
8353 return __kmem_cache_do_shrink(s);
8358 struct kmem_cache *s;
8361 list_for_each_entry(s, &slab_caches, list) {
8362 flush_all_cpus_locked(s);
8363 __kmem_cache_do_shrink(s);
8373 struct kmem_cache *s;
8382 list_for_each_entry(s, &slab_caches, list) {
8389 if (get_node(s, nid))
8392 if (s->cpu_sheaves) {
8415 s->node[nid] = n;
8462 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
8465 memcpy(s, static_cache, kmem_cache->object_size);
8472 __flush_cpu_slab(s, smp_processor_id());
8473 for_each_kmem_cache_node(s, node, n) {
8477 p->slab_cache = s;
8481 p->slab_cache = s;
8484 list_add(&s->list, &slab_caches);
8485 return s;
8553 struct kmem_cache *s;
8555 s = find_mergeable(size, align, flags, name, ctor);
8556 if (s) {
8557 if (sysfs_slab_alias(s, name))
8558 pr_err("SLUB: Unable to add cache alias %s to sysfs\n",
8561 s->refcount++;
8567 s->object_size = max(s->object_size, size);
8568 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
8571 return s;
8574 int do_kmem_cache_create(struct kmem_cache *s, const char *name,
8580 s->name = name;
8581 s->size = s->object_size = size;
8583 s->flags = kmem_cache_flags(flags, s->name);
8585 s->random = get_random_long();
8587 s->align = args->align;
8588 s->ctor = args->ctor;
8590 s->useroffset = args->useroffset;
8591 s->usersize = args->usersize;
8594 if (!calculate_sizes(args, s))
8601 if (get_order(s->size) > get_order(s->object_size)) {
8602 s->flags &= ~DEBUG_METADATA_FLAGS;
8603 s->offset = 0;
8604 if (!calculate_sizes(args, s))
8610 if (system_has_freelist_aba() && !(s->flags & SLAB_NO_CMPXCHG)) {
8612 s->flags |= __CMPXCHG_DOUBLE;
8620 s->min_partial = min_t(unsigned long, MAX_PARTIAL, ilog2(s->size) / 2);
8621 s->min_partial = max_t(unsigned long, MIN_PARTIAL, s->min_partial);
8623 set_cpu_partial(s);
8626 && !(s->flags & SLAB_DEBUG_FLAGS)) {
8627 s->cpu_sheaves = alloc_percpu(struct slub_percpu_sheaves);
8628 if (!s->cpu_sheaves) {
8633 s->sheaf_capacity = args->sheaf_capacity;
8637 s->remote_node_defrag_ratio = 1000;
8642 if (init_cache_random_seq(s))
8646 if (!init_kmem_cache_nodes(s))
8649 if (!alloc_kmem_cache_cpus(s))
8652 if (s->cpu_sheaves) {
8653 err = init_percpu_sheaves(s);
8668 if (sysfs_slab_add(s))
8669 pr_err("SLUB: Unable to add cache %s to sysfs\n", s->name);
8671 if (s->flags & SLAB_STORE_USER)
8672 debugfs_slab_add(s);
8676 __kmem_cache_release(s);
8693 static void validate_slab(struct kmem_cache *s, struct slab *slab,
8700 slab_err(s, slab, "Not a valid slab page");
8704 if (!check_slab(s, slab) || !on_freelist(s, slab, NULL))
8708 __fill_map(obj_map, s, slab);
8709 for_each_object(p, s, addr, slab->objects) {
8710 u8 val = test_bit(__obj_to_index(s, addr, p), obj_map) ?
8713 if (!check_object(s, slab, p, val))
8718 static int validate_slab_node(struct kmem_cache *s,
8728 validate_slab(s, slab, obj_map);
8732 pr_err("SLUB %s: %ld partial slabs counted but counter=%ld\n",
8733 s->name, count, n->nr_partial);
8737 if (!(s->flags & SLAB_STORE_USER))
8741 validate_slab(s, slab, obj_map);
8745 pr_err("SLUB: %s %ld slabs counted but counter=%ld\n",
8746 s->name, count, node_nr_slabs(n));
8755 long validate_slab_cache(struct kmem_cache *s)
8762 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
8766 flush_all(s);
8767 for_each_kmem_cache_node(s, node, n)
8768 count += validate_slab_node(s, n, obj_map);
8832 static int add_location(struct loc_track *t, struct kmem_cache *s,
8841 unsigned int waste = s->object_size - orig_size;
8924 static void process_slab(struct loc_track *t, struct kmem_cache *s,
8932 __fill_map(obj_map, s, slab);
8934 for_each_object(p, s, addr, slab->objects)
8935 if (!test_bit(__obj_to_index(s, addr, p), obj_map))
8936 add_location(t, s, get_track(s, p, alloc),
8937 is_alloc ? get_orig_size(s, p) :
8938 s->object_size);
8958 static ssize_t show_slab_objects(struct kmem_cache *s,
8975 struct kmem_cache_cpu *c = per_cpu_ptr(s->cpu_slab,
9019 * slab_mem_going_offline_callback) here because slab's memory hot
9027 for_each_kmem_cache_node(s, node, n) {
9044 for_each_kmem_cache_node(s, node, n) {
9075 ssize_t (*show)(struct kmem_cache *s, char *buf);
9076 ssize_t (*store)(struct kmem_cache *s, const char *x, size_t count);
9085 static ssize_t slab_size_show(struct kmem_cache *s, char *buf)
9087 return sysfs_emit(buf, "%u\n", s->size);
9091 static ssize_t align_show(struct kmem_cache *s, char *buf)
9093 return sysfs_emit(buf, "%u\n", s->align);
9097 static ssize_t object_size_show(struct kmem_cache *s, char *buf)
9099 return sysfs_emit(buf, "%u\n", s->object_size);
9103 static ssize_t objs_per_slab_show(struct kmem_cache *s, char *buf)
9105 return sysfs_emit(buf, "%u\n", oo_objects(s->oo));
9109 static ssize_t order_show(struct kmem_cache *s, char *buf)
9111 return sysfs_emit(buf, "%u\n", oo_order(s->oo));
9115 static ssize_t sheaf_capacity_show(struct kmem_cache *s, char *buf)
9117 return sysfs_emit(buf, "%u\n", s->sheaf_capacity);
9121 static ssize_t min_partial_show(struct kmem_cache *s, char *buf)
9123 return sysfs_emit(buf, "%lu\n", s->min_partial);
9126 static ssize_t min_partial_store(struct kmem_cache *s, const char *buf,
9136 s->min_partial = min;
9141 static ssize_t cpu_partial_show(struct kmem_cache *s, char *buf)
9145 nr_partial = s->cpu_partial;
9151 static ssize_t cpu_partial_store(struct kmem_cache *s, const char *buf,
9160 if (objects && !kmem_cache_has_cpu_partial(s))
9163 slub_set_cpu_partial(s, objects);
9164 flush_all(s);
9169 static ssize_t ctor_show(struct kmem_cache *s, char *buf)
9171 if (!s->ctor)
9173 return sysfs_emit(buf, "%pS\n", s->ctor);
9177 static ssize_t aliases_show(struct kmem_cache *s, char *buf)
9179 return sysfs_emit(buf, "%d\n", s->refcount < 0 ? 0 : s->refcount - 1);
9183 static ssize_t partial_show(struct kmem_cache *s, char *buf)
9185 return show_slab_objects(s, buf, SO_PARTIAL);
9189 static ssize_t cpu_slabs_show(struct kmem_cache *s, char *buf)
9191 return show_slab_objects(s, buf, SO_CPU);
9195 static ssize_t objects_partial_show(struct kmem_cache *s, char *buf)
9197 return show_slab_objects(s, buf, SO_PARTIAL|SO_OBJECTS);
9201 static ssize_t slabs_cpu_partial_show(struct kmem_cache *s, char *buf)
9212 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
9220 objects = (slabs * oo_objects(s->oo)) / 2;
9227 slab = slub_percpu_partial(per_cpu_ptr(s->cpu_slab, cpu));
9230 objects = (slabs * oo_objects(s->oo)) / 2;
9242 static ssize_t reclaim_account_show(struct kmem_cache *s, char *buf)
9244 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RECLAIM_ACCOUNT));
9248 static ssize_t hwcache_align_show(struct kmem_cache *s, char *buf)
9250 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_HWCACHE_ALIGN));
9255 static ssize_t cache_dma_show(struct kmem_cache *s, char *buf)
9257 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CACHE_DMA));
9263 static ssize_t usersize_show(struct kmem_cache *s, char *buf)
9265 return sysfs_emit(buf, "%u\n", s->usersize);
9270 static ssize_t destroy_by_rcu_show(struct kmem_cache *s, char *buf)
9272 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TYPESAFE_BY_RCU));
9277 static ssize_t slabs_show(struct kmem_cache *s, char *buf)
9279 return show_slab_objects(s, buf, SO_ALL);
9283 static ssize_t total_objects_show(struct kmem_cache *s, char *buf)
9285 return show_slab_objects(s, buf, SO_ALL|SO_TOTAL);
9289 static ssize_t objects_show(struct kmem_cache *s, char *buf)
9291 return show_slab_objects(s, buf, SO_ALL|SO_OBJECTS);
9295 static ssize_t sanity_checks_show(struct kmem_cache *s, char *buf)
9297 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_CONSISTENCY_CHECKS));
9301 static ssize_t trace_show(struct kmem_cache *s, char *buf)
9303 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_TRACE));
9307 static ssize_t red_zone_show(struct kmem_cache *s, char *buf)
9309 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_RED_ZONE));
9314 static ssize_t poison_show(struct kmem_cache *s, char *buf)
9316 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_POISON));
9321 static ssize_t store_user_show(struct kmem_cache *s, char *buf)
9323 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_STORE_USER));
9328 static ssize_t validate_show(struct kmem_cache *s, char *buf)
9333 static ssize_t validate_store(struct kmem_cache *s,
9338 if (buf[0] == '1' && kmem_cache_debug(s)) {
9339 ret = validate_slab_cache(s);
9350 static ssize_t failslab_show(struct kmem_cache *s, char *buf)
9352 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_FAILSLAB));
9355 static ssize_t failslab_store(struct kmem_cache *s, const char *buf,
9358 if (s->refcount > 1)
9362 WRITE_ONCE(s->flags, s->flags | SLAB_FAILSLAB);
9364 WRITE_ONCE(s->flags, s->flags & ~SLAB_FAILSLAB);
9371 static ssize_t shrink_show(struct kmem_cache *s, char *buf)
9376 static ssize_t shrink_store(struct kmem_cache *s,
9380 kmem_cache_shrink(s);
9388 static ssize_t remote_node_defrag_ratio_show(struct kmem_cache *s, char *buf)
9390 return sysfs_emit(buf, "%u\n", s->remote_node_defrag_ratio / 10);
9393 static ssize_t remote_node_defrag_ratio_store(struct kmem_cache *s,
9405 s->remote_node_defrag_ratio = ratio * 10;
9413 static int show_stat(struct kmem_cache *s, char *buf, enum stat_item si)
9424 unsigned x = per_cpu_ptr(s->cpu_slab, cpu)->stat[si];
9445 static void clear_stat(struct kmem_cache *s, enum stat_item si)
9450 per_cpu_ptr(s->cpu_slab, cpu)->stat[si] = 0;
9454 static ssize_t text##_show(struct kmem_cache *s, char *buf) \
9456 return show_stat(s, buf, si); \
9458 static ssize_t text##_store(struct kmem_cache *s, \
9463 clear_stat(s, si); \
9514 static ssize_t skip_kfence_show(struct kmem_cache *s, char *buf)
9516 return sysfs_emit(buf, "%d\n", !!(s->flags & SLAB_SKIP_KFENCE));
9519 static ssize_t skip_kfence_store(struct kmem_cache *s,
9525 s->flags &= ~SLAB_SKIP_KFENCE;
9527 s->flags |= SLAB_SKIP_KFENCE;
9639 struct kmem_cache *s;
9642 s = to_slab(kobj);
9647 return attribute->show(s, buf);
9655 struct kmem_cache *s;
9658 s = to_slab(kobj);
9663 return attribute->store(s, buf, len);
9683 static inline struct kset *cache_kset(struct kmem_cache *s)
9694 static char *create_unique_id(struct kmem_cache *s)
9710 if (s->flags & SLAB_CACHE_DMA)
9712 if (s->flags & SLAB_CACHE_DMA32)
9714 if (s->flags & SLAB_RECLAIM_ACCOUNT)
9716 if (s->flags & SLAB_CONSISTENCY_CHECKS)
9718 if (s->flags & SLAB_ACCOUNT)
9722 p += snprintf(p, ID_STR_LENGTH - (p - name), "%07u", s->size);
9732 static int sysfs_slab_add(struct kmem_cache *s)
9736 struct kset *kset = cache_kset(s);
9737 int unmergeable = slab_unmergeable(s);
9749 sysfs_remove_link(&slab_kset->kobj, s->name);
9750 name = s->name;
9756 name = create_unique_id(s);
9761 s->kobj.kset = kset;
9762 err = kobject_init_and_add(&s->kobj, &slab_ktype, NULL, "%s", name);
9766 err = sysfs_create_group(&s->kobj, &slab_attr_group);
9772 sysfs_slab_alias(s, s->name);
9779 kobject_del(&s->kobj);
9783 void sysfs_slab_unlink(struct kmem_cache *s)
9785 if (s->kobj.state_in_sysfs)
9786 kobject_del(&s->kobj);
9789 void sysfs_slab_release(struct kmem_cache *s)
9791 kobject_put(&s->kobj);
9799 struct kmem_cache *s;
9806 static int sysfs_slab_alias(struct kmem_cache *s, const char *name)
9820 return sysfs_create_link(&slab_kset->kobj, &s->kobj, name);
9827 al->s = s;
9837 struct kmem_cache *s;
9851 list_for_each_entry(s, &slab_caches, list) {
9852 err = sysfs_slab_add(s);
9854 pr_err("SLUB: Unable to add boot slab %s to sysfs\n",
9855 s->name);
9862 err = sysfs_slab_alias(al->s, al->name);
9864 pr_err("SLUB: Unable to add boot slab alias %s to sysfs\n",
9988 struct kmem_cache *s = file_inode(filep)->i_private;
9994 obj_map = bitmap_alloc(oo_objects(s->oo), GFP_KERNEL);
10008 for_each_kmem_cache_node(s, node, n) {
10017 process_slab(t, s, slab, alloc, obj_map);
10019 process_slab(t, s, slab, alloc, obj_map);
10047 static void debugfs_slab_add(struct kmem_cache *s)
10054 slab_cache_dir = debugfs_create_dir(s->name, slab_debugfs_root);
10056 debugfs_create_file_aux_num("alloc_traces", 0400, slab_cache_dir, s,
10059 debugfs_create_file_aux_num("free_traces", 0400, slab_cache_dir, s,
10063 void debugfs_slab_release(struct kmem_cache *s)
10065 debugfs_lookup_and_remove(s->name, slab_debugfs_root);
10070 struct kmem_cache *s;
10074 list_for_each_entry(s, &slab_caches, list)
10075 if (s->flags & SLAB_STORE_USER)
10076 debugfs_slab_add(s);
10087 void get_slabinfo(struct kmem_cache *s, struct slabinfo *sinfo)
10095 for_each_kmem_cache_node(s, node, n) {
10105 sinfo->objects_per_slab = oo_objects(s->oo);
10106 sinfo->cache_order = oo_order(s->oo);