1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Slab allocator functions that are independent of the allocator strategy
4 *
5 * (C) 2012 Christoph Lameter <cl@gentwo.org>
6 */
7 #include <linux/slab.h>
8
9 #include <linux/mm.h>
10 #include <linux/poison.h>
11 #include <linux/interrupt.h>
12 #include <linux/memory.h>
13 #include <linux/cache.h>
14 #include <linux/compiler.h>
15 #include <linux/kfence.h>
16 #include <linux/module.h>
17 #include <linux/cpu.h>
18 #include <linux/uaccess.h>
19 #include <linux/seq_file.h>
20 #include <linux/dma-mapping.h>
21 #include <linux/swiotlb.h>
22 #include <linux/proc_fs.h>
23 #include <linux/debugfs.h>
24 #include <linux/kmemleak.h>
25 #include <linux/kasan.h>
26 #include <asm/cacheflush.h>
27 #include <asm/tlbflush.h>
28 #include <asm/page.h>
29 #include <linux/memcontrol.h>
30 #include <linux/stackdepot.h>
31 #include <trace/events/rcu.h>
32
33 #include "../kernel/rcu/rcu.h"
34 #include "internal.h"
35 #include "slab.h"
36
37 #define CREATE_TRACE_POINTS
38 #include <trace/events/kmem.h>
39
40 enum slab_state slab_state;
41 LIST_HEAD(slab_caches);
42 DEFINE_MUTEX(slab_mutex);
43 struct kmem_cache *kmem_cache;
44
45 /*
46 * Set of flags that will prevent slab merging.
47 * Any flag that adds per-object metadata should be included,
48 * since slab merging can update s->inuse that affects the metadata layout.
49 */
50 #define SLAB_NEVER_MERGE (SLAB_DEBUG_FLAGS | SLAB_TYPESAFE_BY_RCU | \
51 SLAB_NOLEAKTRACE | SLAB_FAILSLAB | SLAB_NO_MERGE | \
52 SLAB_OBJ_EXT_IN_OBJ)
53
54 #define SLAB_MERGE_SAME (SLAB_RECLAIM_ACCOUNT | SLAB_CACHE_DMA | \
55 SLAB_CACHE_DMA32 | SLAB_ACCOUNT | SLAB_MAY_ACCOUNT)
56
57 /*
58 * Merge control. If this is set then no merging of slab caches will occur.
59 */
60 static bool slab_nomerge = !IS_ENABLED(CONFIG_SLAB_MERGE_DEFAULT);
61
setup_slab_nomerge(char * str)62 static int __init setup_slab_nomerge(char *str)
63 {
64 slab_nomerge = true;
65 return 1;
66 }
67
setup_slab_merge(char * str)68 static int __init setup_slab_merge(char *str)
69 {
70 slab_nomerge = false;
71 return 1;
72 }
73
74 __setup_param("slub_nomerge", slub_nomerge, setup_slab_nomerge, 0);
75 __setup_param("slub_merge", slub_merge, setup_slab_merge, 0);
76
77 __setup("slab_nomerge", setup_slab_nomerge);
78 __setup("slab_merge", setup_slab_merge);
79
80 /*
81 * Determine the size of a slab object
82 */
kmem_cache_size(struct kmem_cache * s)83 unsigned int kmem_cache_size(struct kmem_cache *s)
84 {
85 return s->object_size;
86 }
87 EXPORT_SYMBOL(kmem_cache_size);
88
89 #ifdef CONFIG_DEBUG_VM
90
kmem_cache_is_duplicate_name(const char * name)91 static bool kmem_cache_is_duplicate_name(const char *name)
92 {
93 struct kmem_cache *s;
94
95 list_for_each_entry(s, &slab_caches, list) {
96 if (!strcmp(s->name, name))
97 return true;
98 }
99
100 return false;
101 }
102
kmem_cache_sanity_check(const char * name,unsigned int size)103 static int kmem_cache_sanity_check(const char *name, unsigned int size)
104 {
105 if (!name || in_interrupt() || size > KMALLOC_MAX_SIZE) {
106 pr_err("kmem_cache_create(%s) integrity check failed\n", name);
107 return -EINVAL;
108 }
109
110 /* Duplicate names will confuse slabtop, et al */
111 WARN(kmem_cache_is_duplicate_name(name),
112 "kmem_cache of name '%s' already exists\n", name);
113
114 WARN_ON(strchr(name, ' ')); /* It confuses parsers */
115 return 0;
116 }
117 #else
kmem_cache_sanity_check(const char * name,unsigned int size)118 static inline int kmem_cache_sanity_check(const char *name, unsigned int size)
119 {
120 return 0;
121 }
122 #endif
123
124 /*
125 * Figure out what the alignment of the objects will be given a set of
126 * flags, a user specified alignment and the size of the objects.
127 */
calculate_alignment(slab_flags_t flags,unsigned int align,unsigned int size)128 static unsigned int calculate_alignment(slab_flags_t flags,
129 unsigned int align, unsigned int size)
130 {
131 /*
132 * If the user wants hardware cache aligned objects then follow that
133 * suggestion if the object is sufficiently large.
134 *
135 * The hardware cache alignment cannot override the specified
136 * alignment though. If that is greater then use it.
137 */
138 if (flags & SLAB_HWCACHE_ALIGN) {
139 unsigned int ralign;
140
141 ralign = cache_line_size();
142 while (size <= ralign / 2)
143 ralign /= 2;
144 align = max(align, ralign);
145 }
146
147 align = max(align, arch_slab_minalign());
148
149 return ALIGN(align, sizeof(void *));
150 }
151
152 /*
153 * Find a mergeable slab cache
154 */
slab_unmergeable(struct kmem_cache * s)155 int slab_unmergeable(struct kmem_cache *s)
156 {
157 if (slab_nomerge || (s->flags & SLAB_NEVER_MERGE))
158 return 1;
159
160 if (s->ctor)
161 return 1;
162
163 #ifdef CONFIG_HARDENED_USERCOPY
164 if (s->usersize)
165 return 1;
166 #endif
167
168 /*
169 * We may have set a slab to be unmergeable during bootstrap.
170 */
171 if (s->refcount < 0)
172 return 1;
173
174 return 0;
175 }
176
slab_args_unmergeable(struct kmem_cache_args * args,slab_flags_t flags)177 bool slab_args_unmergeable(struct kmem_cache_args *args, slab_flags_t flags)
178 {
179 if (slab_nomerge)
180 return true;
181
182 if (args->ctor)
183 return true;
184
185 if (IS_ENABLED(CONFIG_HARDENED_USERCOPY) && args->usersize)
186 return true;
187
188 if (flags & SLAB_NEVER_MERGE)
189 return true;
190
191 return false;
192 }
193
find_mergeable(unsigned int size,slab_flags_t flags,const char * name,struct kmem_cache_args * args)194 static struct kmem_cache *find_mergeable(unsigned int size, slab_flags_t flags,
195 const char *name, struct kmem_cache_args *args)
196 {
197 struct kmem_cache *s;
198 unsigned int align;
199
200 flags = kmem_cache_flags(flags, name);
201 if (slab_args_unmergeable(args, flags))
202 return NULL;
203
204 size = ALIGN(size, sizeof(void *));
205 align = calculate_alignment(flags, args->align, size);
206 size = ALIGN(size, align);
207
208 list_for_each_entry_reverse(s, &slab_caches, list) {
209 if (slab_unmergeable(s))
210 continue;
211
212 if (size > s->size)
213 continue;
214
215 if ((flags & SLAB_MERGE_SAME) != (s->flags & SLAB_MERGE_SAME))
216 continue;
217 /*
218 * Check if alignment is compatible.
219 * Courtesy of Adrian Drzewiecki
220 */
221 if ((s->size & ~(align - 1)) != s->size)
222 continue;
223
224 if (s->size - size >= sizeof(void *))
225 continue;
226
227 return s;
228 }
229 return NULL;
230 }
231
create_cache(const char * name,unsigned int object_size,struct kmem_cache_args * args,slab_flags_t flags)232 static struct kmem_cache *create_cache(const char *name,
233 unsigned int object_size,
234 struct kmem_cache_args *args,
235 slab_flags_t flags)
236 {
237 struct kmem_cache *s;
238 int err;
239
240 /* If a custom freelist pointer is requested make sure it's sane. */
241 err = -EINVAL;
242 if (args->use_freeptr_offset &&
243 (args->freeptr_offset >= object_size ||
244 (!(flags & SLAB_TYPESAFE_BY_RCU) && !args->ctor) ||
245 !IS_ALIGNED(args->freeptr_offset, __alignof__(freeptr_t))))
246 goto out;
247
248 err = -ENOMEM;
249 s = kmem_cache_zalloc(kmem_cache, GFP_KERNEL);
250 if (!s)
251 goto out;
252 err = do_kmem_cache_create(s, name, object_size, args, flags);
253 if (err)
254 goto out_free_cache;
255
256 s->refcount = 1;
257 list_add(&s->list, &slab_caches);
258 return s;
259
260 out_free_cache:
261 kmem_cache_free(kmem_cache, s);
262 out:
263 return ERR_PTR(err);
264 }
265
266 static struct kmem_cache *
__kmem_cache_alias(const char * name,unsigned int size,slab_flags_t flags,struct kmem_cache_args * args)267 __kmem_cache_alias(const char *name, unsigned int size, slab_flags_t flags,
268 struct kmem_cache_args *args)
269 {
270 struct kmem_cache *s;
271
272 s = find_mergeable(size, flags, name, args);
273 if (s) {
274 if (sysfs_slab_alias(s, name))
275 pr_err("SLUB: Unable to add cache alias %s to sysfs\n",
276 name);
277
278 s->refcount++;
279
280 /*
281 * Adjust the object sizes so that we clear
282 * the complete object on kzalloc.
283 */
284 s->object_size = max(s->object_size, size);
285 s->inuse = max(s->inuse, ALIGN(size, sizeof(void *)));
286 }
287
288 return s;
289 }
290
291 /**
292 * __kmem_cache_create_args - Create a kmem cache.
293 * @name: A string which is used in /proc/slabinfo to identify this cache.
294 * @object_size: The size of objects to be created in this cache.
295 * @args: Additional arguments for the cache creation (see
296 * &struct kmem_cache_args).
297 * @flags: See the descriptions of individual flags. The common ones are listed
298 * in the description below.
299 *
300 * Not to be called directly, use the kmem_cache_create() wrapper with the same
301 * parameters.
302 *
303 * Commonly used @flags:
304 *
305 * &SLAB_ACCOUNT - Account allocations to memcg.
306 *
307 * &SLAB_HWCACHE_ALIGN - Align objects on cache line boundaries.
308 *
309 * &SLAB_RECLAIM_ACCOUNT - Objects are reclaimable.
310 *
311 * &SLAB_TYPESAFE_BY_RCU - Slab page (not individual objects) freeing delayed
312 * by a grace period - see the full description before using.
313 *
314 * Context: Cannot be called within a interrupt, but can be interrupted.
315 *
316 * Return: a pointer to the cache on success, NULL on failure.
317 */
__kmem_cache_create_args(const char * name,unsigned int object_size,struct kmem_cache_args * args,slab_flags_t flags)318 struct kmem_cache *__kmem_cache_create_args(const char *name,
319 unsigned int object_size,
320 struct kmem_cache_args *args,
321 slab_flags_t flags)
322 {
323 struct kmem_cache *s = NULL;
324 const char *cache_name;
325 int err;
326
327 #ifdef CONFIG_SLUB_DEBUG
328 /*
329 * If no slab_debug was enabled globally, the static key is not yet
330 * enabled by setup_slub_debug(). Enable it if the cache is being
331 * created with any of the debugging flags passed explicitly.
332 * It's also possible that this is the first cache created with
333 * SLAB_STORE_USER and we should init stack_depot for it.
334 */
335 if (flags & SLAB_DEBUG_FLAGS)
336 static_branch_enable(&slub_debug_enabled);
337 if (flags & SLAB_STORE_USER)
338 stack_depot_init();
339 #else
340 flags &= ~SLAB_DEBUG_FLAGS;
341 #endif
342
343 /*
344 * Caches with specific capacity are special enough. It's simpler to
345 * make them unmergeable.
346 */
347 if (args->sheaf_capacity)
348 flags |= SLAB_NO_MERGE;
349
350 mutex_lock(&slab_mutex);
351
352 err = kmem_cache_sanity_check(name, object_size);
353 if (err) {
354 goto out_unlock;
355 }
356
357 if (flags & ~SLAB_FLAGS_PERMITTED) {
358 err = -EINVAL;
359 goto out_unlock;
360 }
361
362 /*
363 * For now we assume any cache can be used with __GFP_ACCOUNT and thus
364 * may need to store objcg pointers for objects
365 */
366 if (!mem_cgroup_kmem_disabled())
367 flags |= SLAB_MAY_ACCOUNT;
368
369 /* Fail closed on bad usersize of useroffset values. */
370 if (!IS_ENABLED(CONFIG_HARDENED_USERCOPY) ||
371 WARN_ON(!args->usersize && args->useroffset) ||
372 WARN_ON(object_size < args->usersize ||
373 object_size - args->usersize < args->useroffset))
374 args->usersize = args->useroffset = 0;
375
376 s = __kmem_cache_alias(name, object_size, flags, args);
377 if (s)
378 goto out_unlock;
379
380 cache_name = kstrdup_const(name, GFP_KERNEL);
381 if (!cache_name) {
382 err = -ENOMEM;
383 goto out_unlock;
384 }
385
386 args->align = calculate_alignment(flags, args->align, object_size);
387 s = create_cache(cache_name, object_size, args, flags);
388 if (IS_ERR(s)) {
389 err = PTR_ERR(s);
390 kfree_const(cache_name);
391 }
392
393 out_unlock:
394 mutex_unlock(&slab_mutex);
395
396 if (err) {
397 if (flags & SLAB_PANIC)
398 panic("%s: Failed to create slab '%s'. Error %d\n",
399 __func__, name, err);
400 else {
401 pr_warn("%s(%s) failed with error %d\n",
402 __func__, name, err);
403 dump_stack();
404 }
405 return NULL;
406 }
407 return s;
408 }
409 EXPORT_SYMBOL(__kmem_cache_create_args);
410
411 static struct kmem_cache *kmem_buckets_cache __ro_after_init;
412
413 /**
414 * kmem_buckets_create - Create a set of caches that handle dynamic sized
415 * allocations via kmem_buckets_alloc()
416 * @name: A prefix string which is used in /proc/slabinfo to identify this
417 * cache. The individual caches with have their sizes as the suffix.
418 * @flags: SLAB flags (see kmem_cache_create() for details).
419 * @useroffset: Starting offset within an allocation that may be copied
420 * to/from userspace.
421 * @usersize: How many bytes, starting at @useroffset, may be copied
422 * to/from userspace.
423 * @ctor: A constructor for the objects, run when new allocations are made.
424 *
425 * Cannot be called within an interrupt, but can be interrupted.
426 *
427 * Return: a pointer to the cache on success, NULL on failure. When
428 * CONFIG_SLAB_BUCKETS is not enabled, ZERO_SIZE_PTR is returned, and
429 * subsequent calls to kmem_buckets_alloc() will fall back to kmalloc().
430 * (i.e. callers only need to check for NULL on failure.)
431 */
kmem_buckets_create(const char * name,slab_flags_t flags,unsigned int useroffset,unsigned int usersize,void (* ctor)(void *))432 kmem_buckets *kmem_buckets_create(const char *name, slab_flags_t flags,
433 unsigned int useroffset,
434 unsigned int usersize,
435 void (*ctor)(void *))
436 {
437 unsigned long mask = 0;
438 unsigned int idx;
439 kmem_buckets *b;
440
441 BUILD_BUG_ON(ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]) > BITS_PER_LONG);
442
443 /*
444 * When the separate buckets API is not built in, just return
445 * a non-NULL value for the kmem_buckets pointer, which will be
446 * unused when performing allocations.
447 */
448 if (!IS_ENABLED(CONFIG_SLAB_BUCKETS))
449 return ZERO_SIZE_PTR;
450
451 if (WARN_ON(!kmem_buckets_cache))
452 return NULL;
453
454 b = kmem_cache_alloc(kmem_buckets_cache, GFP_KERNEL|__GFP_ZERO);
455 if (WARN_ON(!b))
456 return NULL;
457
458 flags |= SLAB_NO_MERGE;
459
460 for (idx = 0; idx < ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]); idx++) {
461 char *short_size, *cache_name;
462 unsigned int cache_useroffset, cache_usersize;
463 unsigned int size, aligned_idx;
464
465 if (!kmalloc_caches[KMALLOC_NORMAL][idx])
466 continue;
467
468 size = kmalloc_caches[KMALLOC_NORMAL][idx]->object_size;
469 if (!size)
470 continue;
471
472 short_size = strchr(kmalloc_caches[KMALLOC_NORMAL][idx]->name, '-');
473 if (WARN_ON(!short_size))
474 goto fail;
475
476 if (useroffset >= size) {
477 cache_useroffset = 0;
478 cache_usersize = 0;
479 } else {
480 cache_useroffset = useroffset;
481 cache_usersize = min(size - cache_useroffset, usersize);
482 }
483
484 aligned_idx = __kmalloc_index(size, false);
485 if (!(*b)[aligned_idx]) {
486 cache_name = kasprintf(GFP_KERNEL, "%s-%s", name, short_size + 1);
487 if (WARN_ON(!cache_name))
488 goto fail;
489 (*b)[aligned_idx] = kmem_cache_create_usercopy(cache_name, size,
490 0, flags, cache_useroffset,
491 cache_usersize, ctor);
492 kfree(cache_name);
493 if (WARN_ON(!(*b)[aligned_idx]))
494 goto fail;
495 set_bit(aligned_idx, &mask);
496 }
497 if (idx != aligned_idx)
498 (*b)[idx] = (*b)[aligned_idx];
499 }
500
501 return b;
502
503 fail:
504 for_each_set_bit(idx, &mask, ARRAY_SIZE(kmalloc_caches[KMALLOC_NORMAL]))
505 kmem_cache_destroy((*b)[idx]);
506 kmem_cache_free(kmem_buckets_cache, b);
507
508 return NULL;
509 }
510 EXPORT_SYMBOL(kmem_buckets_create);
511
512 /*
513 * For a given kmem_cache, kmem_cache_destroy() should only be called
514 * once or there will be a use-after-free problem. The actual deletion
515 * and release of the kobject does not need slab_mutex or cpu_hotplug_lock
516 * protection. So they are now done without holding those locks.
517 */
kmem_cache_release(struct kmem_cache * s)518 static void kmem_cache_release(struct kmem_cache *s)
519 {
520 kfence_shutdown_cache(s);
521 if (__is_defined(SLAB_SUPPORTS_SYSFS) && slab_state >= FULL)
522 sysfs_slab_release(s);
523 else
524 slab_kmem_cache_release(s);
525 }
526
slab_kmem_cache_release(struct kmem_cache * s)527 void slab_kmem_cache_release(struct kmem_cache *s)
528 {
529 __kmem_cache_release(s);
530 kfree_const(s->name);
531 kmem_cache_free(kmem_cache, s);
532 }
533
kmem_cache_destroy(struct kmem_cache * s)534 void kmem_cache_destroy(struct kmem_cache *s)
535 {
536 int err;
537
538 if (unlikely(!s) || !kasan_check_byte(s))
539 return;
540
541 /* in-flight kfree_rcu()'s may include objects from our cache */
542 kvfree_rcu_barrier_on_cache(s);
543
544 if (IS_ENABLED(CONFIG_SLUB_RCU_DEBUG) &&
545 (s->flags & SLAB_TYPESAFE_BY_RCU)) {
546 /*
547 * Under CONFIG_SLUB_RCU_DEBUG, when objects in a
548 * SLAB_TYPESAFE_BY_RCU slab are freed, SLUB will internally
549 * defer their freeing with call_rcu().
550 * Wait for such call_rcu() invocations here before actually
551 * destroying the cache.
552 *
553 * It doesn't matter that we haven't looked at the slab refcount
554 * yet - slabs with SLAB_TYPESAFE_BY_RCU can't be merged, so
555 * the refcount should be 1 here.
556 */
557 rcu_barrier();
558 }
559
560 /* Wait for deferred work from kmalloc/kfree_nolock() */
561 deferred_work_barrier();
562
563 cpus_read_lock();
564 mutex_lock(&slab_mutex);
565
566 s->refcount--;
567 if (s->refcount) {
568 mutex_unlock(&slab_mutex);
569 cpus_read_unlock();
570 return;
571 }
572
573 /* free asan quarantined objects */
574 kasan_cache_shutdown(s);
575
576 err = __kmem_cache_shutdown(s);
577 if (!slab_in_kunit_test())
578 WARN(err, "%s %s: Slab cache still has objects when called from %pS",
579 __func__, s->name, (void *)_RET_IP_);
580
581 list_del(&s->list);
582
583 mutex_unlock(&slab_mutex);
584 cpus_read_unlock();
585
586 if (slab_state >= FULL)
587 sysfs_slab_unlink(s);
588 debugfs_slab_release(s);
589
590 if (err)
591 return;
592
593 if (s->flags & SLAB_TYPESAFE_BY_RCU)
594 rcu_barrier();
595
596 kmem_cache_release(s);
597 }
598 EXPORT_SYMBOL(kmem_cache_destroy);
599
600 /**
601 * kmem_cache_shrink - Shrink a cache.
602 * @cachep: The cache to shrink.
603 *
604 * Releases as many slabs as possible for a cache.
605 * To help debugging, a zero exit status indicates all slabs were released.
606 *
607 * Return: %0 if all slabs were released, non-zero otherwise
608 */
kmem_cache_shrink(struct kmem_cache * cachep)609 int kmem_cache_shrink(struct kmem_cache *cachep)
610 {
611 kasan_cache_shrink(cachep);
612
613 return __kmem_cache_shrink(cachep);
614 }
615 EXPORT_SYMBOL(kmem_cache_shrink);
616
slab_is_available(void)617 bool slab_is_available(void)
618 {
619 return slab_state >= UP;
620 }
621
622 #ifdef CONFIG_PRINTK
kmem_obj_info(struct kmem_obj_info * kpp,void * object,struct slab * slab)623 static void kmem_obj_info(struct kmem_obj_info *kpp, void *object, struct slab *slab)
624 {
625 if (__kfence_obj_info(kpp, object, slab))
626 return;
627 __kmem_obj_info(kpp, object, slab);
628 }
629
630 /**
631 * kmem_dump_obj - Print available slab provenance information
632 * @object: slab object for which to find provenance information.
633 *
634 * This function uses pr_cont(), so that the caller is expected to have
635 * printed out whatever preamble is appropriate. The provenance information
636 * depends on the type of object and on how much debugging is enabled.
637 * For a slab-cache object, the fact that it is a slab object is printed,
638 * and, if available, the slab name, return address, and stack trace from
639 * the allocation and last free path of that object.
640 *
641 * Return: %true if the pointer is to a not-yet-freed object from
642 * kmalloc() or kmem_cache_alloc(), either %true or %false if the pointer
643 * is to an already-freed object, and %false otherwise.
644 */
kmem_dump_obj(void * object)645 bool kmem_dump_obj(void *object)
646 {
647 char *cp = IS_ENABLED(CONFIG_MMU) ? "" : "/vmalloc";
648 int i;
649 struct slab *slab;
650 unsigned long ptroffset;
651 struct kmem_obj_info kp = { };
652
653 /* Some arches consider ZERO_SIZE_PTR to be a valid address. */
654 if (object < (void *)PAGE_SIZE || !virt_addr_valid(object))
655 return false;
656 slab = virt_to_slab(object);
657 if (!slab)
658 return false;
659
660 kmem_obj_info(&kp, object, slab);
661 if (kp.kp_slab_cache)
662 pr_cont(" slab%s %s", cp, kp.kp_slab_cache->name);
663 else
664 pr_cont(" slab%s", cp);
665 if (is_kfence_address(object))
666 pr_cont(" (kfence)");
667 if (kp.kp_objp)
668 pr_cont(" start %px", kp.kp_objp);
669 if (kp.kp_data_offset)
670 pr_cont(" data offset %lu", kp.kp_data_offset);
671 if (kp.kp_objp) {
672 ptroffset = ((char *)object - (char *)kp.kp_objp) - kp.kp_data_offset;
673 pr_cont(" pointer offset %lu", ptroffset);
674 }
675 if (kp.kp_slab_cache && kp.kp_slab_cache->object_size)
676 pr_cont(" size %u", kp.kp_slab_cache->object_size);
677 if (kp.kp_ret)
678 pr_cont(" allocated at %pS\n", kp.kp_ret);
679 else
680 pr_cont("\n");
681 for (i = 0; i < ARRAY_SIZE(kp.kp_stack); i++) {
682 if (!kp.kp_stack[i])
683 break;
684 pr_info(" %pS\n", kp.kp_stack[i]);
685 }
686
687 if (kp.kp_free_stack[0])
688 pr_cont(" Free path:\n");
689
690 for (i = 0; i < ARRAY_SIZE(kp.kp_free_stack); i++) {
691 if (!kp.kp_free_stack[i])
692 break;
693 pr_info(" %pS\n", kp.kp_free_stack[i]);
694 }
695
696 return true;
697 }
698 EXPORT_SYMBOL_GPL(kmem_dump_obj);
699 #endif
700
701 /* Create a cache during boot when no slab services are available yet */
create_boot_cache(struct kmem_cache * s,const char * name,unsigned int size,slab_flags_t flags,unsigned int useroffset,unsigned int usersize)702 void __init create_boot_cache(struct kmem_cache *s, const char *name,
703 unsigned int size, slab_flags_t flags,
704 unsigned int useroffset, unsigned int usersize)
705 {
706 int err;
707 unsigned int align = ARCH_KMALLOC_MINALIGN;
708 struct kmem_cache_args kmem_args = {};
709
710 /*
711 * kmalloc caches guarantee alignment of at least the largest
712 * power-of-two divisor of the size. For power-of-two sizes,
713 * it is the size itself.
714 */
715 if (flags & SLAB_KMALLOC)
716 align = max(align, 1U << (ffs(size) - 1));
717 kmem_args.align = calculate_alignment(flags, align, size);
718
719 #ifdef CONFIG_HARDENED_USERCOPY
720 kmem_args.useroffset = useroffset;
721 kmem_args.usersize = usersize;
722 #endif
723
724 err = do_kmem_cache_create(s, name, size, &kmem_args, flags);
725
726 if (err)
727 panic("Creation of kmalloc slab %s size=%u failed. Reason %d\n",
728 name, size, err);
729
730 s->refcount = -1; /* Exempt from merging for now */
731 }
732
create_kmalloc_cache(const char * name,unsigned int size,slab_flags_t flags)733 static struct kmem_cache *__init create_kmalloc_cache(const char *name,
734 unsigned int size,
735 slab_flags_t flags)
736 {
737 struct kmem_cache *s = kmem_cache_zalloc(kmem_cache, GFP_NOWAIT);
738
739 if (!s)
740 panic("Out of memory when creating slab %s\n", name);
741
742 create_boot_cache(s, name, size, flags | SLAB_KMALLOC, 0, size);
743 list_add(&s->list, &slab_caches);
744 s->refcount = 1;
745 return s;
746 }
747
748 kmem_buckets kmalloc_caches[NR_KMALLOC_TYPES] __ro_after_init =
749 { /* initialization for https://llvm.org/pr42570 */ };
750 EXPORT_SYMBOL(kmalloc_caches);
751
752 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM
753 unsigned long random_kmalloc_seed __ro_after_init;
754 EXPORT_SYMBOL(random_kmalloc_seed);
755 #endif
756
757 /*
758 * Conversion table for small slabs sizes / 8 to the index in the
759 * kmalloc array. This is necessary for slabs < 192 since we have non power
760 * of two cache sizes there. The size of larger slabs can be determined using
761 * fls.
762 */
763 u8 kmalloc_size_index[24] __ro_after_init = {
764 3, /* 8 */
765 4, /* 16 */
766 5, /* 24 */
767 5, /* 32 */
768 6, /* 40 */
769 6, /* 48 */
770 6, /* 56 */
771 6, /* 64 */
772 1, /* 72 */
773 1, /* 80 */
774 1, /* 88 */
775 1, /* 96 */
776 7, /* 104 */
777 7, /* 112 */
778 7, /* 120 */
779 7, /* 128 */
780 2, /* 136 */
781 2, /* 144 */
782 2, /* 152 */
783 2, /* 160 */
784 2, /* 168 */
785 2, /* 176 */
786 2, /* 184 */
787 2 /* 192 */
788 };
789
kmalloc_size_roundup(size_t size)790 size_t kmalloc_size_roundup(size_t size)
791 {
792 if (size && size <= KMALLOC_MAX_CACHE_SIZE) {
793 struct kmem_cache *s;
794
795 /*
796 * The flags don't matter since size_index is common to all.
797 * Neither does the caller for just getting ->object_size.
798 */
799 s = kmalloc_slab(size, NULL, GFP_KERNEL, __kmalloc_token(0),
800 SLAB_ALLOC_DEFAULT);
801 return s->object_size;
802 }
803
804 /* Above the smaller buckets, size is a multiple of page size. */
805 if (size && size <= KMALLOC_MAX_SIZE)
806 return PAGE_SIZE << get_order(size);
807
808 /*
809 * Return 'size' for 0 - kmalloc() returns ZERO_SIZE_PTR
810 * and very large size - kmalloc() may fail.
811 */
812 return size;
813
814 }
815 EXPORT_SYMBOL(kmalloc_size_roundup);
816
817 #ifdef CONFIG_ZONE_DMA
818 #define KMALLOC_DMA_NAME(sz) .name[KMALLOC_DMA] = "dma-kmalloc-" #sz,
819 #else
820 #define KMALLOC_DMA_NAME(sz)
821 #endif
822
823 #ifdef CONFIG_MEMCG
824 #define KMALLOC_CGROUP_NAME(sz) .name[KMALLOC_CGROUP] = "kmalloc-cg-" #sz,
825 #else
826 #define KMALLOC_CGROUP_NAME(sz)
827 #endif
828
829 #ifndef CONFIG_SLUB_TINY
830 #define KMALLOC_RCL_NAME(sz) .name[KMALLOC_RECLAIM] = "kmalloc-rcl-" #sz,
831 #else
832 #define KMALLOC_RCL_NAME(sz)
833 #endif
834
835 #ifdef CONFIG_KMALLOC_PARTITION_CACHES
836 #define __KMALLOC_PARTITION_CONCAT(a, b) a ## b
837 #define KMALLOC_PARTITION_NAME(N, sz) __KMALLOC_PARTITION_CONCAT(KMA_PART_, N)(sz)
838 #define KMA_PART_1(sz) .name[KMALLOC_PARTITION_START + 1] = "kmalloc-part-01-" #sz,
839 #define KMA_PART_2(sz) KMA_PART_1(sz) .name[KMALLOC_PARTITION_START + 2] = "kmalloc-part-02-" #sz,
840 #define KMA_PART_3(sz) KMA_PART_2(sz) .name[KMALLOC_PARTITION_START + 3] = "kmalloc-part-03-" #sz,
841 #define KMA_PART_4(sz) KMA_PART_3(sz) .name[KMALLOC_PARTITION_START + 4] = "kmalloc-part-04-" #sz,
842 #define KMA_PART_5(sz) KMA_PART_4(sz) .name[KMALLOC_PARTITION_START + 5] = "kmalloc-part-05-" #sz,
843 #define KMA_PART_6(sz) KMA_PART_5(sz) .name[KMALLOC_PARTITION_START + 6] = "kmalloc-part-06-" #sz,
844 #define KMA_PART_7(sz) KMA_PART_6(sz) .name[KMALLOC_PARTITION_START + 7] = "kmalloc-part-07-" #sz,
845 #define KMA_PART_8(sz) KMA_PART_7(sz) .name[KMALLOC_PARTITION_START + 8] = "kmalloc-part-08-" #sz,
846 #define KMA_PART_9(sz) KMA_PART_8(sz) .name[KMALLOC_PARTITION_START + 9] = "kmalloc-part-09-" #sz,
847 #define KMA_PART_10(sz) KMA_PART_9(sz) .name[KMALLOC_PARTITION_START + 10] = "kmalloc-part-10-" #sz,
848 #define KMA_PART_11(sz) KMA_PART_10(sz) .name[KMALLOC_PARTITION_START + 11] = "kmalloc-part-11-" #sz,
849 #define KMA_PART_12(sz) KMA_PART_11(sz) .name[KMALLOC_PARTITION_START + 12] = "kmalloc-part-12-" #sz,
850 #define KMA_PART_13(sz) KMA_PART_12(sz) .name[KMALLOC_PARTITION_START + 13] = "kmalloc-part-13-" #sz,
851 #define KMA_PART_14(sz) KMA_PART_13(sz) .name[KMALLOC_PARTITION_START + 14] = "kmalloc-part-14-" #sz,
852 #define KMA_PART_15(sz) KMA_PART_14(sz) .name[KMALLOC_PARTITION_START + 15] = "kmalloc-part-15-" #sz,
853 #else // CONFIG_KMALLOC_PARTITION_CACHES
854 #define KMALLOC_PARTITION_NAME(N, sz)
855 #endif
856
857 #ifdef CONFIG_SLAB_OBJ_EXT
858 #define KMALLOC_NO_OBJ_EXT_NAME(sz) .name[KMALLOC_NO_OBJ_EXT] = "kmalloc-no-objext-" #sz,
859 #else
860 #define KMALLOC_NO_OBJ_EXT_NAME(sz)
861 #endif
862
863 #define INIT_KMALLOC_INFO(__size, __short_size) \
864 { \
865 .name[KMALLOC_NORMAL] = "kmalloc-" #__short_size, \
866 KMALLOC_RCL_NAME(__short_size) \
867 KMALLOC_CGROUP_NAME(__short_size) \
868 KMALLOC_DMA_NAME(__short_size) \
869 KMALLOC_PARTITION_NAME(KMALLOC_PARTITION_CACHES_NR, __short_size) \
870 KMALLOC_NO_OBJ_EXT_NAME(__short_size) \
871 .size = __size, \
872 }
873
874 /*
875 * kmalloc_info[] is to make slab_debug=,kmalloc-xx option work at boot time.
876 * kmalloc_index() supports up to 2^21=2MB, so the final entry of the table is
877 * kmalloc-2M.
878 */
879 const struct kmalloc_info_struct kmalloc_info[] __initconst = {
880 INIT_KMALLOC_INFO(0, 0),
881 INIT_KMALLOC_INFO(96, 96),
882 INIT_KMALLOC_INFO(192, 192),
883 INIT_KMALLOC_INFO(8, 8),
884 INIT_KMALLOC_INFO(16, 16),
885 INIT_KMALLOC_INFO(32, 32),
886 INIT_KMALLOC_INFO(64, 64),
887 INIT_KMALLOC_INFO(128, 128),
888 INIT_KMALLOC_INFO(256, 256),
889 INIT_KMALLOC_INFO(512, 512),
890 INIT_KMALLOC_INFO(1024, 1k),
891 INIT_KMALLOC_INFO(2048, 2k),
892 INIT_KMALLOC_INFO(4096, 4k),
893 INIT_KMALLOC_INFO(8192, 8k),
894 INIT_KMALLOC_INFO(16384, 16k),
895 INIT_KMALLOC_INFO(32768, 32k),
896 INIT_KMALLOC_INFO(65536, 64k),
897 INIT_KMALLOC_INFO(131072, 128k),
898 INIT_KMALLOC_INFO(262144, 256k),
899 INIT_KMALLOC_INFO(524288, 512k),
900 INIT_KMALLOC_INFO(1048576, 1M),
901 INIT_KMALLOC_INFO(2097152, 2M)
902 };
903
904 /*
905 * Patch up the size_index table if we have strange large alignment
906 * requirements for the kmalloc array. This is only the case for
907 * MIPS it seems. The standard arches will not generate any code here.
908 *
909 * Largest permitted alignment is 256 bytes due to the way we
910 * handle the index determination for the smaller caches.
911 *
912 * Make sure that nothing crazy happens if someone starts tinkering
913 * around with ARCH_KMALLOC_MINALIGN
914 */
setup_kmalloc_cache_index_table(void)915 void __init setup_kmalloc_cache_index_table(void)
916 {
917 unsigned int i;
918
919 BUILD_BUG_ON(KMALLOC_MIN_SIZE > 256 ||
920 !is_power_of_2(KMALLOC_MIN_SIZE));
921
922 for (i = 8; i < KMALLOC_MIN_SIZE; i += 8) {
923 unsigned int elem = size_index_elem(i);
924
925 if (elem >= ARRAY_SIZE(kmalloc_size_index))
926 break;
927 kmalloc_size_index[elem] = KMALLOC_SHIFT_LOW;
928 }
929
930 if (KMALLOC_MIN_SIZE >= 64) {
931 /*
932 * The 96 byte sized cache is not used if the alignment
933 * is 64 byte.
934 */
935 for (i = 64 + 8; i <= 96; i += 8)
936 kmalloc_size_index[size_index_elem(i)] = 7;
937
938 }
939
940 if (KMALLOC_MIN_SIZE >= 128) {
941 /*
942 * The 192 byte sized cache is not used if the alignment
943 * is 128 byte. Redirect kmalloc to use the 256 byte cache
944 * instead.
945 */
946 for (i = 128 + 8; i <= 192; i += 8)
947 kmalloc_size_index[size_index_elem(i)] = 8;
948 }
949 }
950
__kmalloc_minalign(void)951 static unsigned int __kmalloc_minalign(void)
952 {
953 unsigned int minalign = dma_get_cache_alignment();
954
955 if (IS_ENABLED(CONFIG_DMA_BOUNCE_UNALIGNED_KMALLOC) &&
956 is_swiotlb_allocated())
957 minalign = ARCH_KMALLOC_MINALIGN;
958
959 return max(minalign, arch_slab_minalign());
960 }
961
962 static void __init
new_kmalloc_cache(int idx,enum kmalloc_cache_type type)963 new_kmalloc_cache(int idx, enum kmalloc_cache_type type)
964 {
965 slab_flags_t flags = 0;
966 unsigned int minalign = __kmalloc_minalign();
967 unsigned int aligned_size = kmalloc_info[idx].size;
968 int aligned_idx = idx;
969
970 if ((KMALLOC_RECLAIM != KMALLOC_NORMAL) && (type == KMALLOC_RECLAIM)) {
971 flags |= SLAB_RECLAIM_ACCOUNT;
972 } else if (IS_ENABLED(CONFIG_MEMCG) && (type == KMALLOC_CGROUP)) {
973 if (mem_cgroup_kmem_disabled()) {
974 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
975 return;
976 }
977 flags |= SLAB_ACCOUNT;
978 } else if (IS_ENABLED(CONFIG_SLAB_OBJ_EXT) && type == KMALLOC_NO_OBJ_EXT) {
979 if (!need_kmalloc_no_objext()) {
980 kmalloc_caches[type][idx] = kmalloc_caches[KMALLOC_NORMAL][idx];
981 return;
982 }
983 flags |= SLAB_NO_OBJ_EXT | SLAB_NO_MERGE;
984 } else if (IS_ENABLED(CONFIG_ZONE_DMA) && (type == KMALLOC_DMA)) {
985 flags |= SLAB_CACHE_DMA;
986 }
987
988 #ifdef CONFIG_KMALLOC_PARTITION_CACHES
989 if (type >= KMALLOC_PARTITION_START && type <= KMALLOC_PARTITION_END)
990 flags |= SLAB_NO_MERGE;
991 #endif
992
993 /*
994 * If memcg_kmem is enabled and this is a KMALLOC_NORMAL cache and not
995 * aliased with any other type, make sure it's never merged with any other
996 * cache.
997 *
998 * In other cases the kmalloc cache may end up being used for a
999 * __GFP_ACCOUNT allocation so mark it as such. The exception is a
1000 * KMALLOC_NO_OBJ_EXT cache.
1001 */
1002 if (!mem_cgroup_kmem_disabled()) {
1003 if (type == KMALLOC_NORMAL && KMALLOC_RECLAIM != KMALLOC_NORMAL)
1004 flags |= SLAB_NO_MERGE;
1005 else if (!(flags & SLAB_NO_OBJ_EXT))
1006 flags |= SLAB_MAY_ACCOUNT;
1007 }
1008
1009 if (minalign > ARCH_KMALLOC_MINALIGN) {
1010 aligned_size = ALIGN(aligned_size, minalign);
1011 aligned_idx = __kmalloc_index(aligned_size, false);
1012 }
1013
1014 if (!kmalloc_caches[type][aligned_idx])
1015 kmalloc_caches[type][aligned_idx] = create_kmalloc_cache(
1016 kmalloc_info[aligned_idx].name[type],
1017 aligned_size, flags);
1018 if (idx != aligned_idx)
1019 kmalloc_caches[type][idx] = kmalloc_caches[type][aligned_idx];
1020 }
1021
1022 /*
1023 * Create the kmalloc array. Some of the regular kmalloc arrays
1024 * may already have been created because they were needed to
1025 * enable allocations for slab creation.
1026 */
create_kmalloc_caches(void)1027 void __init create_kmalloc_caches(void)
1028 {
1029 int i;
1030 enum kmalloc_cache_type type;
1031
1032 /*
1033 * Including KMALLOC_CGROUP if CONFIG_MEMCG defined
1034 */
1035 for (type = KMALLOC_NORMAL; type < NR_KMALLOC_TYPES; type++) {
1036 /* Caches that are NOT of the two-to-the-power-of size. */
1037 if (KMALLOC_MIN_SIZE <= 32)
1038 new_kmalloc_cache(1, type);
1039 if (KMALLOC_MIN_SIZE <= 64)
1040 new_kmalloc_cache(2, type);
1041
1042 /* Caches that are of the two-to-the-power-of size. */
1043 for (i = KMALLOC_SHIFT_LOW; i <= KMALLOC_SHIFT_HIGH; i++)
1044 new_kmalloc_cache(i, type);
1045 }
1046 #ifdef CONFIG_KMALLOC_PARTITION_RANDOM
1047 random_kmalloc_seed = get_random_u64();
1048 #endif
1049
1050 /* Kmalloc array is now usable */
1051 slab_state = UP;
1052
1053 if (IS_ENABLED(CONFIG_SLAB_BUCKETS))
1054 kmem_buckets_cache = kmem_cache_create("kmalloc_buckets",
1055 sizeof(kmem_buckets),
1056 0, SLAB_NO_MERGE, NULL);
1057 }
1058
kmalloc_fix_flags(gfp_t flags)1059 gfp_t kmalloc_fix_flags(gfp_t flags)
1060 {
1061 gfp_t invalid_mask = flags & GFP_SLAB_BUG_MASK;
1062
1063 flags &= ~GFP_SLAB_BUG_MASK;
1064 pr_warn("Unexpected gfp: %#x (%pGg). Fixing up to gfp: %#x (%pGg). Fix your code!\n",
1065 invalid_mask, &invalid_mask, flags, &flags);
1066 dump_stack();
1067
1068 return flags;
1069 }
1070
1071 #ifdef CONFIG_SLAB_FREELIST_RANDOM
1072 /* Randomize a generic freelist */
freelist_randomize(unsigned int * list,unsigned int count)1073 static void freelist_randomize(unsigned int *list,
1074 unsigned int count)
1075 {
1076 unsigned int rand;
1077 unsigned int i;
1078
1079 for (i = 0; i < count; i++)
1080 list[i] = i;
1081
1082 /* Fisher-Yates shuffle */
1083 for (i = count - 1; i > 0; i--) {
1084 rand = get_random_u32_below(i + 1);
1085 swap(list[i], list[rand]);
1086 }
1087 }
1088
1089 /* Create a random sequence per cache */
cache_random_seq_create(struct kmem_cache * cachep,unsigned int count,gfp_t gfp)1090 int cache_random_seq_create(struct kmem_cache *cachep, unsigned int count,
1091 gfp_t gfp)
1092 {
1093
1094 if (count < 2 || cachep->random_seq)
1095 return 0;
1096
1097 cachep->random_seq = kcalloc(count, sizeof(unsigned int), gfp);
1098 if (!cachep->random_seq)
1099 return -ENOMEM;
1100
1101 freelist_randomize(cachep->random_seq, count);
1102 return 0;
1103 }
1104
1105 /* Destroy the per-cache random freelist sequence */
cache_random_seq_destroy(struct kmem_cache * cachep)1106 void cache_random_seq_destroy(struct kmem_cache *cachep)
1107 {
1108 kfree(cachep->random_seq);
1109 cachep->random_seq = NULL;
1110 }
1111 #endif /* CONFIG_SLAB_FREELIST_RANDOM */
1112
1113 #ifdef CONFIG_SLUB_DEBUG
1114 #define SLABINFO_RIGHTS (0400)
1115
print_slabinfo_header(struct seq_file * m)1116 static void print_slabinfo_header(struct seq_file *m)
1117 {
1118 /*
1119 * Output format version, so at least we can change it
1120 * without _too_ many complaints.
1121 */
1122 seq_puts(m, "slabinfo - version: 2.1\n");
1123 seq_puts(m, "# name <active_objs> <num_objs> <objsize> <objperslab> <pagesperslab>");
1124 seq_puts(m, " : tunables <limit> <batchcount> <sharedfactor>");
1125 seq_puts(m, " : slabdata <active_slabs> <num_slabs> <sharedavail>");
1126 seq_putc(m, '\n');
1127 }
1128
slab_start(struct seq_file * m,loff_t * pos)1129 static void *slab_start(struct seq_file *m, loff_t *pos)
1130 {
1131 mutex_lock(&slab_mutex);
1132 return seq_list_start(&slab_caches, *pos);
1133 }
1134
slab_next(struct seq_file * m,void * p,loff_t * pos)1135 static void *slab_next(struct seq_file *m, void *p, loff_t *pos)
1136 {
1137 return seq_list_next(p, &slab_caches, pos);
1138 }
1139
slab_stop(struct seq_file * m,void * p)1140 static void slab_stop(struct seq_file *m, void *p)
1141 {
1142 mutex_unlock(&slab_mutex);
1143 }
1144
cache_show(struct kmem_cache * s,struct seq_file * m)1145 static void cache_show(struct kmem_cache *s, struct seq_file *m)
1146 {
1147 struct slabinfo sinfo;
1148
1149 memset(&sinfo, 0, sizeof(sinfo));
1150 get_slabinfo(s, &sinfo);
1151
1152 seq_printf(m, "%-17s %6lu %6lu %6u %4u %4d",
1153 s->name, sinfo.active_objs, sinfo.num_objs, s->size,
1154 sinfo.objects_per_slab, (1 << sinfo.cache_order));
1155
1156 seq_printf(m, " : tunables %4u %4u %4u",
1157 sinfo.limit, sinfo.batchcount, sinfo.shared);
1158 seq_printf(m, " : slabdata %6lu %6lu %6lu",
1159 sinfo.active_slabs, sinfo.num_slabs, sinfo.shared_avail);
1160 seq_putc(m, '\n');
1161 }
1162
slab_show(struct seq_file * m,void * p)1163 static int slab_show(struct seq_file *m, void *p)
1164 {
1165 struct kmem_cache *s = list_entry(p, struct kmem_cache, list);
1166
1167 if (p == slab_caches.next)
1168 print_slabinfo_header(m);
1169 cache_show(s, m);
1170 return 0;
1171 }
1172
dump_unreclaimable_slab(void)1173 void dump_unreclaimable_slab(void)
1174 {
1175 struct kmem_cache *s;
1176 struct slabinfo sinfo;
1177
1178 /*
1179 * Here acquiring slab_mutex is risky since we don't prefer to get
1180 * sleep in oom path. But, without mutex hold, it may introduce a
1181 * risk of crash.
1182 * Use mutex_trylock to protect the list traverse, dump nothing
1183 * without acquiring the mutex.
1184 */
1185 if (!mutex_trylock(&slab_mutex)) {
1186 pr_warn("excessive unreclaimable slab but cannot dump stats\n");
1187 return;
1188 }
1189
1190 pr_info("Unreclaimable slab info:\n");
1191 pr_info("Name Used Total\n");
1192
1193 list_for_each_entry(s, &slab_caches, list) {
1194 if (s->flags & SLAB_RECLAIM_ACCOUNT)
1195 continue;
1196
1197 get_slabinfo(s, &sinfo);
1198
1199 if (sinfo.num_objs > 0)
1200 pr_info("%-17s %10luKB %10luKB\n", s->name,
1201 (sinfo.active_objs * s->size) / 1024,
1202 (sinfo.num_objs * s->size) / 1024);
1203 }
1204 mutex_unlock(&slab_mutex);
1205 }
1206
1207 /*
1208 * slabinfo_op - iterator that generates /proc/slabinfo
1209 *
1210 * Output layout:
1211 * cache-name
1212 * num-active-objs
1213 * total-objs
1214 * object size
1215 * num-active-slabs
1216 * total-slabs
1217 * num-pages-per-slab
1218 * + further values on SMP and with statistics enabled
1219 */
1220 static const struct seq_operations slabinfo_op = {
1221 .start = slab_start,
1222 .next = slab_next,
1223 .stop = slab_stop,
1224 .show = slab_show,
1225 };
1226
slabinfo_open(struct inode * inode,struct file * file)1227 static int slabinfo_open(struct inode *inode, struct file *file)
1228 {
1229 return seq_open(file, &slabinfo_op);
1230 }
1231
1232 static const struct proc_ops slabinfo_proc_ops = {
1233 .proc_flags = PROC_ENTRY_PERMANENT,
1234 .proc_open = slabinfo_open,
1235 .proc_read = seq_read,
1236 .proc_lseek = seq_lseek,
1237 .proc_release = seq_release,
1238 };
1239
slab_proc_init(void)1240 static int __init slab_proc_init(void)
1241 {
1242 proc_create("slabinfo", SLABINFO_RIGHTS, NULL, &slabinfo_proc_ops);
1243 return 0;
1244 }
1245 module_init(slab_proc_init);
1246
1247 #endif /* CONFIG_SLUB_DEBUG */
1248
1249 /**
1250 * kfree_sensitive - Clear sensitive information in memory before freeing
1251 * @p: object to free memory of
1252 *
1253 * The memory of the object @p points to is zeroed before freed.
1254 * If @p is %NULL, kfree_sensitive() does nothing.
1255 *
1256 * Note: this function zeroes the whole allocated buffer which can be a good
1257 * deal bigger than the requested buffer size passed to kmalloc(). So be
1258 * careful when using this function in performance sensitive code.
1259 */
kfree_sensitive(const void * p)1260 void kfree_sensitive(const void *p)
1261 {
1262 size_t ks;
1263 void *mem = (void *)p;
1264
1265 ks = ksize(mem);
1266 if (ks) {
1267 kasan_unpoison_range(mem, ks);
1268 memzero_explicit(mem, ks);
1269 }
1270 kfree(mem);
1271 }
1272 EXPORT_SYMBOL(kfree_sensitive);
1273
1274 #ifdef CONFIG_BPF_SYSCALL
1275 #include <linux/btf.h>
1276
1277 __bpf_kfunc_start_defs();
1278
bpf_get_kmem_cache(u64 addr)1279 __bpf_kfunc struct kmem_cache *bpf_get_kmem_cache(u64 addr)
1280 {
1281 struct slab *slab;
1282
1283 if (!virt_addr_valid((void *)(long)addr))
1284 return NULL;
1285
1286 slab = virt_to_slab((void *)(long)addr);
1287 return slab ? slab->slab_cache : NULL;
1288 }
1289
1290 __bpf_kfunc_end_defs();
1291 #endif /* CONFIG_BPF_SYSCALL */
1292
1293 /* Tracepoints definitions. */
1294 EXPORT_TRACEPOINT_SYMBOL(kmalloc);
1295 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_alloc);
1296 EXPORT_TRACEPOINT_SYMBOL(kfree);
1297 EXPORT_TRACEPOINT_SYMBOL(kmem_cache_free);
1298
kfree_call_rcu_nolock(struct kvfree_rcu_head * head,void * ptr)1299 void kfree_call_rcu_nolock(struct kvfree_rcu_head *head, void *ptr)
1300 {
1301 struct slab *slab;
1302
1303 if (!IS_ENABLED(CONFIG_KVFREE_RCU_BATCHED))
1304 goto fallback;
1305
1306 if (unlikely(is_vmalloc_addr(ptr)))
1307 goto fallback;
1308
1309 slab = virt_to_slab(ptr);
1310 if (unlikely(!slab))
1311 goto fallback;
1312
1313 if (unlikely(IS_ENABLED(CONFIG_NUMA) && slab_nid(slab) != numa_mem_id()))
1314 goto fallback;
1315
1316 if (unlikely(!__kfree_rcu_sheaf(slab->slab_cache, ptr, SLAB_FREE_NOLOCK)))
1317 goto fallback;
1318
1319 return;
1320
1321 fallback:
1322 defer_kfree_rcu(head);
1323 }
1324 EXPORT_SYMBOL_GPL(kfree_call_rcu_nolock);
1325
1326 #ifndef CONFIG_KVFREE_RCU_BATCHED
1327
kvfree_call_rcu(struct kvfree_rcu_head * head,void * ptr)1328 void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr)
1329 {
1330 if (head) {
1331 kasan_record_aux_stack(ptr);
1332 call_rcu(&head->head, kvfree_rcu_cb);
1333 return;
1334 }
1335
1336 // kvfree_rcu(one_arg) call.
1337 might_sleep();
1338 synchronize_rcu();
1339 kvfree(ptr);
1340 }
1341 EXPORT_SYMBOL_GPL(kvfree_call_rcu);
1342
kvfree_rcu_barrier(void)1343 void kvfree_rcu_barrier(void)
1344 {
1345 deferred_work_barrier();
1346 rcu_barrier();
1347 }
1348
kvfree_rcu_barrier_on_cache(struct kmem_cache * s)1349 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
1350 {
1351 deferred_work_barrier();
1352 rcu_barrier();
1353 }
1354
kvfree_rcu_init(void)1355 void __init kvfree_rcu_init(void)
1356 {
1357 }
1358
1359 #else /* CONFIG_KVFREE_RCU_BATCHED */
1360
1361 /*
1362 * This rcu parameter is runtime-read-only. It reflects
1363 * a minimum allowed number of objects which can be cached
1364 * per-CPU. Object size is equal to one page. This value
1365 * can be changed at boot time.
1366 */
1367 static int rcu_min_cached_objs = 5;
1368 module_param(rcu_min_cached_objs, int, 0444);
1369
1370 // A page shrinker can ask for pages to be freed to make them
1371 // available for other parts of the system. This usually happens
1372 // under low memory conditions, and in that case we should also
1373 // defer page-cache filling for a short time period.
1374 //
1375 // The default value is 5 seconds, which is long enough to reduce
1376 // interference with the shrinker while it asks other systems to
1377 // drain their caches.
1378 static int rcu_delay_page_cache_fill_msec = 5000;
1379 module_param(rcu_delay_page_cache_fill_msec, int, 0444);
1380
1381 static struct workqueue_struct *rcu_reclaim_wq;
1382
1383 /* Maximum number of jiffies to wait before draining a batch. */
1384 #define KFREE_DRAIN_JIFFIES (5 * HZ)
1385 #define KFREE_N_BATCHES 2
1386 #define FREE_N_CHANNELS 2
1387
1388 /**
1389 * struct kvfree_rcu_bulk_data - single block to store kvfree_rcu() pointers
1390 * @list: List node. All blocks are linked between each other
1391 * @gp_snap: Snapshot of RCU state for objects placed to this bulk
1392 * @nr_records: Number of active pointers in the array
1393 * @records: Array of the kvfree_rcu() pointers
1394 */
1395 struct kvfree_rcu_bulk_data {
1396 struct list_head list;
1397 struct rcu_gp_seq gp_snap;
1398 unsigned long nr_records;
1399 void *records[] __counted_by(nr_records);
1400 };
1401
1402 /*
1403 * This macro defines how many entries the "records" array
1404 * will contain. It is based on the fact that the size of
1405 * kvfree_rcu_bulk_data structure becomes exactly one page.
1406 */
1407 #define KVFREE_BULK_MAX_ENTR \
1408 ((PAGE_SIZE - sizeof(struct kvfree_rcu_bulk_data)) / sizeof(void *))
1409
1410 /**
1411 * struct kfree_rcu_cpu_work - single batch of kfree_rcu() requests
1412 * @rcu_work: Let queue_rcu_work() invoke workqueue handler after grace period
1413 * @head_free: List of kfree_rcu() objects waiting for a grace period
1414 * @head_free_gp_snap: Grace-period snapshot to check for attempted premature frees.
1415 * @bulk_head_free: Bulk-List of kvfree_rcu() objects waiting for a grace period
1416 * @krcp: Pointer to @kfree_rcu_cpu structure
1417 */
1418
1419 struct kfree_rcu_cpu_work {
1420 struct rcu_work rcu_work;
1421 struct kvfree_rcu_head *head_free;
1422 struct rcu_gp_seq head_free_gp_snap;
1423 struct list_head bulk_head_free[FREE_N_CHANNELS];
1424 struct kfree_rcu_cpu *krcp;
1425 };
1426
1427 /**
1428 * struct kfree_rcu_cpu - batch up kfree_rcu() requests for RCU grace period
1429 * @head: List of kfree_rcu() objects not yet waiting for a grace period
1430 * @head_gp_snap: Snapshot of RCU state for objects placed to "@head"
1431 * @bulk_head: Bulk-List of kvfree_rcu() objects not yet waiting for a grace period
1432 * @krw_arr: Array of batches of kfree_rcu() objects waiting for a grace period
1433 * @lock: Synchronize access to this structure
1434 * @monitor_work: Promote @head to @head_free after KFREE_DRAIN_JIFFIES
1435 * @initialized: The @rcu_work fields have been initialized
1436 * @head_count: Number of objects in rcu_head singular list
1437 * @bulk_count: Number of objects in bulk-list
1438 * @bkvcache:
1439 * A simple cache list that contains objects for reuse purpose.
1440 * In order to save some per-cpu space the list is singular.
1441 * Even though it is lockless an access has to be protected by the
1442 * per-cpu lock.
1443 * @page_cache_work: A work to refill the cache when it is empty
1444 * @backoff_page_cache_fill: Delay cache refills
1445 * @work_in_progress: Indicates that page_cache_work is running
1446 * @hrtimer: A hrtimer for scheduling a page_cache_work
1447 * @nr_bkv_objs: number of allocated objects at @bkvcache.
1448 *
1449 * This is a per-CPU structure. The reason that it is not included in
1450 * the rcu_data structure is to permit this code to be extracted from
1451 * the RCU files. Such extraction could allow further optimization of
1452 * the interactions with the slab allocators.
1453 */
1454 struct kfree_rcu_cpu {
1455 // Objects queued on a linked list
1456 // through their rcu_head structures.
1457 struct kvfree_rcu_head *head;
1458 unsigned long head_gp_snap;
1459 atomic_t head_count;
1460
1461 // Objects queued on a bulk-list.
1462 struct list_head bulk_head[FREE_N_CHANNELS];
1463 atomic_t bulk_count[FREE_N_CHANNELS];
1464
1465 struct kfree_rcu_cpu_work krw_arr[KFREE_N_BATCHES];
1466 raw_spinlock_t lock;
1467 struct delayed_work monitor_work;
1468 bool initialized;
1469
1470 struct delayed_work page_cache_work;
1471 atomic_t backoff_page_cache_fill;
1472 atomic_t work_in_progress;
1473 struct hrtimer hrtimer;
1474
1475 struct llist_head bkvcache;
1476 int nr_bkv_objs;
1477 };
1478
1479 static DEFINE_PER_CPU(struct kfree_rcu_cpu, krc) = {
1480 .lock = __RAW_SPIN_LOCK_UNLOCKED(krc.lock),
1481 };
1482
1483 static __always_inline void
debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data * bhead)1484 debug_rcu_bhead_unqueue(struct kvfree_rcu_bulk_data *bhead)
1485 {
1486 #ifdef CONFIG_DEBUG_OBJECTS_RCU_HEAD
1487 int i;
1488
1489 for (i = 0; i < bhead->nr_records; i++)
1490 debug_rcu_head_unqueue((struct rcu_head *)(bhead->records[i]));
1491 #endif
1492 }
1493
1494 static inline struct kfree_rcu_cpu *
krc_this_cpu_lock(unsigned long * flags)1495 krc_this_cpu_lock(unsigned long *flags)
1496 {
1497 struct kfree_rcu_cpu *krcp;
1498
1499 local_irq_save(*flags); // For safely calling this_cpu_ptr().
1500 krcp = this_cpu_ptr(&krc);
1501 raw_spin_lock(&krcp->lock);
1502
1503 return krcp;
1504 }
1505
1506 static inline void
krc_this_cpu_unlock(struct kfree_rcu_cpu * krcp,unsigned long flags)1507 krc_this_cpu_unlock(struct kfree_rcu_cpu *krcp, unsigned long flags)
1508 {
1509 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1510 }
1511
1512 static inline struct kvfree_rcu_bulk_data *
get_cached_bnode(struct kfree_rcu_cpu * krcp)1513 get_cached_bnode(struct kfree_rcu_cpu *krcp)
1514 {
1515 if (!krcp->nr_bkv_objs)
1516 return NULL;
1517
1518 WRITE_ONCE(krcp->nr_bkv_objs, krcp->nr_bkv_objs - 1);
1519 return (struct kvfree_rcu_bulk_data *)
1520 llist_del_first(&krcp->bkvcache);
1521 }
1522
1523 static inline bool
put_cached_bnode(struct kfree_rcu_cpu * krcp,struct kvfree_rcu_bulk_data * bnode)1524 put_cached_bnode(struct kfree_rcu_cpu *krcp,
1525 struct kvfree_rcu_bulk_data *bnode)
1526 {
1527 // Check the limit.
1528 if (krcp->nr_bkv_objs >= rcu_min_cached_objs)
1529 return false;
1530
1531 llist_add((struct llist_node *) bnode, &krcp->bkvcache);
1532 WRITE_ONCE(krcp->nr_bkv_objs, krcp->nr_bkv_objs + 1);
1533 return true;
1534 }
1535
1536 static int
drain_page_cache(struct kfree_rcu_cpu * krcp)1537 drain_page_cache(struct kfree_rcu_cpu *krcp)
1538 {
1539 unsigned long flags;
1540 struct llist_node *page_list, *pos, *n;
1541 int freed = 0;
1542
1543 if (!rcu_min_cached_objs)
1544 return 0;
1545
1546 raw_spin_lock_irqsave(&krcp->lock, flags);
1547 page_list = llist_del_all(&krcp->bkvcache);
1548 WRITE_ONCE(krcp->nr_bkv_objs, 0);
1549 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1550
1551 llist_for_each_safe(pos, n, page_list) {
1552 free_page((unsigned long)pos);
1553 freed++;
1554 }
1555
1556 return freed;
1557 }
1558
1559 static void
kvfree_rcu_bulk(struct kfree_rcu_cpu * krcp,struct kvfree_rcu_bulk_data * bnode,int idx)1560 kvfree_rcu_bulk(struct kfree_rcu_cpu *krcp,
1561 struct kvfree_rcu_bulk_data *bnode, int idx)
1562 {
1563 unsigned long flags;
1564 int i;
1565
1566 if (!WARN_ON_ONCE(!poll_state_synchronize_rcu_full(&bnode->gp_snap))) {
1567 debug_rcu_bhead_unqueue(bnode);
1568 rcu_lock_acquire(&rcu_callback_map);
1569 if (idx == 0) { // kmalloc() / kfree().
1570 trace_rcu_invoke_kfree_bulk_callback(
1571 "slab", bnode->nr_records,
1572 bnode->records);
1573
1574 kfree_bulk(bnode->nr_records, bnode->records);
1575 } else { // vmalloc() / vfree().
1576 for (i = 0; i < bnode->nr_records; i++) {
1577 trace_rcu_invoke_kvfree_callback(
1578 "slab", bnode->records[i], 0);
1579
1580 vfree(bnode->records[i]);
1581 }
1582 }
1583 rcu_lock_release(&rcu_callback_map);
1584 }
1585
1586 raw_spin_lock_irqsave(&krcp->lock, flags);
1587 if (put_cached_bnode(krcp, bnode))
1588 bnode = NULL;
1589 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1590
1591 if (bnode)
1592 free_page((unsigned long) bnode);
1593
1594 cond_resched_tasks_rcu_qs();
1595 }
1596
1597 static void
kvfree_rcu_list(struct kvfree_rcu_head * head)1598 kvfree_rcu_list(struct kvfree_rcu_head *head)
1599 {
1600 struct kvfree_rcu_head *next;
1601
1602 for (; head; head = next) {
1603 void *ptr = kvmalloc_obj_start_addr(head);
1604 unsigned long offset = (void *) head - ptr;
1605
1606 next = head->next;
1607 debug_rcu_head_unqueue((struct rcu_head *)ptr);
1608 rcu_lock_acquire(&rcu_callback_map);
1609 trace_rcu_invoke_kvfree_callback("slab", head, offset);
1610
1611 kvfree(ptr);
1612
1613 rcu_lock_release(&rcu_callback_map);
1614 cond_resched_tasks_rcu_qs();
1615 }
1616 }
1617
1618 /*
1619 * This function is invoked in workqueue context after a grace period.
1620 * It frees all the objects queued on ->bulk_head_free or ->head_free.
1621 */
kfree_rcu_work(struct work_struct * work)1622 static void kfree_rcu_work(struct work_struct *work)
1623 {
1624 unsigned long flags;
1625 struct kvfree_rcu_bulk_data *bnode, *n;
1626 struct list_head bulk_head[FREE_N_CHANNELS];
1627 struct kvfree_rcu_head *head;
1628 struct kfree_rcu_cpu *krcp;
1629 struct kfree_rcu_cpu_work *krwp;
1630 struct rcu_gp_seq head_gp_snap;
1631 int i;
1632
1633 krwp = container_of(to_rcu_work(work),
1634 struct kfree_rcu_cpu_work, rcu_work);
1635 krcp = krwp->krcp;
1636
1637 raw_spin_lock_irqsave(&krcp->lock, flags);
1638 // Channels 1 and 2.
1639 for (i = 0; i < FREE_N_CHANNELS; i++)
1640 list_replace_init(&krwp->bulk_head_free[i], &bulk_head[i]);
1641
1642 // Channel 3.
1643 head = krwp->head_free;
1644 krwp->head_free = NULL;
1645 head_gp_snap = krwp->head_free_gp_snap;
1646 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1647
1648 // Handle the first two channels.
1649 for (i = 0; i < FREE_N_CHANNELS; i++) {
1650 // Start from the tail page, so a GP is likely passed for it.
1651 list_for_each_entry_safe(bnode, n, &bulk_head[i], list)
1652 kvfree_rcu_bulk(krcp, bnode, i);
1653 }
1654
1655 /*
1656 * This is used when the "bulk" path can not be used for the
1657 * double-argument of kvfree_rcu(). This happens when the
1658 * page-cache is empty, which means that objects are instead
1659 * queued on a linked list through their rcu_head structures.
1660 * This list is named "Channel 3".
1661 */
1662 if (head && !WARN_ON_ONCE(!poll_state_synchronize_rcu_full(&head_gp_snap)))
1663 kvfree_rcu_list(head);
1664 }
1665
kfree_rcu_sheaf(void * obj)1666 static bool kfree_rcu_sheaf(void *obj)
1667 {
1668 struct kmem_cache *s;
1669 struct slab *slab;
1670 unsigned int free_flags = SLAB_FREE_DEFAULT;
1671
1672 /*
1673 * It is not safe to spin on PREEMPT_RT because the kernel might be
1674 * holding a raw spinlock and slab acquires sleeping locks.
1675 */
1676 if (IS_ENABLED(CONFIG_PREEMPT_RT))
1677 free_flags = SLAB_FREE_NOLOCK;
1678
1679 if (is_vmalloc_addr(obj))
1680 return false;
1681
1682 slab = virt_to_slab(obj);
1683 if (unlikely(!slab))
1684 return false;
1685
1686 s = slab->slab_cache;
1687 if (likely(!IS_ENABLED(CONFIG_NUMA) || slab_nid(slab) == numa_mem_id()))
1688 return __kfree_rcu_sheaf(s, obj, free_flags);
1689
1690 return false;
1691 }
1692
1693 static bool
need_offload_krc(struct kfree_rcu_cpu * krcp)1694 need_offload_krc(struct kfree_rcu_cpu *krcp)
1695 {
1696 int i;
1697
1698 for (i = 0; i < FREE_N_CHANNELS; i++)
1699 if (!list_empty(&krcp->bulk_head[i]))
1700 return true;
1701
1702 return !!READ_ONCE(krcp->head);
1703 }
1704
1705 static bool
need_wait_for_krwp_work(struct kfree_rcu_cpu_work * krwp)1706 need_wait_for_krwp_work(struct kfree_rcu_cpu_work *krwp)
1707 {
1708 int i;
1709
1710 for (i = 0; i < FREE_N_CHANNELS; i++)
1711 if (!list_empty(&krwp->bulk_head_free[i]))
1712 return true;
1713
1714 return !!krwp->head_free;
1715 }
1716
krc_count(struct kfree_rcu_cpu * krcp)1717 static int krc_count(struct kfree_rcu_cpu *krcp)
1718 {
1719 int sum = atomic_read(&krcp->head_count);
1720 int i;
1721
1722 for (i = 0; i < FREE_N_CHANNELS; i++)
1723 sum += atomic_read(&krcp->bulk_count[i]);
1724
1725 return sum;
1726 }
1727
1728 static void
__schedule_delayed_monitor_work(struct kfree_rcu_cpu * krcp)1729 __schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)
1730 {
1731 long delay, delay_left;
1732
1733 delay = krc_count(krcp) >= KVFREE_BULK_MAX_ENTR ? 1:KFREE_DRAIN_JIFFIES;
1734 if (delayed_work_pending(&krcp->monitor_work)) {
1735 delay_left = krcp->monitor_work.timer.expires - jiffies;
1736 if (delay < delay_left)
1737 mod_delayed_work(rcu_reclaim_wq, &krcp->monitor_work, delay);
1738 return;
1739 }
1740 queue_delayed_work(rcu_reclaim_wq, &krcp->monitor_work, delay);
1741 }
1742
1743 static void
schedule_delayed_monitor_work(struct kfree_rcu_cpu * krcp)1744 schedule_delayed_monitor_work(struct kfree_rcu_cpu *krcp)
1745 {
1746 unsigned long flags;
1747
1748 raw_spin_lock_irqsave(&krcp->lock, flags);
1749 __schedule_delayed_monitor_work(krcp);
1750 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1751 }
1752
1753 static void
kvfree_rcu_drain_ready(struct kfree_rcu_cpu * krcp)1754 kvfree_rcu_drain_ready(struct kfree_rcu_cpu *krcp)
1755 {
1756 struct list_head bulk_ready[FREE_N_CHANNELS];
1757 struct kvfree_rcu_bulk_data *bnode, *n;
1758 struct kvfree_rcu_head *head_ready = NULL;
1759 unsigned long flags;
1760 int i;
1761
1762 raw_spin_lock_irqsave(&krcp->lock, flags);
1763 for (i = 0; i < FREE_N_CHANNELS; i++) {
1764 INIT_LIST_HEAD(&bulk_ready[i]);
1765
1766 list_for_each_entry_safe_reverse(bnode, n, &krcp->bulk_head[i], list) {
1767 if (!poll_state_synchronize_rcu_full(&bnode->gp_snap))
1768 break;
1769
1770 atomic_sub(bnode->nr_records, &krcp->bulk_count[i]);
1771 list_move(&bnode->list, &bulk_ready[i]);
1772 }
1773 }
1774
1775 if (krcp->head && poll_state_synchronize_rcu(krcp->head_gp_snap)) {
1776 head_ready = krcp->head;
1777 atomic_set(&krcp->head_count, 0);
1778 WRITE_ONCE(krcp->head, NULL);
1779 }
1780 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1781
1782 for (i = 0; i < FREE_N_CHANNELS; i++) {
1783 list_for_each_entry_safe(bnode, n, &bulk_ready[i], list)
1784 kvfree_rcu_bulk(krcp, bnode, i);
1785 }
1786
1787 if (head_ready)
1788 kvfree_rcu_list(head_ready);
1789 }
1790
1791 /*
1792 * Return: %true if a work is queued, %false otherwise.
1793 */
1794 static bool
kvfree_rcu_queue_batch(struct kfree_rcu_cpu * krcp)1795 kvfree_rcu_queue_batch(struct kfree_rcu_cpu *krcp)
1796 {
1797 unsigned long flags;
1798 bool queued = false;
1799 int i, j;
1800
1801 raw_spin_lock_irqsave(&krcp->lock, flags);
1802
1803 // Attempt to start a new batch.
1804 for (i = 0; i < KFREE_N_BATCHES; i++) {
1805 struct kfree_rcu_cpu_work *krwp = &(krcp->krw_arr[i]);
1806
1807 // Try to detach bulk_head or head and attach it, only when
1808 // all channels are free. Any channel is not free means at krwp
1809 // there is on-going rcu work to handle krwp's free business.
1810 if (need_wait_for_krwp_work(krwp))
1811 continue;
1812
1813 // kvfree_rcu_drain_ready() might handle this krcp, if so give up.
1814 if (need_offload_krc(krcp)) {
1815 // Channel 1 corresponds to the SLAB-pointer bulk path.
1816 // Channel 2 corresponds to vmalloc-pointer bulk path.
1817 for (j = 0; j < FREE_N_CHANNELS; j++) {
1818 if (list_empty(&krwp->bulk_head_free[j])) {
1819 atomic_set(&krcp->bulk_count[j], 0);
1820 list_replace_init(&krcp->bulk_head[j],
1821 &krwp->bulk_head_free[j]);
1822 }
1823 }
1824
1825 // Channel 3 corresponds to both SLAB and vmalloc
1826 // objects queued on the linked list.
1827 if (!krwp->head_free) {
1828 krwp->head_free = krcp->head;
1829 get_state_synchronize_rcu_full(&krwp->head_free_gp_snap);
1830 atomic_set(&krcp->head_count, 0);
1831 WRITE_ONCE(krcp->head, NULL);
1832 }
1833
1834 // One work is per one batch, so there are three
1835 // "free channels", the batch can handle. Break
1836 // the loop since it is done with this CPU thus
1837 // queuing an RCU work is _always_ success here.
1838 queued = queue_rcu_work(rcu_reclaim_wq, &krwp->rcu_work);
1839 WARN_ON_ONCE(!queued);
1840 break;
1841 }
1842 }
1843
1844 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1845 return queued;
1846 }
1847
1848 /*
1849 * This function is invoked after the KFREE_DRAIN_JIFFIES timeout.
1850 */
kfree_rcu_monitor(struct work_struct * work)1851 static void kfree_rcu_monitor(struct work_struct *work)
1852 {
1853 struct kfree_rcu_cpu *krcp = container_of(work,
1854 struct kfree_rcu_cpu, monitor_work.work);
1855
1856 // Drain ready for reclaim.
1857 kvfree_rcu_drain_ready(krcp);
1858
1859 // Queue a batch for a rest.
1860 kvfree_rcu_queue_batch(krcp);
1861
1862 // If there is nothing to detach, it means that our job is
1863 // successfully done here. In case of having at least one
1864 // of the channels that is still busy we should rearm the
1865 // work to repeat an attempt. Because previous batches are
1866 // still in progress.
1867 if (need_offload_krc(krcp))
1868 schedule_delayed_monitor_work(krcp);
1869 }
1870
fill_page_cache_func(struct work_struct * work)1871 static void fill_page_cache_func(struct work_struct *work)
1872 {
1873 struct kvfree_rcu_bulk_data *bnode;
1874 struct kfree_rcu_cpu *krcp =
1875 container_of(work, struct kfree_rcu_cpu,
1876 page_cache_work.work);
1877 unsigned long flags;
1878 int nr_pages;
1879 bool pushed;
1880 int i;
1881
1882 nr_pages = atomic_read(&krcp->backoff_page_cache_fill) ?
1883 1 : rcu_min_cached_objs;
1884
1885 for (i = READ_ONCE(krcp->nr_bkv_objs); i < nr_pages; i++) {
1886 bnode = (struct kvfree_rcu_bulk_data *)
1887 __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1888
1889 if (!bnode)
1890 break;
1891
1892 raw_spin_lock_irqsave(&krcp->lock, flags);
1893 pushed = put_cached_bnode(krcp, bnode);
1894 raw_spin_unlock_irqrestore(&krcp->lock, flags);
1895
1896 if (!pushed) {
1897 free_page((unsigned long) bnode);
1898 break;
1899 }
1900 }
1901
1902 atomic_set(&krcp->work_in_progress, 0);
1903 atomic_set(&krcp->backoff_page_cache_fill, 0);
1904 }
1905
1906 // Record ptr in a page managed by krcp, with the pre-krc_this_cpu_lock()
1907 // state specified by flags. If can_alloc is true, the caller must
1908 // be schedulable and not be holding any locks or mutexes that might be
1909 // acquired by the memory allocator or anything that it might invoke.
1910 // Returns true if ptr was successfully recorded, else the caller must
1911 // use a fallback.
1912 static inline bool
add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu ** krcp,unsigned long * flags,void * ptr,bool can_alloc)1913 add_ptr_to_bulk_krc_lock(struct kfree_rcu_cpu **krcp,
1914 unsigned long *flags, void *ptr, bool can_alloc)
1915 {
1916 struct kvfree_rcu_bulk_data *bnode;
1917 int idx;
1918
1919 *krcp = krc_this_cpu_lock(flags);
1920 if (unlikely(!(*krcp)->initialized))
1921 return false;
1922
1923 idx = !!is_vmalloc_addr(ptr);
1924 bnode = list_first_entry_or_null(&(*krcp)->bulk_head[idx],
1925 struct kvfree_rcu_bulk_data, list);
1926
1927 /* Check if a new block is required. */
1928 if (!bnode || bnode->nr_records == KVFREE_BULK_MAX_ENTR) {
1929 bnode = get_cached_bnode(*krcp);
1930 if (!bnode && can_alloc) {
1931 krc_this_cpu_unlock(*krcp, *flags);
1932
1933 // __GFP_NORETRY - allows a light-weight direct reclaim
1934 // what is OK from minimizing of fallback hitting point of
1935 // view. Apart of that it forbids any OOM invoking what is
1936 // also beneficial since we are about to release memory soon.
1937 //
1938 // __GFP_NOMEMALLOC - prevents from consuming of all the
1939 // memory reserves. Please note we have a fallback path.
1940 //
1941 // __GFP_NOWARN - it is supposed that an allocation can
1942 // be failed under low memory or high memory pressure
1943 // scenarios.
1944 bnode = (struct kvfree_rcu_bulk_data *)
1945 __get_free_page(GFP_KERNEL | __GFP_NORETRY | __GFP_NOMEMALLOC | __GFP_NOWARN);
1946 raw_spin_lock_irqsave(&(*krcp)->lock, *flags);
1947 }
1948
1949 if (!bnode)
1950 return false;
1951
1952 // Initialize the new block and attach it.
1953 bnode->nr_records = 0;
1954 list_add(&bnode->list, &(*krcp)->bulk_head[idx]);
1955 }
1956
1957 // Finally insert and update the GP for this page.
1958 bnode->nr_records++;
1959 bnode->records[bnode->nr_records - 1] = ptr;
1960 get_state_synchronize_rcu_full(&bnode->gp_snap);
1961 atomic_inc(&(*krcp)->bulk_count[idx]);
1962
1963 return true;
1964 }
1965
1966 static enum hrtimer_restart
schedule_page_work_fn(struct hrtimer * t)1967 schedule_page_work_fn(struct hrtimer *t)
1968 {
1969 struct kfree_rcu_cpu *krcp =
1970 container_of(t, struct kfree_rcu_cpu, hrtimer);
1971
1972 queue_delayed_work(system_highpri_wq, &krcp->page_cache_work, 0);
1973 return HRTIMER_NORESTART;
1974 }
1975
1976 static void
run_page_cache_worker(struct kfree_rcu_cpu * krcp)1977 run_page_cache_worker(struct kfree_rcu_cpu *krcp)
1978 {
1979 // If cache disabled, bail out.
1980 if (!rcu_min_cached_objs)
1981 return;
1982
1983 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING &&
1984 !atomic_xchg(&krcp->work_in_progress, 1)) {
1985 if (atomic_read(&krcp->backoff_page_cache_fill)) {
1986 queue_delayed_work(rcu_reclaim_wq,
1987 &krcp->page_cache_work,
1988 msecs_to_jiffies(rcu_delay_page_cache_fill_msec));
1989 } else {
1990 hrtimer_setup(&krcp->hrtimer, schedule_page_work_fn, CLOCK_MONOTONIC,
1991 HRTIMER_MODE_REL);
1992 hrtimer_start(&krcp->hrtimer, 0, HRTIMER_MODE_REL);
1993 }
1994 }
1995 }
1996
kfree_rcu_scheduler_running(void)1997 void __init kfree_rcu_scheduler_running(void)
1998 {
1999 int cpu;
2000
2001 for_each_possible_cpu(cpu) {
2002 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
2003
2004 if (need_offload_krc(krcp))
2005 schedule_delayed_monitor_work(krcp);
2006 }
2007 }
2008
2009 /*
2010 * Queue a request for lazy invocation of the appropriate free routine
2011 * after a grace period. Please note that three paths are maintained,
2012 * two for the common case using arrays of pointers and a third one that
2013 * is used only when the main paths cannot be used, for example, due to
2014 * memory pressure.
2015 *
2016 * Each kvfree_call_rcu() request is added to a batch. The batch will be drained
2017 * every KFREE_DRAIN_JIFFIES number of jiffies. All the objects in the batch will
2018 * be free'd in workqueue context. This allows us to: batch requests together to
2019 * reduce the number of grace periods during heavy kfree_rcu()/kvfree_rcu() load.
2020 */
kvfree_call_rcu(struct kvfree_rcu_head * head,void * ptr)2021 void kvfree_call_rcu(struct kvfree_rcu_head *head, void *ptr)
2022 {
2023 unsigned long flags;
2024 struct kfree_rcu_cpu *krcp;
2025 bool success;
2026
2027 /*
2028 * Please note there is a limitation for the head-less
2029 * variant, that is why there is a clear rule for such
2030 * objects: it can be used from might_sleep() context
2031 * only. For other places please embed an rcu_head to
2032 * your data.
2033 */
2034 if (!head)
2035 might_sleep();
2036
2037 if (kfree_rcu_sheaf(ptr))
2038 return;
2039
2040 // Queue the object but don't yet schedule the batch.
2041 if (debug_rcu_head_queue(ptr)) {
2042 // Probable double kfree_rcu(), just leak.
2043 WARN_ONCE(1, "%s(): Double-freed call. rcu_head %p\n",
2044 __func__, head);
2045
2046 // Mark as success and leave.
2047 return;
2048 }
2049
2050 kasan_record_aux_stack(ptr);
2051 success = add_ptr_to_bulk_krc_lock(&krcp, &flags, ptr, !head);
2052 if (!success) {
2053 run_page_cache_worker(krcp);
2054
2055 if (head == NULL)
2056 // Inline if kvfree_rcu(one_arg) call.
2057 goto unlock_return;
2058
2059 head->next = krcp->head;
2060 WRITE_ONCE(krcp->head, head);
2061 atomic_inc(&krcp->head_count);
2062
2063 // Take a snapshot for this krcp.
2064 krcp->head_gp_snap = get_state_synchronize_rcu();
2065 success = true;
2066 }
2067
2068 /*
2069 * The kvfree_rcu() caller considers the pointer freed at this point
2070 * and likely removes any references to it. Since the actual slab
2071 * freeing (and kmemleak_free()) is deferred, tell kmemleak to ignore
2072 * this object (no scanning or false positives reporting).
2073 */
2074 kmemleak_ignore(ptr);
2075
2076 // Set timer to drain after KFREE_DRAIN_JIFFIES.
2077 if (rcu_scheduler_active == RCU_SCHEDULER_RUNNING)
2078 __schedule_delayed_monitor_work(krcp);
2079
2080 unlock_return:
2081 krc_this_cpu_unlock(krcp, flags);
2082
2083 /*
2084 * Inline kvfree() after synchronize_rcu(). We can do
2085 * it from might_sleep() context only, so the current
2086 * CPU can pass the QS state.
2087 */
2088 if (!success) {
2089 debug_rcu_head_unqueue((struct rcu_head *) ptr);
2090 synchronize_rcu();
2091 kvfree(ptr);
2092 }
2093 }
2094 EXPORT_SYMBOL_GPL(kvfree_call_rcu);
2095
__kvfree_rcu_barrier(void)2096 static inline void __kvfree_rcu_barrier(void)
2097 {
2098 struct kfree_rcu_cpu_work *krwp;
2099 struct kfree_rcu_cpu *krcp;
2100 bool queued;
2101 int i, cpu;
2102
2103 /*
2104 * Firstly we detach objects and queue them over an RCU-batch
2105 * for all CPUs. Finally queued works are flushed for each CPU.
2106 *
2107 * Please note. If there are outstanding batches for a particular
2108 * CPU, those have to be finished first following by queuing a new.
2109 */
2110 for_each_possible_cpu(cpu) {
2111 krcp = per_cpu_ptr(&krc, cpu);
2112
2113 /*
2114 * Check if this CPU has any objects which have been queued for a
2115 * new GP completion. If not(means nothing to detach), we are done
2116 * with it. If any batch is pending/running for this "krcp", below
2117 * per-cpu flush_rcu_work() waits its completion(see last step).
2118 */
2119 if (!need_offload_krc(krcp))
2120 continue;
2121
2122 while (1) {
2123 /*
2124 * If we are not able to queue a new RCU work it means:
2125 * - batches for this CPU are still in flight which should
2126 * be flushed first and then repeat;
2127 * - no objects to detach, because of concurrency.
2128 */
2129 queued = kvfree_rcu_queue_batch(krcp);
2130
2131 /*
2132 * Bail out, if there is no need to offload this "krcp"
2133 * anymore. As noted earlier it can run concurrently.
2134 */
2135 if (queued || !need_offload_krc(krcp))
2136 break;
2137
2138 /* There are ongoing batches. */
2139 for (i = 0; i < KFREE_N_BATCHES; i++) {
2140 krwp = &(krcp->krw_arr[i]);
2141 flush_rcu_work(&krwp->rcu_work);
2142 }
2143 }
2144 }
2145
2146 /*
2147 * Now we guarantee that all objects are flushed.
2148 */
2149 for_each_possible_cpu(cpu) {
2150 krcp = per_cpu_ptr(&krc, cpu);
2151
2152 /*
2153 * A monitor work can drain ready to reclaim objects
2154 * directly. Wait its completion if running or pending.
2155 */
2156 cancel_delayed_work_sync(&krcp->monitor_work);
2157
2158 for (i = 0; i < KFREE_N_BATCHES; i++) {
2159 krwp = &(krcp->krw_arr[i]);
2160 flush_rcu_work(&krwp->rcu_work);
2161 }
2162 }
2163 }
2164
2165 /**
2166 * kvfree_rcu_barrier - Wait until all in-flight kvfree_rcu() complete.
2167 *
2168 * Note that a single argument of kvfree_rcu() call has a slow path that
2169 * triggers synchronize_rcu() following by freeing a pointer. It is done
2170 * before the return from the function. Therefore for any single-argument
2171 * call that will result in a kfree() to a cache that is to be destroyed
2172 * during module exit, it is developer's responsibility to ensure that all
2173 * such calls have returned before the call to kmem_cache_destroy().
2174 */
kvfree_rcu_barrier(void)2175 void kvfree_rcu_barrier(void)
2176 {
2177 flush_all_rcu_sheaves();
2178 __kvfree_rcu_barrier();
2179 }
2180
2181 /**
2182 * kvfree_rcu_barrier_on_cache - Wait for in-flight kvfree_rcu() calls on a
2183 * specific slab cache.
2184 * @s: slab cache to wait for
2185 *
2186 * See the description of kvfree_rcu_barrier() for details.
2187 */
kvfree_rcu_barrier_on_cache(struct kmem_cache * s)2188 void kvfree_rcu_barrier_on_cache(struct kmem_cache *s)
2189 {
2190 /* kfree_rcu_nolock() might have deferred frees even without sheaves */
2191 deferred_work_barrier();
2192
2193 if (cache_has_sheaves(s)) {
2194 cpus_read_lock();
2195 flush_rcu_sheaves_on_cache(s);
2196 cpus_read_unlock();
2197 }
2198
2199 rcu_barrier();
2200 __kvfree_rcu_barrier();
2201 }
2202
2203 static unsigned long
kfree_rcu_shrink_count(struct shrinker * shrink,struct shrink_control * sc)2204 kfree_rcu_shrink_count(struct shrinker *shrink, struct shrink_control *sc)
2205 {
2206 int cpu;
2207 unsigned long count = 0;
2208
2209 /* Snapshot count of all CPUs */
2210 for_each_possible_cpu(cpu) {
2211 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
2212
2213 count += krc_count(krcp);
2214 count += READ_ONCE(krcp->nr_bkv_objs);
2215 atomic_set(&krcp->backoff_page_cache_fill, 1);
2216 }
2217
2218 return count == 0 ? SHRINK_EMPTY : count;
2219 }
2220
2221 static unsigned long
kfree_rcu_shrink_scan(struct shrinker * shrink,struct shrink_control * sc)2222 kfree_rcu_shrink_scan(struct shrinker *shrink, struct shrink_control *sc)
2223 {
2224 int cpu, freed = 0;
2225
2226 for_each_possible_cpu(cpu) {
2227 int count;
2228 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
2229
2230 count = krc_count(krcp);
2231 count += drain_page_cache(krcp);
2232 kfree_rcu_monitor(&krcp->monitor_work.work);
2233
2234 sc->nr_to_scan -= count;
2235 freed += count;
2236
2237 if (sc->nr_to_scan <= 0)
2238 break;
2239 }
2240
2241 return freed == 0 ? SHRINK_STOP : freed;
2242 }
2243
kvfree_rcu_init(void)2244 void __init kvfree_rcu_init(void)
2245 {
2246 int cpu;
2247 int i, j;
2248 struct shrinker *kfree_rcu_shrinker;
2249
2250 rcu_reclaim_wq = alloc_workqueue("kvfree_rcu_reclaim",
2251 WQ_UNBOUND | WQ_MEM_RECLAIM, 0);
2252 WARN_ON(!rcu_reclaim_wq);
2253
2254 /* Clamp it to [0:100] seconds interval. */
2255 if (rcu_delay_page_cache_fill_msec < 0 ||
2256 rcu_delay_page_cache_fill_msec > 100 * MSEC_PER_SEC) {
2257
2258 rcu_delay_page_cache_fill_msec =
2259 clamp(rcu_delay_page_cache_fill_msec, 0,
2260 (int) (100 * MSEC_PER_SEC));
2261
2262 pr_info("Adjusting rcutree.rcu_delay_page_cache_fill_msec to %d ms.\n",
2263 rcu_delay_page_cache_fill_msec);
2264 }
2265
2266 for_each_possible_cpu(cpu) {
2267 struct kfree_rcu_cpu *krcp = per_cpu_ptr(&krc, cpu);
2268
2269 for (i = 0; i < KFREE_N_BATCHES; i++) {
2270 INIT_RCU_WORK(&krcp->krw_arr[i].rcu_work, kfree_rcu_work);
2271 krcp->krw_arr[i].krcp = krcp;
2272
2273 for (j = 0; j < FREE_N_CHANNELS; j++)
2274 INIT_LIST_HEAD(&krcp->krw_arr[i].bulk_head_free[j]);
2275 }
2276
2277 for (i = 0; i < FREE_N_CHANNELS; i++)
2278 INIT_LIST_HEAD(&krcp->bulk_head[i]);
2279
2280 INIT_DELAYED_WORK(&krcp->monitor_work, kfree_rcu_monitor);
2281 INIT_DELAYED_WORK(&krcp->page_cache_work, fill_page_cache_func);
2282 krcp->initialized = true;
2283 }
2284
2285 kfree_rcu_shrinker = shrinker_alloc(0, "slab-kvfree-rcu");
2286 if (!kfree_rcu_shrinker) {
2287 pr_err("Failed to allocate kfree_rcu() shrinker!\n");
2288 return;
2289 }
2290
2291 kfree_rcu_shrinker->count_objects = kfree_rcu_shrink_count;
2292 kfree_rcu_shrinker->scan_objects = kfree_rcu_shrink_scan;
2293
2294 shrinker_register(kfree_rcu_shrinker);
2295 }
2296
2297 #endif /* CONFIG_KVFREE_RCU_BATCHED */
2298