1a0503b8aSKuan-Ying Lee // SPDX-License-Identifier: GPL-2.0
2a0503b8aSKuan-Ying Lee /*
3a0503b8aSKuan-Ying Lee * This file contains common tag-based KASAN code.
4a0503b8aSKuan-Ying Lee *
5a0503b8aSKuan-Ying Lee * Copyright (c) 2018 Google, Inc.
6a0503b8aSKuan-Ying Lee * Copyright (c) 2020 Google, Inc.
7a0503b8aSKuan-Ying Lee */
8a0503b8aSKuan-Ying Lee
97bc0584eSAndrey Konovalov #include <linux/atomic.h>
10a0503b8aSKuan-Ying Lee #include <linux/init.h>
11a0503b8aSKuan-Ying Lee #include <linux/kasan.h>
12a0503b8aSKuan-Ying Lee #include <linux/kernel.h>
1380b92bfeSAndrey Konovalov #include <linux/memblock.h>
14a0503b8aSKuan-Ying Lee #include <linux/memory.h>
15a0503b8aSKuan-Ying Lee #include <linux/mm.h>
165d4c6ac9SJuntong Deng #include <linux/sched/clock.h>
17022012dcSAndrey Konovalov #include <linux/stackdepot.h>
18a0503b8aSKuan-Ying Lee #include <linux/static_key.h>
19a0503b8aSKuan-Ying Lee #include <linux/string.h>
20a0503b8aSKuan-Ying Lee #include <linux/types.h>
21a0503b8aSKuan-Ying Lee
22a0503b8aSKuan-Ying Lee #include "kasan.h"
237bc0584eSAndrey Konovalov #include "../slab.h"
247bc0584eSAndrey Konovalov
2580b92bfeSAndrey Konovalov #define KASAN_STACK_RING_SIZE_DEFAULT (32 << 10)
2680b92bfeSAndrey Konovalov
277ebfce33SAndrey Konovalov enum kasan_arg_stacktrace {
287ebfce33SAndrey Konovalov KASAN_ARG_STACKTRACE_DEFAULT,
297ebfce33SAndrey Konovalov KASAN_ARG_STACKTRACE_OFF,
307ebfce33SAndrey Konovalov KASAN_ARG_STACKTRACE_ON,
317ebfce33SAndrey Konovalov };
327ebfce33SAndrey Konovalov
337ebfce33SAndrey Konovalov static enum kasan_arg_stacktrace kasan_arg_stacktrace __initdata;
347ebfce33SAndrey Konovalov
357ebfce33SAndrey Konovalov /* Whether to collect alloc/free stack traces. */
367ebfce33SAndrey Konovalov DEFINE_STATIC_KEY_TRUE(kasan_flag_stacktrace);
377ebfce33SAndrey Konovalov
387bc0584eSAndrey Konovalov /* Non-zero, as initial pointer values are 0. */
397bc0584eSAndrey Konovalov #define STACK_RING_BUSY_PTR ((void *)1)
407bc0584eSAndrey Konovalov
417bc0584eSAndrey Konovalov struct kasan_stack_ring stack_ring = {
427bc0584eSAndrey Konovalov .lock = __RW_LOCK_UNLOCKED(stack_ring.lock)
437bc0584eSAndrey Konovalov };
447bc0584eSAndrey Konovalov
457ebfce33SAndrey Konovalov /* kasan.stacktrace=off/on */
early_kasan_flag_stacktrace(char * arg)467ebfce33SAndrey Konovalov static int __init early_kasan_flag_stacktrace(char *arg)
477ebfce33SAndrey Konovalov {
487ebfce33SAndrey Konovalov if (!arg)
497ebfce33SAndrey Konovalov return -EINVAL;
507ebfce33SAndrey Konovalov
517ebfce33SAndrey Konovalov if (!strcmp(arg, "off"))
527ebfce33SAndrey Konovalov kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_OFF;
537ebfce33SAndrey Konovalov else if (!strcmp(arg, "on"))
547ebfce33SAndrey Konovalov kasan_arg_stacktrace = KASAN_ARG_STACKTRACE_ON;
557ebfce33SAndrey Konovalov else
567ebfce33SAndrey Konovalov return -EINVAL;
577ebfce33SAndrey Konovalov
587ebfce33SAndrey Konovalov return 0;
597ebfce33SAndrey Konovalov }
607ebfce33SAndrey Konovalov early_param("kasan.stacktrace", early_kasan_flag_stacktrace);
617ebfce33SAndrey Konovalov
6280b92bfeSAndrey Konovalov /* kasan.stack_ring_size=<number of entries> */
early_kasan_flag_stack_ring_size(char * arg)6380b92bfeSAndrey Konovalov static int __init early_kasan_flag_stack_ring_size(char *arg)
6480b92bfeSAndrey Konovalov {
6580b92bfeSAndrey Konovalov if (!arg)
6680b92bfeSAndrey Konovalov return -EINVAL;
6780b92bfeSAndrey Konovalov
6880b92bfeSAndrey Konovalov return kstrtoul(arg, 0, &stack_ring.size);
6980b92bfeSAndrey Konovalov }
7080b92bfeSAndrey Konovalov early_param("kasan.stack_ring_size", early_kasan_flag_stack_ring_size);
7180b92bfeSAndrey Konovalov
kasan_init_tags(void)727ebfce33SAndrey Konovalov void __init kasan_init_tags(void)
737ebfce33SAndrey Konovalov {
747ebfce33SAndrey Konovalov switch (kasan_arg_stacktrace) {
757ebfce33SAndrey Konovalov case KASAN_ARG_STACKTRACE_DEFAULT:
767ebfce33SAndrey Konovalov /* Default is specified by kasan_flag_stacktrace definition. */
777ebfce33SAndrey Konovalov break;
787ebfce33SAndrey Konovalov case KASAN_ARG_STACKTRACE_OFF:
797ebfce33SAndrey Konovalov static_branch_disable(&kasan_flag_stacktrace);
807ebfce33SAndrey Konovalov break;
817ebfce33SAndrey Konovalov case KASAN_ARG_STACKTRACE_ON:
827ebfce33SAndrey Konovalov static_branch_enable(&kasan_flag_stacktrace);
837ebfce33SAndrey Konovalov break;
847ebfce33SAndrey Konovalov }
8580b92bfeSAndrey Konovalov
8680b92bfeSAndrey Konovalov if (kasan_stack_collection_enabled()) {
8780b92bfeSAndrey Konovalov if (!stack_ring.size)
8880b92bfeSAndrey Konovalov stack_ring.size = KASAN_STACK_RING_SIZE_DEFAULT;
8980b92bfeSAndrey Konovalov stack_ring.entries = memblock_alloc(
9080b92bfeSAndrey Konovalov sizeof(stack_ring.entries[0]) * stack_ring.size,
9180b92bfeSAndrey Konovalov SMP_CACHE_BYTES);
9280b92bfeSAndrey Konovalov if (WARN_ON(!stack_ring.entries))
9380b92bfeSAndrey Konovalov static_branch_disable(&kasan_flag_stacktrace);
9480b92bfeSAndrey Konovalov }
957ebfce33SAndrey Konovalov }
967ebfce33SAndrey Konovalov
save_stack_info(struct kmem_cache * cache,void * object,gfp_t gfp_flags,bool is_free)977bc0584eSAndrey Konovalov static void save_stack_info(struct kmem_cache *cache, void *object,
987bc0584eSAndrey Konovalov gfp_t gfp_flags, bool is_free)
997bc0584eSAndrey Konovalov {
1007bc0584eSAndrey Konovalov unsigned long flags;
101f816938bSAndrey Konovalov depot_stack_handle_t stack, old_stack;
1027bc0584eSAndrey Konovalov u64 pos;
1037bc0584eSAndrey Konovalov struct kasan_stack_ring_entry *entry;
1047bc0584eSAndrey Konovalov void *old_ptr;
1057bc0584eSAndrey Konovalov
106f816938bSAndrey Konovalov stack = kasan_save_stack(gfp_flags,
107f816938bSAndrey Konovalov STACK_DEPOT_FLAG_CAN_ALLOC | STACK_DEPOT_FLAG_GET);
1087bc0584eSAndrey Konovalov
1097bc0584eSAndrey Konovalov /*
1107bc0584eSAndrey Konovalov * Prevent save_stack_info() from modifying stack ring
1117bc0584eSAndrey Konovalov * when kasan_complete_mode_report_info() is walking it.
1127bc0584eSAndrey Konovalov */
1137bc0584eSAndrey Konovalov read_lock_irqsave(&stack_ring.lock, flags);
1147bc0584eSAndrey Konovalov
1157bc0584eSAndrey Konovalov next:
1167bc0584eSAndrey Konovalov pos = atomic64_fetch_add(1, &stack_ring.pos);
11780b92bfeSAndrey Konovalov entry = &stack_ring.entries[pos % stack_ring.size];
1187bc0584eSAndrey Konovalov
1197bc0584eSAndrey Konovalov /* Detect stack ring entry slots that are being written to. */
1207bc0584eSAndrey Konovalov old_ptr = READ_ONCE(entry->ptr);
1217bc0584eSAndrey Konovalov if (old_ptr == STACK_RING_BUSY_PTR)
1227bc0584eSAndrey Konovalov goto next; /* Busy slot. */
1237bc0584eSAndrey Konovalov if (!try_cmpxchg(&entry->ptr, &old_ptr, STACK_RING_BUSY_PTR))
1247bc0584eSAndrey Konovalov goto next; /* Busy slot. */
1257bc0584eSAndrey Konovalov
12604afc540SAndrey Konovalov old_stack = entry->track.stack;
127f816938bSAndrey Konovalov
128f3b59798SAndrey Konovalov entry->size = cache->object_size;
129*fd4064f6SAndrey Konovalov kasan_set_track(&entry->track, stack);
13004afc540SAndrey Konovalov entry->is_free = is_free;
1317bc0584eSAndrey Konovalov
132f3b59798SAndrey Konovalov entry->ptr = object;
1337bc0584eSAndrey Konovalov
1347bc0584eSAndrey Konovalov read_unlock_irqrestore(&stack_ring.lock, flags);
135f816938bSAndrey Konovalov
136f816938bSAndrey Konovalov if (old_stack)
137f816938bSAndrey Konovalov stack_depot_put(old_stack);
1387bc0584eSAndrey Konovalov }
139a0503b8aSKuan-Ying Lee
kasan_save_alloc_info(struct kmem_cache * cache,void * object,gfp_t flags)140ccf643e6SAndrey Konovalov void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)
141ccf643e6SAndrey Konovalov {
1427bc0584eSAndrey Konovalov save_stack_info(cache, object, flags, false);
143ccf643e6SAndrey Konovalov }
144ccf643e6SAndrey Konovalov
kasan_save_free_info(struct kmem_cache * cache,void * object)1456b074349SAndrey Konovalov void kasan_save_free_info(struct kmem_cache *cache, void *object)
146a0503b8aSKuan-Ying Lee {
147726ccdbaSTetsuo Handa save_stack_info(cache, object, 0, true);
148a0503b8aSKuan-Ying Lee }
149