1e886bf9dSAndrey Konovalov // SPDX-License-Identifier: GPL-2.0
2bffa986cSAndrey Konovalov /*
3bb359dbcSAndrey Konovalov * This file contains common KASAN code.
4bffa986cSAndrey Konovalov *
5bffa986cSAndrey Konovalov * Copyright (c) 2014 Samsung Electronics Co., Ltd.
6bffa986cSAndrey Konovalov * Author: Andrey Ryabinin <ryabinin.a.a@gmail.com>
7bffa986cSAndrey Konovalov *
8bffa986cSAndrey Konovalov * Some code borrowed from https://github.com/xairy/kasan-prototype by
9bffa986cSAndrey Konovalov * Andrey Konovalov <andreyknvl@gmail.com>
10bffa986cSAndrey Konovalov */
11bffa986cSAndrey Konovalov
12bffa986cSAndrey Konovalov #include <linux/export.h>
13bffa986cSAndrey Konovalov #include <linux/init.h>
14bffa986cSAndrey Konovalov #include <linux/kasan.h>
15bffa986cSAndrey Konovalov #include <linux/kernel.h>
16bffa986cSAndrey Konovalov #include <linux/linkage.h>
17bffa986cSAndrey Konovalov #include <linux/memblock.h>
18bffa986cSAndrey Konovalov #include <linux/memory.h>
19bffa986cSAndrey Konovalov #include <linux/mm.h>
20bffa986cSAndrey Konovalov #include <linux/module.h>
21bffa986cSAndrey Konovalov #include <linux/printk.h>
22bffa986cSAndrey Konovalov #include <linux/sched.h>
235d4c6ac9SJuntong Deng #include <linux/sched/clock.h>
24bffa986cSAndrey Konovalov #include <linux/sched/task_stack.h>
25bffa986cSAndrey Konovalov #include <linux/slab.h>
26022012dcSAndrey Konovalov #include <linux/stackdepot.h>
27bffa986cSAndrey Konovalov #include <linux/stacktrace.h>
28bffa986cSAndrey Konovalov #include <linux/string.h>
29bffa986cSAndrey Konovalov #include <linux/types.h>
30bffa986cSAndrey Konovalov #include <linux/bug.h>
31bffa986cSAndrey Konovalov
32bffa986cSAndrey Konovalov #include "kasan.h"
33bffa986cSAndrey Konovalov #include "../slab.h"
34bffa986cSAndrey Konovalov
kasan_addr_to_slab(const void * addr)350f282f15SAndrey Konovalov struct slab *kasan_addr_to_slab(const void *addr)
360f282f15SAndrey Konovalov {
370f282f15SAndrey Konovalov if (virt_addr_valid(addr))
380f282f15SAndrey Konovalov return virt_to_slab(addr);
390f282f15SAndrey Konovalov return NULL;
400f282f15SAndrey Konovalov }
410f282f15SAndrey Konovalov
kasan_save_stack(gfp_t flags,depot_flags_t depot_flags)42022012dcSAndrey Konovalov depot_stack_handle_t kasan_save_stack(gfp_t flags, depot_flags_t depot_flags)
43bffa986cSAndrey Konovalov {
44bffa986cSAndrey Konovalov unsigned long entries[KASAN_STACK_DEPTH];
45880e049cSThomas Gleixner unsigned int nr_entries;
46bffa986cSAndrey Konovalov
47880e049cSThomas Gleixner nr_entries = stack_trace_save(entries, ARRAY_SIZE(entries), 0);
48022012dcSAndrey Konovalov return stack_depot_save_flags(entries, nr_entries, flags, depot_flags);
49bffa986cSAndrey Konovalov }
50bffa986cSAndrey Konovalov
kasan_set_track(struct kasan_track * track,depot_stack_handle_t stack)51fd4064f6SAndrey Konovalov void kasan_set_track(struct kasan_track *track, depot_stack_handle_t stack)
52bffa986cSAndrey Konovalov {
535d4c6ac9SJuntong Deng #ifdef CONFIG_KASAN_EXTRA_INFO
545d4c6ac9SJuntong Deng u32 cpu = raw_smp_processor_id();
555d4c6ac9SJuntong Deng u64 ts_nsec = local_clock();
565d4c6ac9SJuntong Deng
575d4c6ac9SJuntong Deng track->cpu = cpu;
58*952237b5SJuntong Deng track->timestamp = ts_nsec >> 9;
595d4c6ac9SJuntong Deng #endif /* CONFIG_KASAN_EXTRA_INFO */
60bffa986cSAndrey Konovalov track->pid = current->pid;
61fd4064f6SAndrey Konovalov track->stack = stack;
62fd4064f6SAndrey Konovalov }
63fd4064f6SAndrey Konovalov
kasan_save_track(struct kasan_track * track,gfp_t flags)64fd4064f6SAndrey Konovalov void kasan_save_track(struct kasan_track *track, gfp_t flags)
65fd4064f6SAndrey Konovalov {
66fd4064f6SAndrey Konovalov depot_stack_handle_t stack;
67fd4064f6SAndrey Konovalov
68711d3491SMarco Elver stack = kasan_save_stack(flags, STACK_DEPOT_FLAG_CAN_ALLOC);
69fd4064f6SAndrey Konovalov kasan_set_track(track, stack);
70bffa986cSAndrey Konovalov }
71bffa986cSAndrey Konovalov
72d73b4936SAndrey Konovalov #if defined(CONFIG_KASAN_GENERIC) || defined(CONFIG_KASAN_SW_TAGS)
kasan_enable_current(void)73bffa986cSAndrey Konovalov void kasan_enable_current(void)
74bffa986cSAndrey Konovalov {
75bffa986cSAndrey Konovalov current->kasan_depth++;
76bffa986cSAndrey Konovalov }
771f9f78b1SOliver Glitta EXPORT_SYMBOL(kasan_enable_current);
78bffa986cSAndrey Konovalov
kasan_disable_current(void)79bffa986cSAndrey Konovalov void kasan_disable_current(void)
80bffa986cSAndrey Konovalov {
81bffa986cSAndrey Konovalov current->kasan_depth--;
82bffa986cSAndrey Konovalov }
831f9f78b1SOliver Glitta EXPORT_SYMBOL(kasan_disable_current);
841f9f78b1SOliver Glitta
85d73b4936SAndrey Konovalov #endif /* CONFIG_KASAN_GENERIC || CONFIG_KASAN_SW_TAGS */
86bffa986cSAndrey Konovalov
__kasan_unpoison_range(const void * address,size_t size)8734303244SAndrey Konovalov void __kasan_unpoison_range(const void *address, size_t size)
88cebd0eb2SAndrey Konovalov {
8999f3fe41SAndrey Konovalov if (is_kfence_address(address))
9099f3fe41SAndrey Konovalov return;
9199f3fe41SAndrey Konovalov
92aa5c219cSAndrey Konovalov kasan_unpoison(address, size, false);
93cebd0eb2SAndrey Konovalov }
94cebd0eb2SAndrey Konovalov
9502c58773SWalter Wu #ifdef CONFIG_KASAN_STACK
96bffa986cSAndrey Konovalov /* Unpoison the entire stack for a task. */
kasan_unpoison_task_stack(struct task_struct * task)97bffa986cSAndrey Konovalov void kasan_unpoison_task_stack(struct task_struct *task)
98bffa986cSAndrey Konovalov {
9977f57c98SAndrey Konovalov void *base = task_stack_page(task);
10077f57c98SAndrey Konovalov
101aa5c219cSAndrey Konovalov kasan_unpoison(base, THREAD_SIZE, false);
102bffa986cSAndrey Konovalov }
103bffa986cSAndrey Konovalov
104bffa986cSAndrey Konovalov /* Unpoison the stack for the current task beyond a watermark sp value. */
kasan_unpoison_task_stack_below(const void * watermark)105bffa986cSAndrey Konovalov asmlinkage void kasan_unpoison_task_stack_below(const void *watermark)
106bffa986cSAndrey Konovalov {
107bffa986cSAndrey Konovalov /*
108bffa986cSAndrey Konovalov * Calculate the task stack base address. Avoid using 'current'
109bffa986cSAndrey Konovalov * because this function is called by early resume code which hasn't
110bffa986cSAndrey Konovalov * yet set up the percpu register (%gs).
111bffa986cSAndrey Konovalov */
112bffa986cSAndrey Konovalov void *base = (void *)((unsigned long)watermark & ~(THREAD_SIZE - 1));
113bffa986cSAndrey Konovalov
114aa5c219cSAndrey Konovalov kasan_unpoison(base, watermark - base, false);
115bffa986cSAndrey Konovalov }
116d56a9ef8SAndrey Konovalov #endif /* CONFIG_KASAN_STACK */
117bffa986cSAndrey Konovalov
__kasan_unpoison_pages(struct page * page,unsigned int order,bool init)11844383cefSAndrey Konovalov bool __kasan_unpoison_pages(struct page *page, unsigned int order, bool init)
119bffa986cSAndrey Konovalov {
1202813b9c0SAndrey Konovalov u8 tag;
1212813b9c0SAndrey Konovalov unsigned long i;
1222813b9c0SAndrey Konovalov
1237f94ffbcSAndrey Konovalov if (unlikely(PageHighMem(page)))
12444383cefSAndrey Konovalov return false;
12544383cefSAndrey Konovalov
12644383cefSAndrey Konovalov if (!kasan_sample_page_alloc(order))
12744383cefSAndrey Konovalov return false;
1282813b9c0SAndrey Konovalov
129f00748bfSAndrey Konovalov tag = kasan_random_tag();
130ed0a6d1dSCatalin Marinas kasan_unpoison(set_tag(page_address(page), tag),
131ed0a6d1dSCatalin Marinas PAGE_SIZE << order, init);
1322813b9c0SAndrey Konovalov for (i = 0; i < (1 << order); i++)
1332813b9c0SAndrey Konovalov page_kasan_tag_set(page + i, tag);
13444383cefSAndrey Konovalov
13544383cefSAndrey Konovalov return true;
136bffa986cSAndrey Konovalov }
137bffa986cSAndrey Konovalov
__kasan_poison_pages(struct page * page,unsigned int order,bool init)1387a3b8353SPeter Collingbourne void __kasan_poison_pages(struct page *page, unsigned int order, bool init)
139bffa986cSAndrey Konovalov {
140bffa986cSAndrey Konovalov if (likely(!PageHighMem(page)))
141f00748bfSAndrey Konovalov kasan_poison(page_address(page), PAGE_SIZE << order,
14206bc4cf6SAndrey Konovalov KASAN_PAGE_FREE, init);
143bffa986cSAndrey Konovalov }
144bffa986cSAndrey Konovalov
__kasan_poison_slab(struct slab * slab)1456e48a966SMatthew Wilcox (Oracle) void __kasan_poison_slab(struct slab *slab)
146bffa986cSAndrey Konovalov {
1476e48a966SMatthew Wilcox (Oracle) struct page *page = slab_page(slab);
1482813b9c0SAndrey Konovalov unsigned long i;
1492813b9c0SAndrey Konovalov
150d8c6546bSMatthew Wilcox (Oracle) for (i = 0; i < compound_nr(page); i++)
1512813b9c0SAndrey Konovalov page_kasan_tag_reset(page + i);
152f00748bfSAndrey Konovalov kasan_poison(page_address(page), page_size(page),
15306bc4cf6SAndrey Konovalov KASAN_SLAB_REDZONE, false);
154bffa986cSAndrey Konovalov }
155bffa986cSAndrey Konovalov
__kasan_unpoison_new_object(struct kmem_cache * cache,void * object)1561ce9a052SAndrey Konovalov void __kasan_unpoison_new_object(struct kmem_cache *cache, void *object)
157bffa986cSAndrey Konovalov {
158aa5c219cSAndrey Konovalov kasan_unpoison(object, cache->object_size, false);
159bffa986cSAndrey Konovalov }
160bffa986cSAndrey Konovalov
__kasan_poison_new_object(struct kmem_cache * cache,void * object)1611ce9a052SAndrey Konovalov void __kasan_poison_new_object(struct kmem_cache *cache, void *object)
162bffa986cSAndrey Konovalov {
163cde8a7ebSAndrey Konovalov kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
16406bc4cf6SAndrey Konovalov KASAN_SLAB_REDZONE, false);
165bffa986cSAndrey Konovalov }
166bffa986cSAndrey Konovalov
1677f94ffbcSAndrey Konovalov /*
168a3fe7cdfSAndrey Konovalov * This function assigns a tag to an object considering the following:
169a3fe7cdfSAndrey Konovalov * 1. A cache might have a constructor, which might save a pointer to a slab
170a3fe7cdfSAndrey Konovalov * object somewhere (e.g. in the object itself). We preassign a tag for
171a3fe7cdfSAndrey Konovalov * each object in caches with constructors during slab creation and reuse
172a3fe7cdfSAndrey Konovalov * the same tag each time a particular object is allocated.
173a3fe7cdfSAndrey Konovalov * 2. A cache might be SLAB_TYPESAFE_BY_RCU, which means objects can be
174a3fe7cdfSAndrey Konovalov * accessed after being freed. We preassign tags for objects in these
175a3fe7cdfSAndrey Konovalov * caches as well.
1767f94ffbcSAndrey Konovalov */
assign_tag(struct kmem_cache * cache,const void * object,bool init)177c80a0366SAndrey Konovalov static inline u8 assign_tag(struct kmem_cache *cache,
178c80a0366SAndrey Konovalov const void *object, bool init)
1797f94ffbcSAndrey Konovalov {
1801ef3133bSAndrey Konovalov if (IS_ENABLED(CONFIG_KASAN_GENERIC))
1811ef3133bSAndrey Konovalov return 0xff;
1821ef3133bSAndrey Konovalov
183e1db95beSAndrey Konovalov /*
184a3fe7cdfSAndrey Konovalov * If the cache neither has a constructor nor has SLAB_TYPESAFE_BY_RCU
185a3fe7cdfSAndrey Konovalov * set, assign a tag when the object is being allocated (init == false).
186a3fe7cdfSAndrey Konovalov */
187a3fe7cdfSAndrey Konovalov if (!cache->ctor && !(cache->flags & SLAB_TYPESAFE_BY_RCU))
188f00748bfSAndrey Konovalov return init ? KASAN_TAG_KERNEL : kasan_random_tag();
189a3fe7cdfSAndrey Konovalov
190a3fe7cdfSAndrey Konovalov /*
19172786c0aSVlastimil Babka * For caches that either have a constructor or SLAB_TYPESAFE_BY_RCU,
19272786c0aSVlastimil Babka * assign a random tag during slab creation, otherwise reuse
193a3fe7cdfSAndrey Konovalov * the already assigned tag.
194a3fe7cdfSAndrey Konovalov */
195f00748bfSAndrey Konovalov return init ? kasan_random_tag() : get_tag(object);
1967f94ffbcSAndrey Konovalov }
1977f94ffbcSAndrey Konovalov
__kasan_init_slab_obj(struct kmem_cache * cache,const void * object)19834303244SAndrey Konovalov void * __must_check __kasan_init_slab_obj(struct kmem_cache *cache,
19966afc7f1SAndrey Konovalov const void *object)
200bffa986cSAndrey Konovalov {
201836daba0SAndrey Konovalov /* Initialize per-object metadata if it is present. */
202284f8590SAndrey Konovalov if (kasan_requires_meta())
203836daba0SAndrey Konovalov kasan_init_object_meta(cache, object);
204bffa986cSAndrey Konovalov
2051ef3133bSAndrey Konovalov /* Tag is ignored in set_tag() without CONFIG_KASAN_SW/HW_TAGS */
206e2db1a9aSAndrey Konovalov object = set_tag(object, assign_tag(cache, object, true));
2077f94ffbcSAndrey Konovalov
208bffa986cSAndrey Konovalov return (void *)object;
209bffa986cSAndrey Konovalov }
210bffa986cSAndrey Konovalov
poison_slab_object(struct kmem_cache * cache,void * object,unsigned long ip,bool init)211b556a462SAndrey Konovalov static inline bool poison_slab_object(struct kmem_cache *cache, void *object,
212b556a462SAndrey Konovalov unsigned long ip, bool init)
213bffa986cSAndrey Konovalov {
2147f94ffbcSAndrey Konovalov void *tagged_object;
215bffa986cSAndrey Konovalov
216af3751f3SDaniel Axtens if (!kasan_arch_is_ready())
217af3751f3SDaniel Axtens return false;
218af3751f3SDaniel Axtens
2197f94ffbcSAndrey Konovalov tagged_object = object;
220c0054c56SAndrey Konovalov object = kasan_reset_tag(object);
2217f94ffbcSAndrey Konovalov
222b556a462SAndrey Konovalov if (unlikely(nearest_obj(cache, virt_to_slab(object), object) != object)) {
2233de0de75SKuan-Ying Lee kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_INVALID_FREE);
224bffa986cSAndrey Konovalov return true;
225bffa986cSAndrey Konovalov }
226bffa986cSAndrey Konovalov
227b556a462SAndrey Konovalov /* RCU slabs could be legally used after free within the RCU period. */
228bffa986cSAndrey Konovalov if (unlikely(cache->flags & SLAB_TYPESAFE_BY_RCU))
229bffa986cSAndrey Konovalov return false;
230bffa986cSAndrey Konovalov
231611806b4SAndrey Konovalov if (!kasan_byte_accessible(tagged_object)) {
2323de0de75SKuan-Ying Lee kasan_report_invalid_free(tagged_object, ip, KASAN_REPORT_DOUBLE_FREE);
233bffa986cSAndrey Konovalov return true;
234bffa986cSAndrey Konovalov }
235bffa986cSAndrey Konovalov
236cde8a7ebSAndrey Konovalov kasan_poison(object, round_up(cache->object_size, KASAN_GRANULE_SIZE),
23706bc4cf6SAndrey Konovalov KASAN_SLAB_FREE, init);
238bffa986cSAndrey Konovalov
239df54b383SAndrey Konovalov if (kasan_stack_collection_enabled())
2406b074349SAndrey Konovalov kasan_save_free_info(cache, tagged_object);
241ae8f06b3SWalter Wu
242b556a462SAndrey Konovalov return false;
243bffa986cSAndrey Konovalov }
244bffa986cSAndrey Konovalov
__kasan_slab_free(struct kmem_cache * cache,void * object,unsigned long ip,bool init)245d57a964eSAndrey Konovalov bool __kasan_slab_free(struct kmem_cache *cache, void *object,
246d57a964eSAndrey Konovalov unsigned long ip, bool init)
247bffa986cSAndrey Konovalov {
24899f3fe41SAndrey Konovalov if (is_kfence_address(object))
24999f3fe41SAndrey Konovalov return false;
25099f3fe41SAndrey Konovalov
25163b85ac5SAndrey Konovalov /*
25263b85ac5SAndrey Konovalov * If the object is buggy, do not let slab put the object onto the
25363b85ac5SAndrey Konovalov * freelist. The object will thus never be allocated again and its
25463b85ac5SAndrey Konovalov * metadata will never get released.
25563b85ac5SAndrey Konovalov */
25663b85ac5SAndrey Konovalov if (poison_slab_object(cache, object, ip, init))
25763b85ac5SAndrey Konovalov return true;
258b556a462SAndrey Konovalov
25963b85ac5SAndrey Konovalov /*
26063b85ac5SAndrey Konovalov * If the object is put into quarantine, do not let slab put the object
26163b85ac5SAndrey Konovalov * onto the freelist for now. The object's metadata is kept until the
26263b85ac5SAndrey Konovalov * object gets evicted from quarantine.
26363b85ac5SAndrey Konovalov */
26463b85ac5SAndrey Konovalov if (kasan_quarantine_put(cache, object))
26563b85ac5SAndrey Konovalov return true;
26663b85ac5SAndrey Konovalov
26763b85ac5SAndrey Konovalov /*
268711d3491SMarco Elver * Note: Keep per-object metadata to allow KASAN print stack traces for
269711d3491SMarco Elver * use-after-free-before-realloc bugs.
27063b85ac5SAndrey Konovalov */
27163b85ac5SAndrey Konovalov
27263b85ac5SAndrey Konovalov /* Let slab put the object onto the freelist. */
27363b85ac5SAndrey Konovalov return false;
274bffa986cSAndrey Konovalov }
275bffa986cSAndrey Konovalov
check_page_allocation(void * ptr,unsigned long ip)2762e7c954cSAndrey Konovalov static inline bool check_page_allocation(void *ptr, unsigned long ip)
277200072ceSAndrey Konovalov {
27855d77baeSChristophe Leroy if (!kasan_arch_is_ready())
27955d77baeSChristophe Leroy return false;
28055d77baeSChristophe Leroy
281200072ceSAndrey Konovalov if (ptr != page_address(virt_to_head_page(ptr))) {
2823de0de75SKuan-Ying Lee kasan_report_invalid_free(ptr, ip, KASAN_REPORT_INVALID_FREE);
283200072ceSAndrey Konovalov return true;
284200072ceSAndrey Konovalov }
285200072ceSAndrey Konovalov
286200072ceSAndrey Konovalov if (!kasan_byte_accessible(ptr)) {
2873de0de75SKuan-Ying Lee kasan_report_invalid_free(ptr, ip, KASAN_REPORT_DOUBLE_FREE);
288200072ceSAndrey Konovalov return true;
289200072ceSAndrey Konovalov }
290200072ceSAndrey Konovalov
291200072ceSAndrey Konovalov return false;
292200072ceSAndrey Konovalov }
293200072ceSAndrey Konovalov
__kasan_kfree_large(void * ptr,unsigned long ip)294200072ceSAndrey Konovalov void __kasan_kfree_large(void *ptr, unsigned long ip)
295200072ceSAndrey Konovalov {
2962e7c954cSAndrey Konovalov check_page_allocation(ptr, ip);
2972e7c954cSAndrey Konovalov
2982e7c954cSAndrey Konovalov /* The object will be poisoned by kasan_poison_pages(). */
299200072ceSAndrey Konovalov }
300200072ceSAndrey Konovalov
unpoison_slab_object(struct kmem_cache * cache,void * object,gfp_t flags,bool init)30129d7355aSAndrey Konovalov static inline void unpoison_slab_object(struct kmem_cache *cache, void *object,
30229d7355aSAndrey Konovalov gfp_t flags, bool init)
303eeb3160cSAndrey Konovalov {
304eeb3160cSAndrey Konovalov /*
30529d7355aSAndrey Konovalov * Unpoison the whole object. For kmalloc() allocations,
30629d7355aSAndrey Konovalov * poison_kmalloc_redzone() will do precise poisoning.
307eeb3160cSAndrey Konovalov */
30829d7355aSAndrey Konovalov kasan_unpoison(object, cache->object_size, init);
3096e48a966SMatthew Wilcox (Oracle)
31029d7355aSAndrey Konovalov /* Save alloc info (if possible) for non-kmalloc() allocations. */
31129d7355aSAndrey Konovalov if (kasan_stack_collection_enabled() && !is_kmalloc_cache(cache))
31229d7355aSAndrey Konovalov kasan_save_alloc_info(cache, object, flags);
313eeb3160cSAndrey Konovalov }
314eeb3160cSAndrey Konovalov
__kasan_slab_alloc(struct kmem_cache * cache,void * object,gfp_t flags,bool init)315e2db1a9aSAndrey Konovalov void * __must_check __kasan_slab_alloc(struct kmem_cache *cache,
316da844b78SAndrey Konovalov void *object, gfp_t flags, bool init)
317e2db1a9aSAndrey Konovalov {
318e2db1a9aSAndrey Konovalov u8 tag;
319e2db1a9aSAndrey Konovalov void *tagged_object;
320e2db1a9aSAndrey Konovalov
321e2db1a9aSAndrey Konovalov if (gfpflags_allow_blocking(flags))
322e2db1a9aSAndrey Konovalov kasan_quarantine_reduce();
323e2db1a9aSAndrey Konovalov
324e2db1a9aSAndrey Konovalov if (unlikely(object == NULL))
325e2db1a9aSAndrey Konovalov return NULL;
326e2db1a9aSAndrey Konovalov
327e2db1a9aSAndrey Konovalov if (is_kfence_address(object))
328e2db1a9aSAndrey Konovalov return (void *)object;
329e2db1a9aSAndrey Konovalov
330e2db1a9aSAndrey Konovalov /*
331e2db1a9aSAndrey Konovalov * Generate and assign random tag for tag-based modes.
332e2db1a9aSAndrey Konovalov * Tag is ignored in set_tag() for the generic mode.
333e2db1a9aSAndrey Konovalov */
334e2db1a9aSAndrey Konovalov tag = assign_tag(cache, object, false);
335e2db1a9aSAndrey Konovalov tagged_object = set_tag(object, tag);
336e2db1a9aSAndrey Konovalov
33729d7355aSAndrey Konovalov /* Unpoison the object and save alloc info for non-kmalloc() allocations. */
33829d7355aSAndrey Konovalov unpoison_slab_object(cache, tagged_object, flags, init);
339e2db1a9aSAndrey Konovalov
340e2db1a9aSAndrey Konovalov return tagged_object;
341e2db1a9aSAndrey Konovalov }
342e2db1a9aSAndrey Konovalov
poison_kmalloc_redzone(struct kmem_cache * cache,const void * object,size_t size,gfp_t flags)343ce37eec0SAndrey Konovalov static inline void poison_kmalloc_redzone(struct kmem_cache *cache,
344c80a0366SAndrey Konovalov const void *object, size_t size, gfp_t flags)
345bffa986cSAndrey Konovalov {
346bffa986cSAndrey Konovalov unsigned long redzone_start;
347bffa986cSAndrey Konovalov unsigned long redzone_end;
348bffa986cSAndrey Konovalov
349e2db1a9aSAndrey Konovalov /*
350e2db1a9aSAndrey Konovalov * The redzone has byte-level precision for the generic mode.
351e2db1a9aSAndrey Konovalov * Partially poison the last object granule to cover the unaligned
352e2db1a9aSAndrey Konovalov * part of the redzone.
353e2db1a9aSAndrey Konovalov */
354e2db1a9aSAndrey Konovalov if (IS_ENABLED(CONFIG_KASAN_GENERIC))
355e2db1a9aSAndrey Konovalov kasan_poison_last_granule((void *)object, size);
356e2db1a9aSAndrey Konovalov
357e2db1a9aSAndrey Konovalov /* Poison the aligned part of the redzone. */
358bffa986cSAndrey Konovalov redzone_start = round_up((unsigned long)(object + size),
3591f600626SAndrey Konovalov KASAN_GRANULE_SIZE);
360cde8a7ebSAndrey Konovalov redzone_end = round_up((unsigned long)(object + cache->object_size),
361cde8a7ebSAndrey Konovalov KASAN_GRANULE_SIZE);
362f00748bfSAndrey Konovalov kasan_poison((void *)redzone_start, redzone_end - redzone_start,
36306bc4cf6SAndrey Konovalov KASAN_SLAB_REDZONE, false);
364bffa986cSAndrey Konovalov
365e2db1a9aSAndrey Konovalov /*
366e2db1a9aSAndrey Konovalov * Save alloc info (if possible) for kmalloc() allocations.
367e2db1a9aSAndrey Konovalov * This also rewrites the alloc info when called from kasan_krealloc().
368e2db1a9aSAndrey Konovalov */
369bbc61844SFeng Tang if (kasan_stack_collection_enabled() && is_kmalloc_cache(cache))
370ccf643e6SAndrey Konovalov kasan_save_alloc_info(cache, (void *)object, flags);
371bffa986cSAndrey Konovalov
372e1db95beSAndrey Konovalov }
373e1db95beSAndrey Konovalov
__kasan_kmalloc(struct kmem_cache * cache,const void * object,size_t size,gfp_t flags)37434303244SAndrey Konovalov void * __must_check __kasan_kmalloc(struct kmem_cache *cache, const void *object,
375a3fe7cdfSAndrey Konovalov size_t size, gfp_t flags)
376a3fe7cdfSAndrey Konovalov {
377ce37eec0SAndrey Konovalov if (gfpflags_allow_blocking(flags))
378ce37eec0SAndrey Konovalov kasan_quarantine_reduce();
379ce37eec0SAndrey Konovalov
380ce37eec0SAndrey Konovalov if (unlikely(object == NULL))
381ce37eec0SAndrey Konovalov return NULL;
382ce37eec0SAndrey Konovalov
38399f3fe41SAndrey Konovalov if (is_kfence_address(object))
384ce37eec0SAndrey Konovalov return (void *)object;
385ce37eec0SAndrey Konovalov
386ce37eec0SAndrey Konovalov /* The object has already been unpoisoned by kasan_slab_alloc(). */
387ce37eec0SAndrey Konovalov poison_kmalloc_redzone(cache, object, size, flags);
388ce37eec0SAndrey Konovalov
389ce37eec0SAndrey Konovalov /* Keep the tag that was set by kasan_slab_alloc(). */
390ce37eec0SAndrey Konovalov return (void *)object;
391a3fe7cdfSAndrey Konovalov }
39234303244SAndrey Konovalov EXPORT_SYMBOL(__kasan_kmalloc);
393bffa986cSAndrey Konovalov
poison_kmalloc_large_redzone(const void * ptr,size_t size,gfp_t flags)3940cc9fdbfSAndrey Konovalov static inline void poison_kmalloc_large_redzone(const void *ptr, size_t size,
39566afc7f1SAndrey Konovalov gfp_t flags)
396bffa986cSAndrey Konovalov {
397bffa986cSAndrey Konovalov unsigned long redzone_start;
398bffa986cSAndrey Konovalov unsigned long redzone_end;
399bffa986cSAndrey Konovalov
40043a219cbSAndrey Konovalov /*
40143a219cbSAndrey Konovalov * The redzone has byte-level precision for the generic mode.
40243a219cbSAndrey Konovalov * Partially poison the last object granule to cover the unaligned
40343a219cbSAndrey Konovalov * part of the redzone.
40443a219cbSAndrey Konovalov */
40543a219cbSAndrey Konovalov if (IS_ENABLED(CONFIG_KASAN_GENERIC))
40643a219cbSAndrey Konovalov kasan_poison_last_granule(ptr, size);
40743a219cbSAndrey Konovalov
40843a219cbSAndrey Konovalov /* Poison the aligned part of the redzone. */
4090cc9fdbfSAndrey Konovalov redzone_start = round_up((unsigned long)(ptr + size), KASAN_GRANULE_SIZE);
41043a219cbSAndrey Konovalov redzone_end = (unsigned long)ptr + page_size(virt_to_page(ptr));
411f00748bfSAndrey Konovalov kasan_poison((void *)redzone_start, redzone_end - redzone_start,
412aa5c219cSAndrey Konovalov KASAN_PAGE_REDZONE, false);
4130cc9fdbfSAndrey Konovalov }
414bffa986cSAndrey Konovalov
__kasan_kmalloc_large(const void * ptr,size_t size,gfp_t flags)4150cc9fdbfSAndrey Konovalov void * __must_check __kasan_kmalloc_large(const void *ptr, size_t size,
4160cc9fdbfSAndrey Konovalov gfp_t flags)
4170cc9fdbfSAndrey Konovalov {
4180cc9fdbfSAndrey Konovalov if (gfpflags_allow_blocking(flags))
4190cc9fdbfSAndrey Konovalov kasan_quarantine_reduce();
4200cc9fdbfSAndrey Konovalov
4210cc9fdbfSAndrey Konovalov if (unlikely(ptr == NULL))
4220cc9fdbfSAndrey Konovalov return NULL;
4230cc9fdbfSAndrey Konovalov
4240cc9fdbfSAndrey Konovalov /* The object has already been unpoisoned by kasan_unpoison_pages(). */
4250cc9fdbfSAndrey Konovalov poison_kmalloc_large_redzone(ptr, size, flags);
4260cc9fdbfSAndrey Konovalov
4270cc9fdbfSAndrey Konovalov /* Keep the tag that was set by alloc_pages(). */
428bffa986cSAndrey Konovalov return (void *)ptr;
429bffa986cSAndrey Konovalov }
430bffa986cSAndrey Konovalov
__kasan_krealloc(const void * object,size_t size,gfp_t flags)43134303244SAndrey Konovalov void * __must_check __kasan_krealloc(const void *object, size_t size, gfp_t flags)
432bffa986cSAndrey Konovalov {
4336e48a966SMatthew Wilcox (Oracle) struct slab *slab;
434bffa986cSAndrey Konovalov
4350cc9fdbfSAndrey Konovalov if (gfpflags_allow_blocking(flags))
4360cc9fdbfSAndrey Konovalov kasan_quarantine_reduce();
4370cc9fdbfSAndrey Konovalov
438bffa986cSAndrey Konovalov if (unlikely(object == ZERO_SIZE_PTR))
439bffa986cSAndrey Konovalov return (void *)object;
440bffa986cSAndrey Konovalov
44199f3fe41SAndrey Konovalov if (is_kfence_address(object))
442ce37eec0SAndrey Konovalov return (void *)object;
443ce37eec0SAndrey Konovalov
444d12d9ad8SAndrey Konovalov /*
445d12d9ad8SAndrey Konovalov * Unpoison the object's data.
446d12d9ad8SAndrey Konovalov * Part of it might already have been unpoisoned, but it's unknown
447d12d9ad8SAndrey Konovalov * how big that part is.
448d12d9ad8SAndrey Konovalov */
449aa5c219cSAndrey Konovalov kasan_unpoison(object, size, false);
450d12d9ad8SAndrey Konovalov
4516e48a966SMatthew Wilcox (Oracle) slab = virt_to_slab(object);
452bffa986cSAndrey Konovalov
453d12d9ad8SAndrey Konovalov /* Piggy-back on kmalloc() instrumentation to poison the redzone. */
4546e48a966SMatthew Wilcox (Oracle) if (unlikely(!slab))
4550cc9fdbfSAndrey Konovalov poison_kmalloc_large_redzone(object, size, flags);
456bffa986cSAndrey Konovalov else
457ce37eec0SAndrey Konovalov poison_kmalloc_redzone(slab->slab_cache, object, size, flags);
4580cc9fdbfSAndrey Konovalov
459ce37eec0SAndrey Konovalov return (void *)object;
460ce37eec0SAndrey Konovalov }
461bffa986cSAndrey Konovalov
__kasan_mempool_poison_pages(struct page * page,unsigned int order,unsigned long ip)462f129c310SAndrey Konovalov bool __kasan_mempool_poison_pages(struct page *page, unsigned int order,
463f129c310SAndrey Konovalov unsigned long ip)
464f129c310SAndrey Konovalov {
465f129c310SAndrey Konovalov unsigned long *ptr;
466f129c310SAndrey Konovalov
467f129c310SAndrey Konovalov if (unlikely(PageHighMem(page)))
468f129c310SAndrey Konovalov return true;
469f129c310SAndrey Konovalov
470f129c310SAndrey Konovalov /* Bail out if allocation was excluded due to sampling. */
471f129c310SAndrey Konovalov if (!IS_ENABLED(CONFIG_KASAN_GENERIC) &&
472f129c310SAndrey Konovalov page_kasan_tag(page) == KASAN_TAG_KERNEL)
473f129c310SAndrey Konovalov return true;
474f129c310SAndrey Konovalov
475f129c310SAndrey Konovalov ptr = page_address(page);
476f129c310SAndrey Konovalov
477f129c310SAndrey Konovalov if (check_page_allocation(ptr, ip))
478f129c310SAndrey Konovalov return false;
479f129c310SAndrey Konovalov
480f129c310SAndrey Konovalov kasan_poison(ptr, PAGE_SIZE << order, KASAN_PAGE_FREE, false);
481f129c310SAndrey Konovalov
482f129c310SAndrey Konovalov return true;
483f129c310SAndrey Konovalov }
484f129c310SAndrey Konovalov
__kasan_mempool_unpoison_pages(struct page * page,unsigned int order,unsigned long ip)4859f41c59aSAndrey Konovalov void __kasan_mempool_unpoison_pages(struct page *page, unsigned int order,
4869f41c59aSAndrey Konovalov unsigned long ip)
4879f41c59aSAndrey Konovalov {
4889f41c59aSAndrey Konovalov __kasan_unpoison_pages(page, order, false);
4899f41c59aSAndrey Konovalov }
4909f41c59aSAndrey Konovalov
__kasan_mempool_poison_object(void * ptr,unsigned long ip)4912e7c954cSAndrey Konovalov bool __kasan_mempool_poison_object(void *ptr, unsigned long ip)
4929b94fe91SAndrey Konovalov {
493cf0da2afSAndrey Konovalov struct folio *folio = virt_to_folio(ptr);
494cf0da2afSAndrey Konovalov struct slab *slab;
4959b94fe91SAndrey Konovalov
4969b94fe91SAndrey Konovalov /*
497cf0da2afSAndrey Konovalov * This function can be called for large kmalloc allocation that get
498cf0da2afSAndrey Konovalov * their memory from page_alloc. Thus, the folio might not be a slab.
4999b94fe91SAndrey Konovalov */
5009b94fe91SAndrey Konovalov if (unlikely(!folio_test_slab(folio))) {
5012e7c954cSAndrey Konovalov if (check_page_allocation(ptr, ip))
5022e7c954cSAndrey Konovalov return false;
5039b94fe91SAndrey Konovalov kasan_poison(ptr, folio_size(folio), KASAN_PAGE_FREE, false);
5042e7c954cSAndrey Konovalov return true;
5059b94fe91SAndrey Konovalov }
506cf0da2afSAndrey Konovalov
50799f3fe41SAndrey Konovalov if (is_kfence_address(ptr))
50899f3fe41SAndrey Konovalov return false;
50999f3fe41SAndrey Konovalov
510cf0da2afSAndrey Konovalov slab = folio_slab(folio);
511b556a462SAndrey Konovalov return !poison_slab_object(slab->slab_cache, ptr, ip, false);
5129b94fe91SAndrey Konovalov }
5139b94fe91SAndrey Konovalov
__kasan_mempool_unpoison_object(void * ptr,size_t size,unsigned long ip)51419568327SAndrey Konovalov void __kasan_mempool_unpoison_object(void *ptr, size_t size, unsigned long ip)
51519568327SAndrey Konovalov {
51629d7355aSAndrey Konovalov struct slab *slab;
51729d7355aSAndrey Konovalov gfp_t flags = 0; /* Might be executing under a lock. */
51829d7355aSAndrey Konovalov
51929d7355aSAndrey Konovalov slab = virt_to_slab(ptr);
52029d7355aSAndrey Konovalov
52129d7355aSAndrey Konovalov /*
52229d7355aSAndrey Konovalov * This function can be called for large kmalloc allocation that get
52329d7355aSAndrey Konovalov * their memory from page_alloc.
52429d7355aSAndrey Konovalov */
52529d7355aSAndrey Konovalov if (unlikely(!slab)) {
52619568327SAndrey Konovalov kasan_unpoison(ptr, size, false);
52729d7355aSAndrey Konovalov poison_kmalloc_large_redzone(ptr, size, flags);
52829d7355aSAndrey Konovalov return;
52929d7355aSAndrey Konovalov }
53029d7355aSAndrey Konovalov
53199f3fe41SAndrey Konovalov if (is_kfence_address(ptr))
53299f3fe41SAndrey Konovalov return;
53399f3fe41SAndrey Konovalov
53429d7355aSAndrey Konovalov /* Unpoison the object and save alloc info for non-kmalloc() allocations. */
53529d7355aSAndrey Konovalov unpoison_slab_object(slab->slab_cache, ptr, size, flags);
53629d7355aSAndrey Konovalov
53729d7355aSAndrey Konovalov /* Poison the redzone and save alloc info for kmalloc() allocations. */
53829d7355aSAndrey Konovalov if (is_kmalloc_cache(slab->slab_cache))
53929d7355aSAndrey Konovalov poison_kmalloc_redzone(slab->slab_cache, ptr, size, flags);
540bffa986cSAndrey Konovalov }
541bffa986cSAndrey Konovalov
__kasan_check_byte(const void * address,unsigned long ip)542611806b4SAndrey Konovalov bool __kasan_check_byte(const void *address, unsigned long ip)
543611806b4SAndrey Konovalov {
544611806b4SAndrey Konovalov if (!kasan_byte_accessible(address)) {
545bb6e04a1SArnd Bergmann kasan_report(address, 1, false, ip);
546611806b4SAndrey Konovalov return false;
547611806b4SAndrey Konovalov }
548611806b4SAndrey Konovalov return true;
549611806b4SAndrey Konovalov }
550