1 // SPDX-License-Identifier: GPL-2.0
8  * Some code borrowed from https://github.com/xairy/kasan-prototype by
53  * depending on memory access size X.
69 						unsigned long size)  in memory_is_poisoned_2_4_8()  argument
74 	 * Access crosses 8(shadow size)-byte boundary. Such access maps  in memory_is_poisoned_2_4_8()
77 	if (unlikely((((unsigned long)addr + size - 1) & KASAN_GRANULE_MASK) < size - 1))  in memory_is_poisoned_2_4_8()
78 		return *shadow_addr || memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
80 	return memory_is_poisoned_1(addr + size - 1);  in memory_is_poisoned_2_4_8()
87 	/* Unaligned 16-bytes access maps into 3 shadow bytes. */  in memory_is_poisoned_16()
95 					size_t size)  in bytes_is_nonzero()  argument
97 	while (size) {  in bytes_is_nonzero()
101 		size--;  in bytes_is_nonzero()
114 	if (end - start <= 16)  in memory_is_nonzero()
115 		return bytes_is_nonzero(start, end - start);  in memory_is_nonzero()
118 		prefix = 8 - prefix;  in memory_is_nonzero()
125 	words = (end - start) / 8;  in memory_is_nonzero()
130 		words--;  in memory_is_nonzero()
133 	return bytes_is_nonzero(start, (end - start) % 8);  in memory_is_nonzero()
136 static __always_inline bool memory_is_poisoned_n(const void *addr, size_t size)  in memory_is_poisoned_n()  argument
141 			kasan_mem_to_shadow(addr + size - 1) + 1);  in memory_is_poisoned_n()
144 		const void *last_byte = addr + size - 1;  in memory_is_poisoned_n()
155 static __always_inline bool memory_is_poisoned(const void *addr, size_t size)  in memory_is_poisoned()  argument
157 	if (__builtin_constant_p(size)) {  in memory_is_poisoned()
158 		switch (size) {  in memory_is_poisoned()
164 			return memory_is_poisoned_2_4_8(addr, size);  in memory_is_poisoned()
172 	return memory_is_poisoned_n(addr, size);  in memory_is_poisoned()
176 						size_t size, bool write,  in check_region_inline()  argument
182 	if (unlikely(size == 0))  in check_region_inline()
185 	if (unlikely(addr + size < addr))  in check_region_inline()
186 		return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
189 		return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
191 	if (likely(!memory_is_poisoned(addr, size)))  in check_region_inline()
194 	return !kasan_report(addr, size, write, ret_ip);  in check_region_inline()
197 bool kasan_check_range(const void *addr, size_t size, bool write,  in kasan_check_range()  argument
200 	return check_region_inline(addr, size, write, ret_ip);  in kasan_check_range()
215 void kasan_cache_shrink(struct kmem_cache *cache)  in kasan_cache_shrink()  argument
217 	kasan_quarantine_remove_cache(cache);  in kasan_cache_shrink()
220 void kasan_cache_shutdown(struct kmem_cache *cache)  in kasan_cache_shutdown()  argument
222 	if (!__kmem_cache_empty(cache))  in kasan_cache_shutdown()
223 		kasan_quarantine_remove_cache(cache);  in kasan_cache_shutdown()
228 	size_t aligned_size = round_up(global->size, KASAN_GRANULE_SIZE);  in register_global()
230 	kasan_unpoison(global->beg, global->size, false);  in register_global()
232 	kasan_poison(global->beg + aligned_size,  in register_global()
233 		     global->size_with_redzone - aligned_size,  in register_global()
237 void __asan_register_globals(void *ptr, ssize_t size)  in __asan_register_globals()  argument
242 	for (i = 0; i < size; i++)  in __asan_register_globals()
247 void __asan_unregister_globals(void *ptr, ssize_t size)  in __asan_unregister_globals()  argument
252 #define DEFINE_ASAN_LOAD_STORE(size)					\  argument
253 	void __asan_load##size(void *addr)				\
255 		check_region_inline(addr, size, false, _RET_IP_);	\
257 	EXPORT_SYMBOL(__asan_load##size);				\
258 	__alias(__asan_load##size)					\
259 	void __asan_load##size##_noabort(void *);			\
260 	EXPORT_SYMBOL(__asan_load##size##_noabort);			\
261 	void __asan_store##size(void *addr)				\
263 		check_region_inline(addr, size, true, _RET_IP_);	\
265 	EXPORT_SYMBOL(__asan_store##size);				\
266 	__alias(__asan_store##size)					\
267 	void __asan_store##size##_noabort(void *);			\
268 	EXPORT_SYMBOL(__asan_store##size##_noabort)
276 void __asan_loadN(void *addr, ssize_t size)  in __asan_loadN()  argument
278 	kasan_check_range(addr, size, false, _RET_IP_);  in __asan_loadN()
286 void __asan_storeN(void *addr, ssize_t size)  in __asan_storeN()  argument
288 	kasan_check_range(addr, size, true, _RET_IP_);  in __asan_storeN()
301 void __asan_alloca_poison(void *addr, ssize_t size)  in __asan_alloca_poison()  argument
303 	size_t rounded_up_size = round_up(size, KASAN_GRANULE_SIZE);  in __asan_alloca_poison()
304 	size_t padding_size = round_up(size, KASAN_ALLOCA_REDZONE_SIZE) -  in __asan_alloca_poison()
306 	size_t rounded_down_size = round_down(size, KASAN_GRANULE_SIZE);  in __asan_alloca_poison()
308 	const void *left_redzone = (const void *)(addr -  in __asan_alloca_poison()
315 			size - rounded_down_size, false);  in __asan_alloca_poison()
329 	kasan_unpoison(stack_top, (void *)stack_bottom - stack_top, false);  in __asan_allocas_unpoison()
335 	void __asan_set_shadow_##byte(const void *addr, ssize_t size)	\
337 		__memset((void *)addr, 0x##byte, size);			\
355 		object_size <= 64        - 16   ? 16 :  in optimal_redzone()
356 		object_size <= 128       - 32   ? 32 :  in optimal_redzone()
357 		object_size <= 512       - 64   ? 64 :  in optimal_redzone()
358 		object_size <= 4096      - 128  ? 128 :  in optimal_redzone()
359 		object_size <= (1 << 14) - 256  ? 256 :  in optimal_redzone()
360 		object_size <= (1 << 15) - 512  ? 512 :  in optimal_redzone()
361 		object_size <= (1 << 16) - 1024 ? 1024 : 2048;  in optimal_redzone()
364 void kasan_cache_create(struct kmem_cache *cache, unsigned int *size,  in kasan_cache_create()  argument
377 	 * that thus have per-object metadata. Currently, this flag is used in  in kasan_cache_create()
378 	 * slab_ksize() to account for per-object metadata when calculating the  in kasan_cache_create()
379 	 * size of the accessible memory within the object. Additionally, we use  in kasan_cache_create()
380 	 * SLAB_NO_MERGE to prevent merging of caches with per-object metadata.  in kasan_cache_create()
384 	ok_size = *size;  in kasan_cache_create()
387 	cache->kasan_info.alloc_meta_offset = *size;  in kasan_cache_create()
388 	*size += sizeof(struct kasan_alloc_meta);  in kasan_cache_create()
391 	if (*size > KMALLOC_MAX_SIZE) {  in kasan_cache_create()
392 		cache->kasan_info.alloc_meta_offset = 0;  in kasan_cache_create()
393 		*size = ok_size;  in kasan_cache_create()
397 	ok_size = *size;  in kasan_cache_create()
398 	orig_alloc_meta_offset = cache->kasan_info.alloc_meta_offset;  in kasan_cache_create()
407 	 * 3. It is from a kmalloc cache which enables the debug option  in kasan_cache_create()
408 	 *    to store original size.  in kasan_cache_create()
410 	if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor ||  in kasan_cache_create()
411 	     slub_debug_orig_size(cache)) {  in kasan_cache_create()
412 		cache->kasan_info.free_meta_offset = *size;  in kasan_cache_create()
413 		*size += sizeof(struct kasan_free_meta);  in kasan_cache_create()
421 	if (sizeof(struct kasan_free_meta) <= cache->object_size) {  in kasan_cache_create()
422 		/* cache->kasan_info.free_meta_offset = 0 is implied. */  in kasan_cache_create()
434 		rem_free_meta_size = sizeof(struct kasan_free_meta) -  in kasan_cache_create()
435 							cache->object_size;  in kasan_cache_create()
436 		*size += rem_free_meta_size;  in kasan_cache_create()
437 		if (cache->kasan_info.alloc_meta_offset != 0)  in kasan_cache_create()
438 			cache->kasan_info.alloc_meta_offset += rem_free_meta_size;  in kasan_cache_create()
446 	cache->kasan_info.free_meta_offset = *size;  in kasan_cache_create()
447 	*size += sizeof(struct kasan_free_meta);  in kasan_cache_create()
451 	if (*size > KMALLOC_MAX_SIZE) {  in kasan_cache_create()
452 		cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META;  in kasan_cache_create()
453 		cache->kasan_info.alloc_meta_offset = orig_alloc_meta_offset;  in kasan_cache_create()
454 		*size = ok_size;  in kasan_cache_create()
457 	/* Calculate size with optimal redzone. */  in kasan_cache_create()
458 	optimal_size = cache->object_size + optimal_redzone(cache->object_size);  in kasan_cache_create()
462 	/* Use optimal size if the size with added metas is not large enough. */  in kasan_cache_create()
463 	if (*size < optimal_size)  in kasan_cache_create()
464 		*size = optimal_size;  in kasan_cache_create()
467 struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache,  in kasan_get_alloc_meta()  argument
470 	if (!cache->kasan_info.alloc_meta_offset)  in kasan_get_alloc_meta()
472 	return (void *)object + cache->kasan_info.alloc_meta_offset;  in kasan_get_alloc_meta()
475 struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache,  in kasan_get_free_meta()  argument
479 	if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META)  in kasan_get_free_meta()
481 	return (void *)object + cache->kasan_info.free_meta_offset;  in kasan_get_free_meta()
484 void kasan_init_object_meta(struct kmem_cache *cache, const void *object)  in kasan_init_object_meta()  argument
488 	alloc_meta = kasan_get_alloc_meta(cache, object);  in kasan_init_object_meta()
520 size_t kasan_metadata_size(struct kmem_cache *cache, bool in_object)  in kasan_metadata_size()  argument
522 	struct kasan_cache *info = &cache->kasan_info;  in kasan_metadata_size()
528 		return (info->free_meta_offset ?  in kasan_metadata_size()
531 		return (info->alloc_meta_offset ?  in kasan_metadata_size()
533 			((info->free_meta_offset &&  in kasan_metadata_size()
534 			info->free_meta_offset != KASAN_NO_FREE_META) ?  in kasan_metadata_size()
545 	struct kmem_cache *cache;  in kasan_record_aux_stack()  local
552 	cache = slab->slab_cache;  in kasan_record_aux_stack()
553 	object = nearest_obj(cache, slab, addr);  in kasan_record_aux_stack()
554 	alloc_meta = kasan_get_alloc_meta(cache, object);  in kasan_record_aux_stack()
558 	alloc_meta->aux_stack[1] = alloc_meta->aux_stack[0];  in kasan_record_aux_stack()
559 	alloc_meta->aux_stack[0] = kasan_save_stack(0, 0);  in kasan_record_aux_stack()
562 void kasan_save_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags)  in kasan_save_alloc_info()  argument
566 	alloc_meta = kasan_get_alloc_meta(cache, object);  in kasan_save_alloc_info()
573 	kasan_save_track(&alloc_meta->alloc_track, flags);  in kasan_save_alloc_info()
576 void __kasan_save_free_info(struct kmem_cache *cache, void *object)  in __kasan_save_free_info()  argument
580 	free_meta = kasan_get_free_meta(cache, object);  in __kasan_save_free_info()
587 	kasan_save_track(&free_meta->free_track, 0);  in __kasan_save_free_info()