1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2017-2022 Google LLC
21 #include <linux/percpu-defs.h>
35  * Per-CPU KMSAN context to be used in interrupts, where current->kmsan is
42 	struct kmsan_ctx *ctx = &task->kmsan_ctx;  in kmsan_internal_task_create()
58 	kmsan_internal_set_shadow_origin(address, size, -1, handle, checked);  in kmsan_internal_poison_memory()
113 	step = backwards ? -1 : 1;  in kmsan_internal_memmove_metadata()
114 	iter = backwards ? n - 1 : 0;  in kmsan_internal_memmove_metadata()
118 	/* Copy shadow bytes one by one, updating the origins if necessary. */  in kmsan_internal_memmove_metadata()
185 	 * @entries is a local var in non-instrumented code, so KMSAN does not  in kmsan_internal_chain_origin()
206 		 * kmsan_metadata_is_contiguous() is true, so either all shadow  in kmsan_internal_set_shadow_origin()
207 		 * and origin pages are NULL, or all are non-NULL.  in kmsan_internal_set_shadow_origin()
210 			pr_err("%s: not memsetting %ld bytes starting at %px, because the shadow is NULL\n",  in kmsan_internal_set_shadow_origin()
222 		address -= pad;  in kmsan_internal_set_shadow_origin()
223 		aligned_shadow = shadow_start - pad;  in kmsan_internal_set_shadow_origin()
231 	 * If the new origin is non-zero, assume that the shadow byte is also non-zero,  in kmsan_internal_set_shadow_origin()
234 	 * corresponding shadow slot is zero.  in kmsan_internal_set_shadow_origin()
262 	unsigned char *shadow = NULL;  in kmsan_internal_check_memory()  local
263 	int cur_off_start = -1;  in kmsan_internal_check_memory()
271 		chunk_size = min(size - pos,  in kmsan_internal_check_memory()
272 				 PAGE_SIZE - ((addr64 + pos) % PAGE_SIZE));  in kmsan_internal_check_memory()
273 		shadow = kmsan_get_metadata((void *)(addr64 + pos),  in kmsan_internal_check_memory()
275 		if (!shadow) {  in kmsan_internal_check_memory()
282 					     cur_off_start, pos - 1, user_addr,  in kmsan_internal_check_memory()
286 			cur_off_start = -1;  in kmsan_internal_check_memory()
291 			if (!shadow[i]) {  in kmsan_internal_check_memory()
298 						     cur_off_start, pos + i - 1,  in kmsan_internal_check_memory()
302 				cur_off_start = -1;  in kmsan_internal_check_memory()
310 			 * Encountered new origin - report the previous  in kmsan_internal_check_memory()
316 						     cur_off_start, pos + i - 1,  in kmsan_internal_check_memory()
327 		kmsan_report(cur_origin, addr, size, cur_off_start, pos - 1,  in kmsan_internal_check_memory()
344 	if (ALIGN_DOWN(cur_addr + size - 1, PAGE_SIZE) ==  in kmsan_metadata_is_contiguous()
366 		if (((u64)cur_shadow == ((u64)next_shadow - PAGE_SIZE)) &&  in kmsan_metadata_is_contiguous()
367 		    ((u64)cur_origin == ((u64)next_origin - PAGE_SIZE)))  in kmsan_metadata_is_contiguous()
374 	pr_err("%s: attempting to access two shadow page ranges.\n", __func__);  in kmsan_metadata_is_contiguous()
378 	pr_err("page[0].shadow: %px, page[1].shadow: %px\n", cur_shadow,  in kmsan_metadata_is_contiguous()