1 // SPDX-License-Identifier: GPL-2.0
5  * Copyright (C) 2017-2022 Google LLC
21 #include <linux/percpu-defs.h>
35  * Per-CPU KMSAN context to be used in interrupts, where current->kmsan is
42 	struct kmsan_ctx *ctx = &task->kmsan_ctx;  in kmsan_internal_task_create()
58 	kmsan_internal_set_shadow_origin(address, size, -1, handle, checked);  in kmsan_internal_poison_memory()
113 	step = backwards ? -1 : 1;  in kmsan_internal_memmove_metadata()
114 	iter = backwards ? n - 1 : 0;  in kmsan_internal_memmove_metadata()
136 			 * origin value.  in kmsan_internal_memmove_metadata()
172 	 * are copied around many times. Origin chains for such structures are  in kmsan_internal_chain_origin()
185 	 * @entries is a local var in non-instrumented code, so KMSAN does not  in kmsan_internal_chain_origin()
195 				      u32 origin, bool checked)  in kmsan_internal_set_shadow_origin()  argument
207 		 * and origin pages are NULL, or all are non-NULL.  in kmsan_internal_set_shadow_origin()
222 		address -= pad;  in kmsan_internal_set_shadow_origin()
223 		aligned_shadow = shadow_start - pad;  in kmsan_internal_set_shadow_origin()
231 	 * If the new origin is non-zero, assume that the shadow byte is also non-zero,  in kmsan_internal_set_shadow_origin()
232 	 * and unconditionally overwrite the old origin slot.  in kmsan_internal_set_shadow_origin()
233 	 * If the new origin is zero, overwrite the old origin slot iff the  in kmsan_internal_set_shadow_origin()
237 		if (origin || !aligned_shadow[i])  in kmsan_internal_set_shadow_origin()
238 			origin_start[i] = origin;  in kmsan_internal_set_shadow_origin()
261 	depot_stack_handle_t *origin = NULL;  in kmsan_internal_check_memory()  local
263 	int cur_off_start = -1;  in kmsan_internal_check_memory()
271 		chunk_size = min(size - pos,  in kmsan_internal_check_memory()
272 				 PAGE_SIZE - ((addr64 + pos) % PAGE_SIZE));  in kmsan_internal_check_memory()
282 					     cur_off_start, pos - 1, user_addr,  in kmsan_internal_check_memory()
286 			cur_off_start = -1;  in kmsan_internal_check_memory()
298 						     cur_off_start, pos + i - 1,  in kmsan_internal_check_memory()
302 				cur_off_start = -1;  in kmsan_internal_check_memory()
305 			origin = kmsan_get_metadata((void *)(addr64 + pos + i),  in kmsan_internal_check_memory()
307 			KMSAN_WARN_ON(!origin);  in kmsan_internal_check_memory()
308 			new_origin = *origin;  in kmsan_internal_check_memory()
310 			 * Encountered new origin - report the previous  in kmsan_internal_check_memory()
316 						     cur_off_start, pos + i - 1,  in kmsan_internal_check_memory()
327 		kmsan_report(cur_origin, addr, size, cur_off_start, pos - 1,  in kmsan_internal_check_memory()
344 	if (ALIGN_DOWN(cur_addr + size - 1, PAGE_SIZE) ==  in kmsan_metadata_is_contiguous()
366 		if (((u64)cur_shadow == ((u64)next_shadow - PAGE_SIZE)) &&  in kmsan_metadata_is_contiguous()
367 		    ((u64)cur_origin == ((u64)next_origin - PAGE_SIZE)))  in kmsan_metadata_is_contiguous()
380 	pr_err("page[0].origin: %px, page[1].origin: %px\n", cur_origin,  in kmsan_metadata_is_contiguous()
384 		pr_err("Origin: %08x\n", *origin_p);  in kmsan_metadata_is_contiguous()
387 		pr_err("Origin: unavailable\n");  in kmsan_metadata_is_contiguous()