1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * KMSAN hooks for kernel subsystems. 4 * 5 * These functions handle creation of KMSAN metadata for memory allocations. 6 * 7 * Copyright (C) 2018-2022 Google LLC 8 * Author: Alexander Potapenko <glider@google.com> 9 * 10 */ 11 12 #include <linux/cacheflush.h> 13 #include <linux/dma-direction.h> 14 #include <linux/gfp.h> 15 #include <linux/kmsan.h> 16 #include <linux/mm.h> 17 #include <linux/mm_types.h> 18 #include <linux/scatterlist.h> 19 #include <linux/slab.h> 20 #include <linux/uaccess.h> 21 #include <linux/usb.h> 22 23 #include "../internal.h" 24 #include "../vmalloc.h" 25 #include "../slab.h" 26 #include "kmsan.h" 27 28 /* 29 * Instrumented functions shouldn't be called under 30 * kmsan_enter_runtime()/kmsan_leave_runtime(), because this will lead to 31 * skipping effects of functions like memset() inside instrumented code. 32 */ 33 34 void kmsan_task_create(struct task_struct *task) 35 { 36 kmsan_enter_runtime(); 37 kmsan_internal_task_create(task); 38 kmsan_leave_runtime(); 39 } 40 41 void kmsan_task_exit(struct task_struct *task) 42 { 43 if (!kmsan_enabled || kmsan_in_runtime()) 44 return; 45 46 kmsan_disable_current(); 47 } 48 49 void kmsan_slab_alloc(struct kmem_cache *s, void *object, gfp_t flags) 50 { 51 if (unlikely(object == NULL)) 52 return; 53 if (!kmsan_enabled || kmsan_in_runtime()) 54 return; 55 /* 56 * There's a ctor or this is an RCU cache - do nothing. The memory 57 * status hasn't changed since last use. 58 */ 59 if (s->ctor || (s->flags & SLAB_TYPESAFE_BY_RCU)) 60 return; 61 62 kmsan_enter_runtime(); 63 if (flags & __GFP_ZERO) 64 kmsan_internal_unpoison_memory(object, s->object_size, 65 KMSAN_POISON_CHECK); 66 else 67 kmsan_internal_poison_memory(object, s->object_size, flags, 68 KMSAN_POISON_CHECK); 69 kmsan_leave_runtime(); 70 } 71 72 void kmsan_slab_free(struct kmem_cache *s, void *object) 73 { 74 if (!kmsan_enabled || kmsan_in_runtime()) 75 return; 76 77 /* RCU slabs could be legally used after free within the RCU period */ 78 if (unlikely(s->flags & SLAB_TYPESAFE_BY_RCU)) 79 return; 80 /* 81 * If there's a constructor, freed memory must remain in the same state 82 * until the next allocation. We cannot save its state to detect 83 * use-after-free bugs, instead we just keep it unpoisoned. 84 */ 85 if (s->ctor) 86 return; 87 kmsan_enter_runtime(); 88 kmsan_internal_poison_memory(object, s->object_size, 89 GFP_KERNEL & ~(__GFP_RECLAIM), 90 KMSAN_POISON_CHECK | KMSAN_POISON_FREE); 91 kmsan_leave_runtime(); 92 } 93 94 void kmsan_kmalloc_large(const void *ptr, size_t size, gfp_t flags) 95 { 96 if (unlikely(ptr == NULL)) 97 return; 98 if (!kmsan_enabled || kmsan_in_runtime()) 99 return; 100 kmsan_enter_runtime(); 101 if (flags & __GFP_ZERO) 102 kmsan_internal_unpoison_memory((void *)ptr, size, 103 /*checked*/ true); 104 else 105 kmsan_internal_poison_memory((void *)ptr, size, flags, 106 KMSAN_POISON_CHECK); 107 kmsan_leave_runtime(); 108 } 109 110 void kmsan_kfree_large(const void *ptr) 111 { 112 struct page *page; 113 114 if (!kmsan_enabled || kmsan_in_runtime()) 115 return; 116 kmsan_enter_runtime(); 117 page = virt_to_head_page((void *)ptr); 118 KMSAN_WARN_ON(ptr != page_address(page)); 119 kmsan_internal_poison_memory((void *)ptr, page_size(page), 120 GFP_KERNEL & ~(__GFP_RECLAIM), 121 KMSAN_POISON_CHECK | KMSAN_POISON_FREE); 122 kmsan_leave_runtime(); 123 } 124 125 static unsigned long vmalloc_shadow(unsigned long addr) 126 { 127 return (unsigned long)kmsan_get_metadata((void *)addr, 128 KMSAN_META_SHADOW); 129 } 130 131 static unsigned long vmalloc_origin(unsigned long addr) 132 { 133 return (unsigned long)kmsan_get_metadata((void *)addr, 134 KMSAN_META_ORIGIN); 135 } 136 137 void kmsan_vunmap_range_noflush(unsigned long start, unsigned long end) 138 { 139 __vunmap_range_noflush(vmalloc_shadow(start), vmalloc_shadow(end)); 140 __vunmap_range_noflush(vmalloc_origin(start), vmalloc_origin(end)); 141 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); 142 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); 143 } 144 145 /* 146 * This function creates new shadow/origin pages for the physical pages mapped 147 * into the virtual memory. If those physical pages already had shadow/origin, 148 * those are ignored. 149 */ 150 int kmsan_ioremap_page_range(unsigned long start, unsigned long end, 151 phys_addr_t phys_addr, pgprot_t prot, 152 unsigned int page_shift) 153 { 154 gfp_t gfp_mask = GFP_KERNEL | __GFP_ZERO; 155 struct page *shadow, *origin; 156 unsigned long off = 0; 157 int nr, err = 0, clean = 0, mapped; 158 159 if (!kmsan_enabled || kmsan_in_runtime()) 160 return 0; 161 162 nr = (end - start) / PAGE_SIZE; 163 kmsan_enter_runtime(); 164 for (int i = 0; i < nr; i++, off += PAGE_SIZE, clean = i) { 165 shadow = alloc_pages(gfp_mask, 1); 166 origin = alloc_pages(gfp_mask, 1); 167 if (!shadow || !origin) { 168 err = -ENOMEM; 169 goto ret; 170 } 171 mapped = __vmap_pages_range_noflush( 172 vmalloc_shadow(start + off), 173 vmalloc_shadow(start + off + PAGE_SIZE), prot, &shadow, 174 PAGE_SHIFT); 175 if (mapped) { 176 err = mapped; 177 goto ret; 178 } 179 shadow = NULL; 180 mapped = __vmap_pages_range_noflush( 181 vmalloc_origin(start + off), 182 vmalloc_origin(start + off + PAGE_SIZE), prot, &origin, 183 PAGE_SHIFT); 184 if (mapped) { 185 __vunmap_range_noflush( 186 vmalloc_shadow(start + off), 187 vmalloc_shadow(start + off + PAGE_SIZE)); 188 err = mapped; 189 goto ret; 190 } 191 origin = NULL; 192 } 193 /* Page mapping loop finished normally, nothing to clean up. */ 194 clean = 0; 195 196 ret: 197 if (clean > 0) { 198 /* 199 * Something went wrong. Clean up shadow/origin pages allocated 200 * on the last loop iteration, then delete mappings created 201 * during the previous iterations. 202 */ 203 if (shadow) 204 __free_pages(shadow, 1); 205 if (origin) 206 __free_pages(origin, 1); 207 __vunmap_range_noflush( 208 vmalloc_shadow(start), 209 vmalloc_shadow(start + clean * PAGE_SIZE)); 210 __vunmap_range_noflush( 211 vmalloc_origin(start), 212 vmalloc_origin(start + clean * PAGE_SIZE)); 213 } 214 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); 215 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); 216 kmsan_leave_runtime(); 217 return err; 218 } 219 220 void kmsan_iounmap_page_range(unsigned long start, unsigned long end) 221 { 222 unsigned long v_shadow, v_origin; 223 struct page *shadow, *origin; 224 int nr; 225 226 if (!kmsan_enabled || kmsan_in_runtime()) 227 return; 228 229 nr = (end - start) / PAGE_SIZE; 230 kmsan_enter_runtime(); 231 v_shadow = (unsigned long)vmalloc_shadow(start); 232 v_origin = (unsigned long)vmalloc_origin(start); 233 for (int i = 0; i < nr; 234 i++, v_shadow += PAGE_SIZE, v_origin += PAGE_SIZE) { 235 shadow = kmsan_vmalloc_to_page_or_null((void *)v_shadow); 236 origin = kmsan_vmalloc_to_page_or_null((void *)v_origin); 237 __vunmap_range_noflush(v_shadow, vmalloc_shadow(end)); 238 __vunmap_range_noflush(v_origin, vmalloc_origin(end)); 239 if (shadow) 240 __free_pages(shadow, 1); 241 if (origin) 242 __free_pages(origin, 1); 243 } 244 flush_cache_vmap(vmalloc_shadow(start), vmalloc_shadow(end)); 245 flush_cache_vmap(vmalloc_origin(start), vmalloc_origin(end)); 246 kmsan_leave_runtime(); 247 } 248 249 void kmsan_copy_to_user(void __user *to, const void *from, size_t to_copy, 250 size_t left) 251 { 252 unsigned long ua_flags; 253 254 if (!kmsan_enabled || kmsan_in_runtime()) 255 return; 256 /* 257 * At this point we've copied the memory already. It's hard to check it 258 * before copying, as the size of actually copied buffer is unknown. 259 */ 260 261 /* copy_to_user() may copy zero bytes. No need to check. */ 262 if (!to_copy) 263 return; 264 /* Or maybe copy_to_user() failed to copy anything. */ 265 if (to_copy <= left) 266 return; 267 268 ua_flags = user_access_save(); 269 if (!IS_ENABLED(CONFIG_ARCH_HAS_NON_OVERLAPPING_ADDRESS_SPACE) || 270 (u64)to < TASK_SIZE) { 271 /* This is a user memory access, check it. */ 272 kmsan_internal_check_memory((void *)from, to_copy - left, to, 273 REASON_COPY_TO_USER); 274 } else { 275 /* Otherwise this is a kernel memory access. This happens when a 276 * compat syscall passes an argument allocated on the kernel 277 * stack to a real syscall. 278 * Don't check anything, just copy the shadow of the copied 279 * bytes. 280 */ 281 kmsan_enter_runtime(); 282 kmsan_internal_memmove_metadata((void *)to, (void *)from, 283 to_copy - left); 284 kmsan_leave_runtime(); 285 } 286 user_access_restore(ua_flags); 287 } 288 EXPORT_SYMBOL(kmsan_copy_to_user); 289 290 void kmsan_memmove(void *to, const void *from, size_t size) 291 { 292 if (!kmsan_enabled || kmsan_in_runtime()) 293 return; 294 295 kmsan_enter_runtime(); 296 kmsan_internal_memmove_metadata(to, (void *)from, size); 297 kmsan_leave_runtime(); 298 } 299 EXPORT_SYMBOL(kmsan_memmove); 300 301 /* Helper function to check an URB. */ 302 void kmsan_handle_urb(const struct urb *urb, bool is_out) 303 { 304 if (!urb) 305 return; 306 if (is_out) 307 kmsan_internal_check_memory(urb->transfer_buffer, 308 urb->transfer_buffer_length, 309 /*user_addr*/ NULL, 310 REASON_SUBMIT_URB); 311 else 312 kmsan_internal_unpoison_memory(urb->transfer_buffer, 313 urb->transfer_buffer_length, 314 /*checked*/ false); 315 } 316 EXPORT_SYMBOL_GPL(kmsan_handle_urb); 317 318 static void kmsan_handle_dma_page(const void *addr, size_t size, 319 enum dma_data_direction dir) 320 { 321 switch (dir) { 322 case DMA_BIDIRECTIONAL: 323 kmsan_internal_check_memory((void *)addr, size, 324 /*user_addr*/ NULL, REASON_ANY); 325 kmsan_internal_unpoison_memory((void *)addr, size, 326 /*checked*/ false); 327 break; 328 case DMA_TO_DEVICE: 329 kmsan_internal_check_memory((void *)addr, size, 330 /*user_addr*/ NULL, REASON_ANY); 331 break; 332 case DMA_FROM_DEVICE: 333 kmsan_internal_unpoison_memory((void *)addr, size, 334 /*checked*/ false); 335 break; 336 case DMA_NONE: 337 break; 338 } 339 } 340 341 /* Helper function to handle DMA data transfers. */ 342 void kmsan_handle_dma(phys_addr_t phys, size_t size, 343 enum dma_data_direction dir) 344 { 345 u64 page_offset, to_go; 346 void *addr; 347 348 if (PhysHighMem(phys)) 349 return; 350 addr = phys_to_virt(phys); 351 /* 352 * The kernel may occasionally give us adjacent DMA pages not belonging 353 * to the same allocation. Process them separately to avoid triggering 354 * internal KMSAN checks. 355 */ 356 while (size > 0) { 357 page_offset = offset_in_page(addr); 358 to_go = min(PAGE_SIZE - page_offset, (u64)size); 359 kmsan_handle_dma_page((void *)addr, to_go, dir); 360 addr += to_go; 361 size -= to_go; 362 } 363 } 364 EXPORT_SYMBOL_GPL(kmsan_handle_dma); 365 366 void kmsan_handle_dma_sg(struct scatterlist *sg, int nents, 367 enum dma_data_direction dir) 368 { 369 struct scatterlist *item; 370 int i; 371 372 for_each_sg(sg, item, nents, i) 373 kmsan_handle_dma(sg_phys(item), item->length, dir); 374 } 375 376 /* Functions from kmsan-checks.h follow. */ 377 378 /* 379 * To create an origin, kmsan_poison_memory() unwinds the stacks and stores it 380 * into the stack depot. This may cause deadlocks if done from within KMSAN 381 * runtime, therefore we bail out if kmsan_in_runtime(). 382 */ 383 void kmsan_poison_memory(const void *address, size_t size, gfp_t flags) 384 { 385 if (!kmsan_enabled || kmsan_in_runtime()) 386 return; 387 kmsan_enter_runtime(); 388 /* The users may want to poison/unpoison random memory. */ 389 kmsan_internal_poison_memory((void *)address, size, flags, 390 KMSAN_POISON_NOCHECK); 391 kmsan_leave_runtime(); 392 } 393 EXPORT_SYMBOL(kmsan_poison_memory); 394 395 /* 396 * Unlike kmsan_poison_memory(), this function can be used from within KMSAN 397 * runtime, because it does not trigger allocations or call instrumented code. 398 */ 399 void kmsan_unpoison_memory(const void *address, size_t size) 400 { 401 unsigned long ua_flags; 402 403 if (!kmsan_enabled) 404 return; 405 406 ua_flags = user_access_save(); 407 /* The users may want to poison/unpoison random memory. */ 408 kmsan_internal_unpoison_memory((void *)address, size, 409 KMSAN_POISON_NOCHECK); 410 user_access_restore(ua_flags); 411 } 412 EXPORT_SYMBOL(kmsan_unpoison_memory); 413 414 /* 415 * Version of kmsan_unpoison_memory() called from IRQ entry functions. 416 */ 417 void kmsan_unpoison_entry_regs(const struct pt_regs *regs) 418 { 419 kmsan_unpoison_memory((void *)regs, sizeof(*regs)); 420 } 421 422 void kmsan_check_memory(const void *addr, size_t size) 423 { 424 if (!kmsan_enabled) 425 return; 426 return kmsan_internal_check_memory((void *)addr, size, 427 /*user_addr*/ NULL, REASON_ANY); 428 } 429 EXPORT_SYMBOL(kmsan_check_memory); 430 431 void kmsan_enable_current(void) 432 { 433 KMSAN_WARN_ON(current->kmsan_ctx.depth == 0); 434 current->kmsan_ctx.depth--; 435 } 436 EXPORT_SYMBOL(kmsan_enable_current); 437 438 void kmsan_disable_current(void) 439 { 440 current->kmsan_ctx.depth++; 441 KMSAN_WARN_ON(current->kmsan_ctx.depth == 0); 442 } 443 EXPORT_SYMBOL(kmsan_disable_current); 444