12e903b91SAndrey Konovalov // SPDX-License-Identifier: GPL-2.0 22e903b91SAndrey Konovalov /* 32e903b91SAndrey Konovalov * This file contains core hardware tag-based KASAN code. 42e903b91SAndrey Konovalov * 52e903b91SAndrey Konovalov * Copyright (c) 2020 Google, Inc. 62e903b91SAndrey Konovalov * Author: Andrey Konovalov <andreyknvl@google.com> 72e903b91SAndrey Konovalov */ 82e903b91SAndrey Konovalov 92e903b91SAndrey Konovalov #define pr_fmt(fmt) "kasan: " fmt 102e903b91SAndrey Konovalov 118028caacSAndrey Konovalov #include <linux/init.h> 122e903b91SAndrey Konovalov #include <linux/kasan.h> 132e903b91SAndrey Konovalov #include <linux/kernel.h> 142e903b91SAndrey Konovalov #include <linux/memory.h> 152e903b91SAndrey Konovalov #include <linux/mm.h> 168028caacSAndrey Konovalov #include <linux/static_key.h> 172e903b91SAndrey Konovalov #include <linux/string.h> 182e903b91SAndrey Konovalov #include <linux/types.h> 19*0069455bSKent Overstreet #include <linux/vmalloc.h> 202e903b91SAndrey Konovalov 212e903b91SAndrey Konovalov #include "kasan.h" 222e903b91SAndrey Konovalov 2376bc99e8SAndrey Konovalov enum kasan_arg { 2476bc99e8SAndrey Konovalov KASAN_ARG_DEFAULT, 2576bc99e8SAndrey Konovalov KASAN_ARG_OFF, 2676bc99e8SAndrey Konovalov KASAN_ARG_ON, 278028caacSAndrey Konovalov }; 288028caacSAndrey Konovalov 292603f8a7SVincenzo Frascino enum kasan_arg_mode { 302603f8a7SVincenzo Frascino KASAN_ARG_MODE_DEFAULT, 312603f8a7SVincenzo Frascino KASAN_ARG_MODE_SYNC, 322603f8a7SVincenzo Frascino KASAN_ARG_MODE_ASYNC, 332d27e585SVincenzo Frascino KASAN_ARG_MODE_ASYMM, 342603f8a7SVincenzo Frascino }; 352603f8a7SVincenzo Frascino 36551b2bcbSAndrey Konovalov enum kasan_arg_vmalloc { 37551b2bcbSAndrey Konovalov KASAN_ARG_VMALLOC_DEFAULT, 38551b2bcbSAndrey Konovalov KASAN_ARG_VMALLOC_OFF, 39551b2bcbSAndrey Konovalov KASAN_ARG_VMALLOC_ON, 40551b2bcbSAndrey Konovalov }; 41551b2bcbSAndrey Konovalov 4276bc99e8SAndrey Konovalov static enum kasan_arg kasan_arg __ro_after_init; 432603f8a7SVincenzo Frascino static enum kasan_arg_mode kasan_arg_mode __ro_after_init; 44551b2bcbSAndrey Konovalov static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata; 458028caacSAndrey Konovalov 46241944d1SAndrey Konovalov /* 47241944d1SAndrey Konovalov * Whether KASAN is enabled at all. 48241944d1SAndrey Konovalov * The value remains false until KASAN is initialized by kasan_init_hw_tags(). 49241944d1SAndrey Konovalov */ 508028caacSAndrey Konovalov DEFINE_STATIC_KEY_FALSE(kasan_flag_enabled); 518028caacSAndrey Konovalov EXPORT_SYMBOL(kasan_flag_enabled); 528028caacSAndrey Konovalov 53241944d1SAndrey Konovalov /* 54241944d1SAndrey Konovalov * Whether the selected mode is synchronous, asynchronous, or asymmetric. 55241944d1SAndrey Konovalov * Defaults to KASAN_MODE_SYNC. 56241944d1SAndrey Konovalov */ 572d27e585SVincenzo Frascino enum kasan_mode kasan_mode __ro_after_init; 582d27e585SVincenzo Frascino EXPORT_SYMBOL_GPL(kasan_mode); 592603f8a7SVincenzo Frascino 60551b2bcbSAndrey Konovalov /* Whether to enable vmalloc tagging. */ 6158ee788cSAndrey Konovalov #ifdef CONFIG_KASAN_VMALLOC 62551b2bcbSAndrey Konovalov DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc); 6358ee788cSAndrey Konovalov #else 6458ee788cSAndrey Konovalov DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc); 6558ee788cSAndrey Konovalov #endif 6614c99b99SAndrey Konovalov EXPORT_SYMBOL_GPL(kasan_flag_vmalloc); 67551b2bcbSAndrey Konovalov 6844383cefSAndrey Konovalov #define PAGE_ALLOC_SAMPLE_DEFAULT 1 6944383cefSAndrey Konovalov #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3 7044383cefSAndrey Konovalov 7144383cefSAndrey Konovalov /* 7244383cefSAndrey Konovalov * Sampling interval of page_alloc allocation (un)poisoning. 7344383cefSAndrey Konovalov * Defaults to no sampling. 7444383cefSAndrey Konovalov */ 7544383cefSAndrey Konovalov unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT; 7644383cefSAndrey Konovalov 7744383cefSAndrey Konovalov /* 7844383cefSAndrey Konovalov * Minimum order of page_alloc allocations to be affected by sampling. 7944383cefSAndrey Konovalov * The default value is chosen to match both 8044383cefSAndrey Konovalov * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER. 8144383cefSAndrey Konovalov */ 8244383cefSAndrey Konovalov unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT; 8344383cefSAndrey Konovalov 8444383cefSAndrey Konovalov DEFINE_PER_CPU(long, kasan_page_alloc_skip); 8544383cefSAndrey Konovalov 8676bc99e8SAndrey Konovalov /* kasan=off/on */ 8776bc99e8SAndrey Konovalov static int __init early_kasan_flag(char *arg) 888028caacSAndrey Konovalov { 898028caacSAndrey Konovalov if (!arg) 908028caacSAndrey Konovalov return -EINVAL; 918028caacSAndrey Konovalov 928028caacSAndrey Konovalov if (!strcmp(arg, "off")) 9376bc99e8SAndrey Konovalov kasan_arg = KASAN_ARG_OFF; 9476bc99e8SAndrey Konovalov else if (!strcmp(arg, "on")) 9576bc99e8SAndrey Konovalov kasan_arg = KASAN_ARG_ON; 968028caacSAndrey Konovalov else 978028caacSAndrey Konovalov return -EINVAL; 988028caacSAndrey Konovalov 998028caacSAndrey Konovalov return 0; 1008028caacSAndrey Konovalov } 10176bc99e8SAndrey Konovalov early_param("kasan", early_kasan_flag); 1028028caacSAndrey Konovalov 1032d27e585SVincenzo Frascino /* kasan.mode=sync/async/asymm */ 1042603f8a7SVincenzo Frascino static int __init early_kasan_mode(char *arg) 1052603f8a7SVincenzo Frascino { 1062603f8a7SVincenzo Frascino if (!arg) 1072603f8a7SVincenzo Frascino return -EINVAL; 1082603f8a7SVincenzo Frascino 1092603f8a7SVincenzo Frascino if (!strcmp(arg, "sync")) 1102603f8a7SVincenzo Frascino kasan_arg_mode = KASAN_ARG_MODE_SYNC; 1112603f8a7SVincenzo Frascino else if (!strcmp(arg, "async")) 1122603f8a7SVincenzo Frascino kasan_arg_mode = KASAN_ARG_MODE_ASYNC; 1132d27e585SVincenzo Frascino else if (!strcmp(arg, "asymm")) 1142d27e585SVincenzo Frascino kasan_arg_mode = KASAN_ARG_MODE_ASYMM; 1152603f8a7SVincenzo Frascino else 1162603f8a7SVincenzo Frascino return -EINVAL; 1172603f8a7SVincenzo Frascino 1182603f8a7SVincenzo Frascino return 0; 1192603f8a7SVincenzo Frascino } 1202603f8a7SVincenzo Frascino early_param("kasan.mode", early_kasan_mode); 1212603f8a7SVincenzo Frascino 122551b2bcbSAndrey Konovalov /* kasan.vmalloc=off/on */ 123551b2bcbSAndrey Konovalov static int __init early_kasan_flag_vmalloc(char *arg) 124551b2bcbSAndrey Konovalov { 125551b2bcbSAndrey Konovalov if (!arg) 126551b2bcbSAndrey Konovalov return -EINVAL; 127551b2bcbSAndrey Konovalov 12858ee788cSAndrey Konovalov if (!IS_ENABLED(CONFIG_KASAN_VMALLOC)) 12958ee788cSAndrey Konovalov return 0; 13058ee788cSAndrey Konovalov 131551b2bcbSAndrey Konovalov if (!strcmp(arg, "off")) 132551b2bcbSAndrey Konovalov kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF; 133551b2bcbSAndrey Konovalov else if (!strcmp(arg, "on")) 134551b2bcbSAndrey Konovalov kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON; 135551b2bcbSAndrey Konovalov else 136551b2bcbSAndrey Konovalov return -EINVAL; 137551b2bcbSAndrey Konovalov 138551b2bcbSAndrey Konovalov return 0; 139551b2bcbSAndrey Konovalov } 140551b2bcbSAndrey Konovalov early_param("kasan.vmalloc", early_kasan_flag_vmalloc); 141551b2bcbSAndrey Konovalov 142b873e986SKuan-Ying Lee static inline const char *kasan_mode_info(void) 143b873e986SKuan-Ying Lee { 144b873e986SKuan-Ying Lee if (kasan_mode == KASAN_MODE_ASYNC) 145b873e986SKuan-Ying Lee return "async"; 146b873e986SKuan-Ying Lee else if (kasan_mode == KASAN_MODE_ASYMM) 147b873e986SKuan-Ying Lee return "asymm"; 148b873e986SKuan-Ying Lee else 149b873e986SKuan-Ying Lee return "sync"; 150b873e986SKuan-Ying Lee } 151b873e986SKuan-Ying Lee 15244383cefSAndrey Konovalov /* kasan.page_alloc.sample=<sampling interval> */ 15344383cefSAndrey Konovalov static int __init early_kasan_flag_page_alloc_sample(char *arg) 15444383cefSAndrey Konovalov { 15544383cefSAndrey Konovalov int rv; 15644383cefSAndrey Konovalov 15744383cefSAndrey Konovalov if (!arg) 15844383cefSAndrey Konovalov return -EINVAL; 15944383cefSAndrey Konovalov 16044383cefSAndrey Konovalov rv = kstrtoul(arg, 0, &kasan_page_alloc_sample); 16144383cefSAndrey Konovalov if (rv) 16244383cefSAndrey Konovalov return rv; 16344383cefSAndrey Konovalov 16444383cefSAndrey Konovalov if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) { 16544383cefSAndrey Konovalov kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT; 16644383cefSAndrey Konovalov return -EINVAL; 16744383cefSAndrey Konovalov } 16844383cefSAndrey Konovalov 16944383cefSAndrey Konovalov return 0; 17044383cefSAndrey Konovalov } 17144383cefSAndrey Konovalov early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample); 17244383cefSAndrey Konovalov 17344383cefSAndrey Konovalov /* kasan.page_alloc.sample.order=<minimum page order> */ 17444383cefSAndrey Konovalov static int __init early_kasan_flag_page_alloc_sample_order(char *arg) 17544383cefSAndrey Konovalov { 17644383cefSAndrey Konovalov int rv; 17744383cefSAndrey Konovalov 17844383cefSAndrey Konovalov if (!arg) 17944383cefSAndrey Konovalov return -EINVAL; 18044383cefSAndrey Konovalov 18144383cefSAndrey Konovalov rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order); 18244383cefSAndrey Konovalov if (rv) 18344383cefSAndrey Konovalov return rv; 18444383cefSAndrey Konovalov 18544383cefSAndrey Konovalov if (kasan_page_alloc_sample_order > INT_MAX) { 18644383cefSAndrey Konovalov kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT; 18744383cefSAndrey Konovalov return -EINVAL; 18844383cefSAndrey Konovalov } 18944383cefSAndrey Konovalov 19044383cefSAndrey Konovalov return 0; 19144383cefSAndrey Konovalov } 19244383cefSAndrey Konovalov early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order); 19344383cefSAndrey Konovalov 1941eeac51eSAndrey Konovalov /* 1951eeac51eSAndrey Konovalov * kasan_init_hw_tags_cpu() is called for each CPU. 1961eeac51eSAndrey Konovalov * Not marked as __init as a CPU can be hot-plugged after boot. 1971eeac51eSAndrey Konovalov */ 1982e903b91SAndrey Konovalov void kasan_init_hw_tags_cpu(void) 1992e903b91SAndrey Konovalov { 2008028caacSAndrey Konovalov /* 2018028caacSAndrey Konovalov * There's no need to check that the hardware is MTE-capable here, 2028028caacSAndrey Konovalov * as this function is only called for MTE-capable hardware. 2038028caacSAndrey Konovalov */ 2048028caacSAndrey Konovalov 205241944d1SAndrey Konovalov /* 206241944d1SAndrey Konovalov * If KASAN is disabled via command line, don't initialize it. 207241944d1SAndrey Konovalov * When this function is called, kasan_flag_enabled is not yet 208241944d1SAndrey Konovalov * set by kasan_init_hw_tags(). Thus, check kasan_arg instead. 209241944d1SAndrey Konovalov */ 21076bc99e8SAndrey Konovalov if (kasan_arg == KASAN_ARG_OFF) 2118028caacSAndrey Konovalov return; 2128028caacSAndrey Konovalov 2132603f8a7SVincenzo Frascino /* 2142d27e585SVincenzo Frascino * Enable async or asymm modes only when explicitly requested 2152d27e585SVincenzo Frascino * through the command line. 2162603f8a7SVincenzo Frascino */ 2170eafff1cSAndrey Konovalov kasan_enable_hw_tags(); 2182e903b91SAndrey Konovalov } 2192e903b91SAndrey Konovalov 2202e903b91SAndrey Konovalov /* kasan_init_hw_tags() is called once on boot CPU. */ 2212e903b91SAndrey Konovalov void __init kasan_init_hw_tags(void) 2222e903b91SAndrey Konovalov { 22376bc99e8SAndrey Konovalov /* If hardware doesn't support MTE, don't initialize KASAN. */ 2248028caacSAndrey Konovalov if (!system_supports_mte()) 2258028caacSAndrey Konovalov return; 2268028caacSAndrey Konovalov 22776bc99e8SAndrey Konovalov /* If KASAN is disabled via command line, don't initialize it. */ 22876bc99e8SAndrey Konovalov if (kasan_arg == KASAN_ARG_OFF) 2298028caacSAndrey Konovalov return; 2308028caacSAndrey Konovalov 2312603f8a7SVincenzo Frascino switch (kasan_arg_mode) { 2322603f8a7SVincenzo Frascino case KASAN_ARG_MODE_DEFAULT: 233241944d1SAndrey Konovalov /* Default is specified by kasan_mode definition. */ 234241944d1SAndrey Konovalov break; 2352603f8a7SVincenzo Frascino case KASAN_ARG_MODE_SYNC: 2362d27e585SVincenzo Frascino kasan_mode = KASAN_MODE_SYNC; 2372603f8a7SVincenzo Frascino break; 2382603f8a7SVincenzo Frascino case KASAN_ARG_MODE_ASYNC: 2392d27e585SVincenzo Frascino kasan_mode = KASAN_MODE_ASYNC; 2402d27e585SVincenzo Frascino break; 2412d27e585SVincenzo Frascino case KASAN_ARG_MODE_ASYMM: 2422d27e585SVincenzo Frascino kasan_mode = KASAN_MODE_ASYMM; 2432603f8a7SVincenzo Frascino break; 2442603f8a7SVincenzo Frascino } 2452603f8a7SVincenzo Frascino 246551b2bcbSAndrey Konovalov switch (kasan_arg_vmalloc) { 247551b2bcbSAndrey Konovalov case KASAN_ARG_VMALLOC_DEFAULT: 248551b2bcbSAndrey Konovalov /* Default is specified by kasan_flag_vmalloc definition. */ 249551b2bcbSAndrey Konovalov break; 250551b2bcbSAndrey Konovalov case KASAN_ARG_VMALLOC_OFF: 251551b2bcbSAndrey Konovalov static_branch_disable(&kasan_flag_vmalloc); 252551b2bcbSAndrey Konovalov break; 253551b2bcbSAndrey Konovalov case KASAN_ARG_VMALLOC_ON: 254551b2bcbSAndrey Konovalov static_branch_enable(&kasan_flag_vmalloc); 255551b2bcbSAndrey Konovalov break; 256551b2bcbSAndrey Konovalov } 257551b2bcbSAndrey Konovalov 2587ebfce33SAndrey Konovalov kasan_init_tags(); 2598028caacSAndrey Konovalov 260241944d1SAndrey Konovalov /* KASAN is now initialized, enable it. */ 261241944d1SAndrey Konovalov static_branch_enable(&kasan_flag_enabled); 262241944d1SAndrey Konovalov 263551b2bcbSAndrey Konovalov pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s)\n", 264b873e986SKuan-Ying Lee kasan_mode_info(), 265551b2bcbSAndrey Konovalov kasan_vmalloc_enabled() ? "on" : "off", 266b873e986SKuan-Ying Lee kasan_stack_collection_enabled() ? "on" : "off"); 2672e903b91SAndrey Konovalov } 2682e903b91SAndrey Konovalov 26923689e91SAndrey Konovalov #ifdef CONFIG_KASAN_VMALLOC 27023689e91SAndrey Konovalov 27123689e91SAndrey Konovalov static void unpoison_vmalloc_pages(const void *addr, u8 tag) 27223689e91SAndrey Konovalov { 27323689e91SAndrey Konovalov struct vm_struct *area; 27423689e91SAndrey Konovalov int i; 27523689e91SAndrey Konovalov 27623689e91SAndrey Konovalov /* 27723689e91SAndrey Konovalov * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations 27823689e91SAndrey Konovalov * (see the comment in __kasan_unpoison_vmalloc), all of the pages 27923689e91SAndrey Konovalov * should belong to a single area. 28023689e91SAndrey Konovalov */ 28123689e91SAndrey Konovalov area = find_vm_area((void *)addr); 28223689e91SAndrey Konovalov if (WARN_ON(!area)) 28323689e91SAndrey Konovalov return; 28423689e91SAndrey Konovalov 28523689e91SAndrey Konovalov for (i = 0; i < area->nr_pages; i++) { 28623689e91SAndrey Konovalov struct page *page = area->pages[i]; 28723689e91SAndrey Konovalov 28823689e91SAndrey Konovalov page_kasan_tag_set(page, tag); 28923689e91SAndrey Konovalov } 29023689e91SAndrey Konovalov } 29123689e91SAndrey Konovalov 2926c2f761dSAndrey Konovalov static void init_vmalloc_pages(const void *start, unsigned long size) 2936c2f761dSAndrey Konovalov { 2946c2f761dSAndrey Konovalov const void *addr; 2956c2f761dSAndrey Konovalov 2966c2f761dSAndrey Konovalov for (addr = start; addr < start + size; addr += PAGE_SIZE) { 29729083fd8SMark Rutland struct page *page = vmalloc_to_page(addr); 2986c2f761dSAndrey Konovalov 2996c2f761dSAndrey Konovalov clear_highpage_kasan_tagged(page); 3006c2f761dSAndrey Konovalov } 3016c2f761dSAndrey Konovalov } 3026c2f761dSAndrey Konovalov 30323689e91SAndrey Konovalov void *__kasan_unpoison_vmalloc(const void *start, unsigned long size, 30423689e91SAndrey Konovalov kasan_vmalloc_flags_t flags) 30523689e91SAndrey Konovalov { 30623689e91SAndrey Konovalov u8 tag; 30723689e91SAndrey Konovalov unsigned long redzone_start, redzone_size; 30823689e91SAndrey Konovalov 30929083fd8SMark Rutland if (!kasan_vmalloc_enabled()) { 3106c2f761dSAndrey Konovalov if (flags & KASAN_VMALLOC_INIT) 3116c2f761dSAndrey Konovalov init_vmalloc_pages(start, size); 312551b2bcbSAndrey Konovalov return (void *)start; 3136c2f761dSAndrey Konovalov } 31423689e91SAndrey Konovalov 31523689e91SAndrey Konovalov /* 3166c2f761dSAndrey Konovalov * Don't tag non-VM_ALLOC mappings, as: 31723689e91SAndrey Konovalov * 31823689e91SAndrey Konovalov * 1. Unlike the software KASAN modes, hardware tag-based KASAN only 31923689e91SAndrey Konovalov * supports tagging physical memory. Therefore, it can only tag a 32023689e91SAndrey Konovalov * single mapping of normal physical pages. 32123689e91SAndrey Konovalov * 2. Hardware tag-based KASAN can only tag memory mapped with special 3226c2f761dSAndrey Konovalov * mapping protection bits, see arch_vmap_pgprot_tagged(). 32323689e91SAndrey Konovalov * As non-VM_ALLOC mappings can be mapped outside of vmalloc code, 32423689e91SAndrey Konovalov * providing these bits would require tracking all non-VM_ALLOC 32523689e91SAndrey Konovalov * mappers. 32623689e91SAndrey Konovalov * 32723689e91SAndrey Konovalov * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags 32823689e91SAndrey Konovalov * the first virtual mapping, which is created by vmalloc(). 32923689e91SAndrey Konovalov * Tagging the page_alloc memory backing that vmalloc() allocation is 3300a54864fSPeter Collingbourne * skipped, see ___GFP_SKIP_KASAN. 33123689e91SAndrey Konovalov * 33223689e91SAndrey Konovalov * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual. 33323689e91SAndrey Konovalov */ 3346c2f761dSAndrey Konovalov if (!(flags & KASAN_VMALLOC_VM_ALLOC)) { 3356c2f761dSAndrey Konovalov WARN_ON(flags & KASAN_VMALLOC_INIT); 33623689e91SAndrey Konovalov return (void *)start; 3376c2f761dSAndrey Konovalov } 33823689e91SAndrey Konovalov 339f6e39794SAndrey Konovalov /* 340f6e39794SAndrey Konovalov * Don't tag executable memory. 341f6e39794SAndrey Konovalov * The kernel doesn't tolerate having the PC register tagged. 342f6e39794SAndrey Konovalov */ 3436c2f761dSAndrey Konovalov if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) { 3446c2f761dSAndrey Konovalov WARN_ON(flags & KASAN_VMALLOC_INIT); 345f6e39794SAndrey Konovalov return (void *)start; 3466c2f761dSAndrey Konovalov } 347f6e39794SAndrey Konovalov 34823689e91SAndrey Konovalov tag = kasan_random_tag(); 34923689e91SAndrey Konovalov start = set_tag(start, tag); 35023689e91SAndrey Konovalov 35123689e91SAndrey Konovalov /* Unpoison and initialize memory up to size. */ 35223689e91SAndrey Konovalov kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT); 35323689e91SAndrey Konovalov 35423689e91SAndrey Konovalov /* 35523689e91SAndrey Konovalov * Explicitly poison and initialize the in-page vmalloc() redzone. 35623689e91SAndrey Konovalov * Unlike software KASAN modes, hardware tag-based KASAN doesn't 35723689e91SAndrey Konovalov * unpoison memory when populating shadow for vmalloc() space. 35823689e91SAndrey Konovalov */ 35923689e91SAndrey Konovalov redzone_start = round_up((unsigned long)start + size, 36023689e91SAndrey Konovalov KASAN_GRANULE_SIZE); 36123689e91SAndrey Konovalov redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start; 36223689e91SAndrey Konovalov kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID, 36323689e91SAndrey Konovalov flags & KASAN_VMALLOC_INIT); 36423689e91SAndrey Konovalov 36523689e91SAndrey Konovalov /* 36623689e91SAndrey Konovalov * Set per-page tag flags to allow accessing physical memory for the 36723689e91SAndrey Konovalov * vmalloc() mapping through page_address(vmalloc_to_page()). 36823689e91SAndrey Konovalov */ 36923689e91SAndrey Konovalov unpoison_vmalloc_pages(start, tag); 37023689e91SAndrey Konovalov 37123689e91SAndrey Konovalov return (void *)start; 37223689e91SAndrey Konovalov } 37323689e91SAndrey Konovalov 37423689e91SAndrey Konovalov void __kasan_poison_vmalloc(const void *start, unsigned long size) 37523689e91SAndrey Konovalov { 37623689e91SAndrey Konovalov /* 37723689e91SAndrey Konovalov * No tagging here. 37823689e91SAndrey Konovalov * The physical pages backing the vmalloc() allocation are poisoned 37923689e91SAndrey Konovalov * through the usual page_alloc paths. 38023689e91SAndrey Konovalov */ 38123689e91SAndrey Konovalov } 38223689e91SAndrey Konovalov 38323689e91SAndrey Konovalov #endif 38423689e91SAndrey Konovalov 3850eafff1cSAndrey Konovalov void kasan_enable_hw_tags(void) 386f05842cfSAndrey Konovalov { 387ed6d7444SAndrey Konovalov if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC) 3880eafff1cSAndrey Konovalov hw_enable_tag_checks_async(); 389ed6d7444SAndrey Konovalov else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM) 3900eafff1cSAndrey Konovalov hw_enable_tag_checks_asymm(); 391ed6d7444SAndrey Konovalov else 3920eafff1cSAndrey Konovalov hw_enable_tag_checks_sync(); 393f05842cfSAndrey Konovalov } 394b1add418SVincenzo Frascino 395b1add418SVincenzo Frascino #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST) 396b1add418SVincenzo Frascino 3970eafff1cSAndrey Konovalov EXPORT_SYMBOL_GPL(kasan_enable_hw_tags); 398f05842cfSAndrey Konovalov 399e80a76aaSAndrey Konovalov void kasan_force_async_fault(void) 400e80a76aaSAndrey Konovalov { 401e80a76aaSAndrey Konovalov hw_force_async_tag_fault(); 402e80a76aaSAndrey Konovalov } 403e80a76aaSAndrey Konovalov EXPORT_SYMBOL_GPL(kasan_force_async_fault); 404e80a76aaSAndrey Konovalov 405f05842cfSAndrey Konovalov #endif 406