1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * This file contains core hardware tag-based KASAN code.
4 *
5 * Copyright (c) 2020 Google, Inc.
6 * Author: Andrey Konovalov <andreyknvl@google.com>
7 */
8
9 #define pr_fmt(fmt) "kasan: " fmt
10
11 #include <kunit/visibility.h>
12 #include <linux/init.h>
13 #include <linux/kasan.h>
14 #include <linux/kernel.h>
15 #include <linux/memory.h>
16 #include <linux/mm.h>
17 #include <linux/static_key.h>
18 #include <linux/string.h>
19 #include <linux/string_choices.h>
20 #include <linux/types.h>
21 #include <linux/vmalloc.h>
22
23 #include "kasan.h"
24
25 enum kasan_arg {
26 KASAN_ARG_DEFAULT,
27 KASAN_ARG_OFF,
28 KASAN_ARG_ON,
29 };
30
31 enum kasan_arg_mode {
32 KASAN_ARG_MODE_DEFAULT,
33 KASAN_ARG_MODE_SYNC,
34 KASAN_ARG_MODE_ASYNC,
35 KASAN_ARG_MODE_ASYMM,
36 };
37
38 enum kasan_arg_vmalloc {
39 KASAN_ARG_VMALLOC_DEFAULT,
40 KASAN_ARG_VMALLOC_OFF,
41 KASAN_ARG_VMALLOC_ON,
42 };
43
44 static enum kasan_arg kasan_arg __ro_after_init;
45 static enum kasan_arg_mode kasan_arg_mode __ro_after_init;
46 static enum kasan_arg_vmalloc kasan_arg_vmalloc __initdata;
47
48 /*
49 * Whether the selected mode is synchronous, asynchronous, or asymmetric.
50 * Defaults to KASAN_MODE_SYNC.
51 */
52 enum kasan_mode kasan_mode __ro_after_init;
53 EXPORT_SYMBOL_GPL(kasan_mode);
54
55 /* Whether to enable vmalloc tagging. */
56 #ifdef CONFIG_KASAN_VMALLOC
57 DEFINE_STATIC_KEY_TRUE(kasan_flag_vmalloc);
58 #else
59 DEFINE_STATIC_KEY_FALSE(kasan_flag_vmalloc);
60 #endif
61 EXPORT_SYMBOL_GPL(kasan_flag_vmalloc);
62
63 /* Whether to check write accesses only. */
64 static bool kasan_flag_write_only = false;
65
66 #define PAGE_ALLOC_SAMPLE_DEFAULT 1
67 #define PAGE_ALLOC_SAMPLE_ORDER_DEFAULT 3
68
69 /*
70 * Sampling interval of page_alloc allocation (un)poisoning.
71 * Defaults to no sampling.
72 */
73 unsigned long kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
74
75 /*
76 * Minimum order of page_alloc allocations to be affected by sampling.
77 * The default value is chosen to match both
78 * PAGE_ALLOC_COSTLY_ORDER and SKB_FRAG_PAGE_ORDER.
79 */
80 unsigned int kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
81
82 DEFINE_PER_CPU(long, kasan_page_alloc_skip);
83
84 /* kasan=off/on */
early_kasan_flag(char * arg)85 static int __init early_kasan_flag(char *arg)
86 {
87 if (!arg)
88 return -EINVAL;
89
90 if (!strcmp(arg, "off"))
91 kasan_arg = KASAN_ARG_OFF;
92 else if (!strcmp(arg, "on"))
93 kasan_arg = KASAN_ARG_ON;
94 else
95 return -EINVAL;
96
97 return 0;
98 }
99 early_param("kasan", early_kasan_flag);
100
101 /* kasan.mode=sync/async/asymm */
early_kasan_mode(char * arg)102 static int __init early_kasan_mode(char *arg)
103 {
104 if (!arg)
105 return -EINVAL;
106
107 if (!strcmp(arg, "sync"))
108 kasan_arg_mode = KASAN_ARG_MODE_SYNC;
109 else if (!strcmp(arg, "async"))
110 kasan_arg_mode = KASAN_ARG_MODE_ASYNC;
111 else if (!strcmp(arg, "asymm"))
112 kasan_arg_mode = KASAN_ARG_MODE_ASYMM;
113 else
114 return -EINVAL;
115
116 return 0;
117 }
118 early_param("kasan.mode", early_kasan_mode);
119
120 /* kasan.vmalloc=off/on */
early_kasan_flag_vmalloc(char * arg)121 static int __init early_kasan_flag_vmalloc(char *arg)
122 {
123 if (!arg)
124 return -EINVAL;
125
126 if (!IS_ENABLED(CONFIG_KASAN_VMALLOC))
127 return 0;
128
129 if (!strcmp(arg, "off"))
130 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_OFF;
131 else if (!strcmp(arg, "on"))
132 kasan_arg_vmalloc = KASAN_ARG_VMALLOC_ON;
133 else
134 return -EINVAL;
135
136 return 0;
137 }
138 early_param("kasan.vmalloc", early_kasan_flag_vmalloc);
139
140 /* kasan.write_only=off/on */
early_kasan_flag_write_only(char * arg)141 static int __init early_kasan_flag_write_only(char *arg)
142 {
143 if (!arg)
144 return -EINVAL;
145
146 if (!strcmp(arg, "off"))
147 kasan_flag_write_only = false;
148 else if (!strcmp(arg, "on"))
149 kasan_flag_write_only = true;
150 else
151 return -EINVAL;
152
153 return 0;
154 }
155 early_param("kasan.write_only", early_kasan_flag_write_only);
156
kasan_mode_info(void)157 static inline const char *kasan_mode_info(void)
158 {
159 if (kasan_mode == KASAN_MODE_ASYNC)
160 return "async";
161 else if (kasan_mode == KASAN_MODE_ASYMM)
162 return "asymm";
163 else
164 return "sync";
165 }
166
167 /* kasan.page_alloc.sample=<sampling interval> */
early_kasan_flag_page_alloc_sample(char * arg)168 static int __init early_kasan_flag_page_alloc_sample(char *arg)
169 {
170 int rv;
171
172 if (!arg)
173 return -EINVAL;
174
175 rv = kstrtoul(arg, 0, &kasan_page_alloc_sample);
176 if (rv)
177 return rv;
178
179 if (!kasan_page_alloc_sample || kasan_page_alloc_sample > LONG_MAX) {
180 kasan_page_alloc_sample = PAGE_ALLOC_SAMPLE_DEFAULT;
181 return -EINVAL;
182 }
183
184 return 0;
185 }
186 early_param("kasan.page_alloc.sample", early_kasan_flag_page_alloc_sample);
187
188 /* kasan.page_alloc.sample.order=<minimum page order> */
early_kasan_flag_page_alloc_sample_order(char * arg)189 static int __init early_kasan_flag_page_alloc_sample_order(char *arg)
190 {
191 int rv;
192
193 if (!arg)
194 return -EINVAL;
195
196 rv = kstrtouint(arg, 0, &kasan_page_alloc_sample_order);
197 if (rv)
198 return rv;
199
200 if (kasan_page_alloc_sample_order > INT_MAX) {
201 kasan_page_alloc_sample_order = PAGE_ALLOC_SAMPLE_ORDER_DEFAULT;
202 return -EINVAL;
203 }
204
205 return 0;
206 }
207 early_param("kasan.page_alloc.sample.order", early_kasan_flag_page_alloc_sample_order);
208
209 /*
210 * kasan_init_hw_tags_cpu() is called for each CPU.
211 * Not marked as __init as a CPU can be hot-plugged after boot.
212 */
kasan_init_hw_tags_cpu(void)213 void kasan_init_hw_tags_cpu(void)
214 {
215 /*
216 * There's no need to check that the hardware is MTE-capable here,
217 * as this function is only called for MTE-capable hardware.
218 */
219
220 /*
221 * If KASAN is disabled via command line, don't initialize it.
222 * When this function is called, kasan_flag_enabled is not yet
223 * set by kasan_init_hw_tags(). Thus, check kasan_arg instead.
224 */
225 if (kasan_arg == KASAN_ARG_OFF)
226 return;
227
228 /*
229 * Enable async or asymm modes only when explicitly requested
230 * through the command line.
231 */
232 kasan_enable_hw_tags();
233 }
234
235 /* kasan_init_hw_tags() is called once on boot CPU. */
kasan_init_hw_tags(void)236 void __init kasan_init_hw_tags(void)
237 {
238 /* If hardware doesn't support MTE, don't initialize KASAN. */
239 if (!system_supports_mte())
240 return;
241
242 /* If KASAN is disabled via command line, don't initialize it. */
243 if (kasan_arg == KASAN_ARG_OFF)
244 return;
245
246 switch (kasan_arg_mode) {
247 case KASAN_ARG_MODE_DEFAULT:
248 /* Default is specified by kasan_mode definition. */
249 break;
250 case KASAN_ARG_MODE_SYNC:
251 kasan_mode = KASAN_MODE_SYNC;
252 break;
253 case KASAN_ARG_MODE_ASYNC:
254 kasan_mode = KASAN_MODE_ASYNC;
255 break;
256 case KASAN_ARG_MODE_ASYMM:
257 kasan_mode = KASAN_MODE_ASYMM;
258 break;
259 }
260
261 switch (kasan_arg_vmalloc) {
262 case KASAN_ARG_VMALLOC_DEFAULT:
263 /* Default is specified by kasan_flag_vmalloc definition. */
264 break;
265 case KASAN_ARG_VMALLOC_OFF:
266 static_branch_disable(&kasan_flag_vmalloc);
267 break;
268 case KASAN_ARG_VMALLOC_ON:
269 static_branch_enable(&kasan_flag_vmalloc);
270 break;
271 }
272
273 kasan_init_tags();
274
275 /* KASAN is now initialized, enable it. */
276 kasan_enable();
277
278 pr_info("KernelAddressSanitizer initialized (hw-tags, mode=%s, vmalloc=%s, stacktrace=%s, write_only=%s)\n",
279 kasan_mode_info(),
280 str_on_off(kasan_vmalloc_enabled()),
281 str_on_off(kasan_stack_collection_enabled()),
282 str_on_off(kasan_flag_write_only));
283 }
284
285 #ifdef CONFIG_KASAN_VMALLOC
286
unpoison_vmalloc_pages(const void * addr,u8 tag)287 static void unpoison_vmalloc_pages(const void *addr, u8 tag)
288 {
289 struct vm_struct *area;
290 int i;
291
292 /*
293 * As hardware tag-based KASAN only tags VM_ALLOC vmalloc allocations
294 * (see the comment in __kasan_unpoison_vmalloc), all of the pages
295 * should belong to a single area.
296 */
297 area = find_vm_area((void *)addr);
298 if (WARN_ON(!area))
299 return;
300
301 for (i = 0; i < area->nr_pages; i++) {
302 struct page *page = area->pages[i];
303
304 page_kasan_tag_set(page, tag);
305 }
306 }
307
init_vmalloc_pages(const void * start,unsigned long size)308 static void init_vmalloc_pages(const void *start, unsigned long size)
309 {
310 const void *addr;
311
312 for (addr = start; addr < start + size; addr += PAGE_SIZE) {
313 struct page *page = vmalloc_to_page(addr);
314
315 clear_highpage_kasan_tagged(page);
316 }
317 }
318
__kasan_unpoison_vmalloc(const void * start,unsigned long size,kasan_vmalloc_flags_t flags)319 void *__kasan_unpoison_vmalloc(const void *start, unsigned long size,
320 kasan_vmalloc_flags_t flags)
321 {
322 u8 tag;
323 unsigned long redzone_start, redzone_size;
324
325 if (!kasan_vmalloc_enabled()) {
326 if (flags & KASAN_VMALLOC_INIT)
327 init_vmalloc_pages(start, size);
328 return (void *)start;
329 }
330
331 /*
332 * Don't tag non-VM_ALLOC mappings, as:
333 *
334 * 1. Unlike the software KASAN modes, hardware tag-based KASAN only
335 * supports tagging physical memory. Therefore, it can only tag a
336 * single mapping of normal physical pages.
337 * 2. Hardware tag-based KASAN can only tag memory mapped with special
338 * mapping protection bits, see arch_vmap_pgprot_tagged().
339 * As non-VM_ALLOC mappings can be mapped outside of vmalloc code,
340 * providing these bits would require tracking all non-VM_ALLOC
341 * mappers.
342 *
343 * Thus, for VM_ALLOC mappings, hardware tag-based KASAN only tags
344 * the first virtual mapping, which is created by vmalloc().
345 * Tagging the page_alloc memory backing that vmalloc() allocation is
346 * skipped, see ___GFP_SKIP_KASAN.
347 *
348 * For non-VM_ALLOC allocations, page_alloc memory is tagged as usual.
349 */
350 if (!(flags & KASAN_VMALLOC_VM_ALLOC)) {
351 WARN_ON(flags & KASAN_VMALLOC_INIT);
352 return (void *)start;
353 }
354
355 /*
356 * Don't tag executable memory.
357 * The kernel doesn't tolerate having the PC register tagged.
358 */
359 if (!(flags & KASAN_VMALLOC_PROT_NORMAL)) {
360 WARN_ON(flags & KASAN_VMALLOC_INIT);
361 return (void *)start;
362 }
363
364 tag = kasan_random_tag();
365 start = set_tag(start, tag);
366
367 /* Unpoison and initialize memory up to size. */
368 kasan_unpoison(start, size, flags & KASAN_VMALLOC_INIT);
369
370 /*
371 * Explicitly poison and initialize the in-page vmalloc() redzone.
372 * Unlike software KASAN modes, hardware tag-based KASAN doesn't
373 * unpoison memory when populating shadow for vmalloc() space.
374 */
375 redzone_start = round_up((unsigned long)start + size,
376 KASAN_GRANULE_SIZE);
377 redzone_size = round_up(redzone_start, PAGE_SIZE) - redzone_start;
378 kasan_poison((void *)redzone_start, redzone_size, KASAN_TAG_INVALID,
379 flags & KASAN_VMALLOC_INIT);
380
381 /*
382 * Set per-page tag flags to allow accessing physical memory for the
383 * vmalloc() mapping through page_address(vmalloc_to_page()).
384 */
385 unpoison_vmalloc_pages(start, tag);
386
387 return (void *)start;
388 }
389
__kasan_poison_vmalloc(const void * start,unsigned long size)390 void __kasan_poison_vmalloc(const void *start, unsigned long size)
391 {
392 /*
393 * No tagging here.
394 * The physical pages backing the vmalloc() allocation are poisoned
395 * through the usual page_alloc paths.
396 */
397 }
398
399 #endif
400
kasan_enable_hw_tags(void)401 void kasan_enable_hw_tags(void)
402 {
403 if (kasan_arg_mode == KASAN_ARG_MODE_ASYNC)
404 hw_enable_tag_checks_async();
405 else if (kasan_arg_mode == KASAN_ARG_MODE_ASYMM)
406 hw_enable_tag_checks_asymm();
407 else
408 hw_enable_tag_checks_sync();
409
410 /*
411 * CPUs can only be in one of two states:
412 * - All CPUs support the write_only feature
413 * - No CPUs support the write_only feature
414 *
415 * If the first CPU attempts hw_enable_tag_checks_write_only() and
416 * finds the feature unsupported, kasan_flag_write_only is set to OFF
417 * to avoid further unnecessary calls on other CPUs.
418 */
419 if (kasan_flag_write_only && hw_enable_tag_checks_write_only()) {
420 kasan_flag_write_only = false;
421 pr_err_once("write-only mode is not supported and thus not enabled\n");
422 }
423 }
424
425 #if IS_ENABLED(CONFIG_KASAN_KUNIT_TEST)
426
427 EXPORT_SYMBOL_IF_KUNIT(kasan_enable_hw_tags);
428
kasan_force_async_fault(void)429 VISIBLE_IF_KUNIT void kasan_force_async_fault(void)
430 {
431 hw_force_async_tag_fault();
432 }
433 EXPORT_SYMBOL_IF_KUNIT(kasan_force_async_fault);
434
kasan_write_only_enabled(void)435 VISIBLE_IF_KUNIT bool kasan_write_only_enabled(void)
436 {
437 return kasan_flag_write_only;
438 }
439 EXPORT_SYMBOL_IF_KUNIT(kasan_write_only_enabled);
440
441 #endif
442