1d2912cb1SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2f5509cc1SKees Cook /*
3f5509cc1SKees Cook * This implements the various checks for CONFIG_HARDENED_USERCOPY*,
4f5509cc1SKees Cook * which are designed to protect kernel memory from needless exposure
5f5509cc1SKees Cook * and overwrite under many unintended conditions. This code is based
6f5509cc1SKees Cook * on PAX_USERCOPY, which is:
7f5509cc1SKees Cook *
8f5509cc1SKees Cook * Copyright (C) 2001-2016 PaX Team, Bradley Spengler, Open Source
9f5509cc1SKees Cook * Security Inc.
10f5509cc1SKees Cook */
11f5509cc1SKees Cook #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
12f5509cc1SKees Cook
13f5509cc1SKees Cook #include <linux/mm.h>
14314eed30SKees Cook #include <linux/highmem.h>
15f15be1b8SChristophe JAILLET #include <linux/kstrtox.h>
16f5509cc1SKees Cook #include <linux/slab.h>
175b825c3aSIngo Molnar #include <linux/sched.h>
1829930025SIngo Molnar #include <linux/sched/task.h>
1929930025SIngo Molnar #include <linux/sched/task_stack.h>
2096dc4f9fSSahara #include <linux/thread_info.h>
210aef499fSMatthew Wilcox (Oracle) #include <linux/vmalloc.h>
22b5cb15d9SChris von Recklinghausen #include <linux/atomic.h>
23b5cb15d9SChris von Recklinghausen #include <linux/jump_label.h>
24f5509cc1SKees Cook #include <asm/sections.h>
250b3eb091SMatthew Wilcox (Oracle) #include "slab.h"
26f5509cc1SKees Cook
27f5509cc1SKees Cook /*
28f5509cc1SKees Cook * Checks if a given pointer and length is contained by the current
29f5509cc1SKees Cook * stack frame (if possible).
30f5509cc1SKees Cook *
31f5509cc1SKees Cook * Returns:
32f5509cc1SKees Cook * NOT_STACK: not at all on the stack
33f5509cc1SKees Cook * GOOD_FRAME: fully within a valid stack frame
342792d84eSKees Cook * GOOD_STACK: within the current stack (when can't frame-check exactly)
35f5509cc1SKees Cook * BAD_STACK: error condition (invalid stack position or bad stack frame)
36f5509cc1SKees Cook */
check_stack_object(const void * obj,unsigned long len)37f5509cc1SKees Cook static noinline int check_stack_object(const void *obj, unsigned long len)
38f5509cc1SKees Cook {
39f5509cc1SKees Cook const void * const stack = task_stack_page(current);
40f5509cc1SKees Cook const void * const stackend = stack + THREAD_SIZE;
41f5509cc1SKees Cook int ret;
42f5509cc1SKees Cook
43f5509cc1SKees Cook /* Object is not on the stack at all. */
44f5509cc1SKees Cook if (obj + len <= stack || stackend <= obj)
45f5509cc1SKees Cook return NOT_STACK;
46f5509cc1SKees Cook
47f5509cc1SKees Cook /*
48f5509cc1SKees Cook * Reject: object partially overlaps the stack (passing the
495ce1be0eSRandy Dunlap * check above means at least one end is within the stack,
50f5509cc1SKees Cook * so if this check fails, the other end is outside the stack).
51f5509cc1SKees Cook */
52f5509cc1SKees Cook if (obj < stack || stackend < obj + len)
53f5509cc1SKees Cook return BAD_STACK;
54f5509cc1SKees Cook
55f5509cc1SKees Cook /* Check if object is safely within a valid frame. */
56f5509cc1SKees Cook ret = arch_within_stack_frames(stack, stackend, obj, len);
57f5509cc1SKees Cook if (ret)
58f5509cc1SKees Cook return ret;
59f5509cc1SKees Cook
602792d84eSKees Cook /* Finally, check stack depth if possible. */
612792d84eSKees Cook #ifdef CONFIG_ARCH_HAS_CURRENT_STACK_POINTER
622792d84eSKees Cook if (IS_ENABLED(CONFIG_STACK_GROWSUP)) {
632792d84eSKees Cook if ((void *)current_stack_pointer < obj + len)
642792d84eSKees Cook return BAD_STACK;
652792d84eSKees Cook } else {
662792d84eSKees Cook if (obj < (void *)current_stack_pointer)
672792d84eSKees Cook return BAD_STACK;
682792d84eSKees Cook }
692792d84eSKees Cook #endif
702792d84eSKees Cook
71f5509cc1SKees Cook return GOOD_STACK;
72f5509cc1SKees Cook }
73f5509cc1SKees Cook
74b394d468SKees Cook /*
75afcc90f8SKees Cook * If these functions are reached, then CONFIG_HARDENED_USERCOPY has found
76afcc90f8SKees Cook * an unexpected state during a copy_from_user() or copy_to_user() call.
77b394d468SKees Cook * There are several checks being performed on the buffer by the
78b394d468SKees Cook * __check_object_size() function. Normal stack buffer usage should never
79b394d468SKees Cook * trip the checks, and kernel text addressing will always trip the check.
80afcc90f8SKees Cook * For cache objects, it is checking that only the whitelisted range of
81afcc90f8SKees Cook * bytes for a given cache is being accessed (via the cache's usersize and
82afcc90f8SKees Cook * useroffset fields). To adjust a cache whitelist, use the usercopy-aware
83afcc90f8SKees Cook * kmem_cache_create_usercopy() function to create the cache (and
84afcc90f8SKees Cook * carefully audit the whitelist range).
85b394d468SKees Cook */
usercopy_abort(const char * name,const char * detail,bool to_user,unsigned long offset,unsigned long len)86b394d468SKees Cook void __noreturn usercopy_abort(const char *name, const char *detail,
87b394d468SKees Cook bool to_user, unsigned long offset,
88b394d468SKees Cook unsigned long len)
89f5509cc1SKees Cook {
90b394d468SKees Cook pr_emerg("Kernel memory %s attempt detected %s %s%s%s%s (offset %lu, size %lu)!\n",
91f5509cc1SKees Cook to_user ? "exposure" : "overwrite",
92b394d468SKees Cook to_user ? "from" : "to",
93b394d468SKees Cook name ? : "unknown?!",
94b394d468SKees Cook detail ? " '" : "", detail ? : "", detail ? "'" : "",
95b394d468SKees Cook offset, len);
96b394d468SKees Cook
97f5509cc1SKees Cook /*
98f5509cc1SKees Cook * For greater effect, it would be nice to do do_group_exit(),
99f5509cc1SKees Cook * but BUG() actually hooks all the lock-breaking and per-arch
100f5509cc1SKees Cook * Oops code, so that is used here instead.
101f5509cc1SKees Cook */
102f5509cc1SKees Cook BUG();
103f5509cc1SKees Cook }
104f5509cc1SKees Cook
105f5509cc1SKees Cook /* Returns true if any portion of [ptr,ptr+n) over laps with [low,high). */
overlaps(const unsigned long ptr,unsigned long n,unsigned long low,unsigned long high)106f4e6e289SKees Cook static bool overlaps(const unsigned long ptr, unsigned long n,
107f4e6e289SKees Cook unsigned long low, unsigned long high)
108f5509cc1SKees Cook {
109f4e6e289SKees Cook const unsigned long check_low = ptr;
110f5509cc1SKees Cook unsigned long check_high = check_low + n;
111f5509cc1SKees Cook
112f5509cc1SKees Cook /* Does not overlap if entirely above or entirely below. */
11394cd97afSJosh Poimboeuf if (check_low >= high || check_high <= low)
114f5509cc1SKees Cook return false;
115f5509cc1SKees Cook
116f5509cc1SKees Cook return true;
117f5509cc1SKees Cook }
118f5509cc1SKees Cook
119f5509cc1SKees Cook /* Is this address range in the kernel text area? */
check_kernel_text_object(const unsigned long ptr,unsigned long n,bool to_user)120f4e6e289SKees Cook static inline void check_kernel_text_object(const unsigned long ptr,
121f4e6e289SKees Cook unsigned long n, bool to_user)
122f5509cc1SKees Cook {
123f5509cc1SKees Cook unsigned long textlow = (unsigned long)_stext;
124f5509cc1SKees Cook unsigned long texthigh = (unsigned long)_etext;
125f5509cc1SKees Cook unsigned long textlow_linear, texthigh_linear;
126f5509cc1SKees Cook
127f5509cc1SKees Cook if (overlaps(ptr, n, textlow, texthigh))
128f4e6e289SKees Cook usercopy_abort("kernel text", NULL, to_user, ptr - textlow, n);
129f5509cc1SKees Cook
130f5509cc1SKees Cook /*
131f5509cc1SKees Cook * Some architectures have virtual memory mappings with a secondary
132f5509cc1SKees Cook * mapping of the kernel text, i.e. there is more than one virtual
133f5509cc1SKees Cook * kernel address that points to the kernel image. It is usually
134f5509cc1SKees Cook * when there is a separate linear physical memory mapping, in that
135f5509cc1SKees Cook * __pa() is not just the reverse of __va(). This can be detected
136f5509cc1SKees Cook * and checked:
137f5509cc1SKees Cook */
13846f6236aSLaura Abbott textlow_linear = (unsigned long)lm_alias(textlow);
139f5509cc1SKees Cook /* No different mapping: we're done. */
140f5509cc1SKees Cook if (textlow_linear == textlow)
141f4e6e289SKees Cook return;
142f5509cc1SKees Cook
143f5509cc1SKees Cook /* Check the secondary mapping... */
14446f6236aSLaura Abbott texthigh_linear = (unsigned long)lm_alias(texthigh);
145f5509cc1SKees Cook if (overlaps(ptr, n, textlow_linear, texthigh_linear))
146f4e6e289SKees Cook usercopy_abort("linear kernel text", NULL, to_user,
147f4e6e289SKees Cook ptr - textlow_linear, n);
148f5509cc1SKees Cook }
149f5509cc1SKees Cook
check_bogus_address(const unsigned long ptr,unsigned long n,bool to_user)150f4e6e289SKees Cook static inline void check_bogus_address(const unsigned long ptr, unsigned long n,
151f4e6e289SKees Cook bool to_user)
152f5509cc1SKees Cook {
153f5509cc1SKees Cook /* Reject if object wraps past end of memory. */
15495153169SIsaac J. Manjarres if (ptr + (n - 1) < ptr)
155f4e6e289SKees Cook usercopy_abort("wrapped address", NULL, to_user, 0, ptr + n);
156f5509cc1SKees Cook
157f5509cc1SKees Cook /* Reject if NULL or ZERO-allocation. */
158f5509cc1SKees Cook if (ZERO_OR_NULL_PTR(ptr))
159f4e6e289SKees Cook usercopy_abort("null address", NULL, to_user, ptr, n);
160f5509cc1SKees Cook }
161f5509cc1SKees Cook
check_heap_object(const void * ptr,unsigned long n,bool to_user)162f4e6e289SKees Cook static inline void check_heap_object(const void *ptr, unsigned long n,
1638e1f74eaSKees Cook bool to_user)
1648e1f74eaSKees Cook {
165170b2c35SJason A. Donenfeld unsigned long addr = (unsigned long)ptr;
1661dfbe9fcSMatthew Wilcox (Oracle) unsigned long offset;
1670b3eb091SMatthew Wilcox (Oracle) struct folio *folio;
1688e1f74eaSKees Cook
1694e140f59SMatthew Wilcox (Oracle) if (is_kmap_addr(ptr)) {
1701dfbe9fcSMatthew Wilcox (Oracle) offset = offset_in_page(ptr);
1711dfbe9fcSMatthew Wilcox (Oracle) if (n > PAGE_SIZE - offset)
1721dfbe9fcSMatthew Wilcox (Oracle) usercopy_abort("kmap", NULL, to_user, offset, n);
1734e140f59SMatthew Wilcox (Oracle) return;
1744e140f59SMatthew Wilcox (Oracle) }
1754e140f59SMatthew Wilcox (Oracle)
176*d319f344SAlexei Starovoitov if (is_vmalloc_addr(ptr) && !pagefault_disabled()) {
17735fb9ae4SMatthew Wilcox (Oracle) struct vmap_area *area = find_vmap_area(addr);
1780aef499fSMatthew Wilcox (Oracle)
179993d0b28SMatthew Wilcox (Oracle) if (!area)
1800aef499fSMatthew Wilcox (Oracle) usercopy_abort("vmalloc", "no area", to_user, 0, n);
1810aef499fSMatthew Wilcox (Oracle)
1821dfbe9fcSMatthew Wilcox (Oracle) if (n > area->va_end - addr) {
18335fb9ae4SMatthew Wilcox (Oracle) offset = addr - area->va_start;
1840aef499fSMatthew Wilcox (Oracle) usercopy_abort("vmalloc", NULL, to_user, offset, n);
1851dfbe9fcSMatthew Wilcox (Oracle) }
1860aef499fSMatthew Wilcox (Oracle) return;
1870aef499fSMatthew Wilcox (Oracle) }
1880aef499fSMatthew Wilcox (Oracle)
189a5f4d9dfSYuanzheng Song if (!virt_addr_valid(ptr))
190a5f4d9dfSYuanzheng Song return;
191a5f4d9dfSYuanzheng Song
1924e140f59SMatthew Wilcox (Oracle) folio = virt_to_folio(ptr);
1938e1f74eaSKees Cook
1940b3eb091SMatthew Wilcox (Oracle) if (folio_test_slab(folio)) {
1958e1f74eaSKees Cook /* Check slab allocator for flags and size. */
1960b3eb091SMatthew Wilcox (Oracle) __check_heap_object(ptr, n, folio_slab(folio), to_user);
197ab502103SMatthew Wilcox (Oracle) } else if (folio_test_large(folio)) {
1981dfbe9fcSMatthew Wilcox (Oracle) offset = ptr - folio_address(folio);
1991dfbe9fcSMatthew Wilcox (Oracle) if (n > folio_size(folio) - offset)
200ab502103SMatthew Wilcox (Oracle) usercopy_abort("page alloc", NULL, to_user, offset, n);
201f4e6e289SKees Cook }
202f5509cc1SKees Cook }
203f5509cc1SKees Cook
204b5cb15d9SChris von Recklinghausen static DEFINE_STATIC_KEY_FALSE_RO(bypass_usercopy_checks);
205b5cb15d9SChris von Recklinghausen
206f5509cc1SKees Cook /*
207f5509cc1SKees Cook * Validates that the given object is:
208f5509cc1SKees Cook * - not bogus address
2097bff3c06SQian Cai * - fully contained by stack (or stack frame, when available)
2107bff3c06SQian Cai * - fully within SLAB object (or object whitelist area, when available)
211f5509cc1SKees Cook * - not in kernel text
212f5509cc1SKees Cook */
__check_object_size(const void * ptr,unsigned long n,bool to_user)213f5509cc1SKees Cook void __check_object_size(const void *ptr, unsigned long n, bool to_user)
214f5509cc1SKees Cook {
215b5cb15d9SChris von Recklinghausen if (static_branch_unlikely(&bypass_usercopy_checks))
216b5cb15d9SChris von Recklinghausen return;
217b5cb15d9SChris von Recklinghausen
218f5509cc1SKees Cook /* Skip all tests if size is zero. */
219f5509cc1SKees Cook if (!n)
220f5509cc1SKees Cook return;
221f5509cc1SKees Cook
222f5509cc1SKees Cook /* Check for invalid addresses. */
223f4e6e289SKees Cook check_bogus_address((const unsigned long)ptr, n, to_user);
224f5509cc1SKees Cook
225f5509cc1SKees Cook /* Check for bad stack object. */
226f5509cc1SKees Cook switch (check_stack_object(ptr, n)) {
227f5509cc1SKees Cook case NOT_STACK:
228f5509cc1SKees Cook /* Object is not touching the current process stack. */
229f5509cc1SKees Cook break;
230f5509cc1SKees Cook case GOOD_FRAME:
231f5509cc1SKees Cook case GOOD_STACK:
232f5509cc1SKees Cook /*
233f5509cc1SKees Cook * Object is either in the correct frame (when it
234f5509cc1SKees Cook * is possible to check) or just generally on the
235f5509cc1SKees Cook * process stack (when frame checking not available).
236f5509cc1SKees Cook */
237f5509cc1SKees Cook return;
238f5509cc1SKees Cook default:
2392792d84eSKees Cook usercopy_abort("process stack", NULL, to_user,
2402792d84eSKees Cook #ifdef CONFIG_ARCH_HAS_CURRENT_STACK_POINTER
2412792d84eSKees Cook IS_ENABLED(CONFIG_STACK_GROWSUP) ?
2422792d84eSKees Cook ptr - (void *)current_stack_pointer :
2432792d84eSKees Cook (void *)current_stack_pointer - ptr,
2442792d84eSKees Cook #else
2452792d84eSKees Cook 0,
2462792d84eSKees Cook #endif
2472792d84eSKees Cook n);
248f5509cc1SKees Cook }
249f5509cc1SKees Cook
2507bff3c06SQian Cai /* Check for bad heap object. */
2517bff3c06SQian Cai check_heap_object(ptr, n, to_user);
2527bff3c06SQian Cai
253f5509cc1SKees Cook /* Check for object in kernel to avoid text exposure. */
254f4e6e289SKees Cook check_kernel_text_object((const unsigned long)ptr, n, to_user);
255f5509cc1SKees Cook }
256f5509cc1SKees Cook EXPORT_SYMBOL(__check_object_size);
257b5cb15d9SChris von Recklinghausen
258b5cb15d9SChris von Recklinghausen static bool enable_checks __initdata = true;
259b5cb15d9SChris von Recklinghausen
parse_hardened_usercopy(char * str)260b5cb15d9SChris von Recklinghausen static int __init parse_hardened_usercopy(char *str)
261b5cb15d9SChris von Recklinghausen {
262f15be1b8SChristophe JAILLET if (kstrtobool(str, &enable_checks))
26305fe3c10SRandy Dunlap pr_warn("Invalid option string for hardened_usercopy: '%s'\n",
26405fe3c10SRandy Dunlap str);
26505fe3c10SRandy Dunlap return 1;
266b5cb15d9SChris von Recklinghausen }
267b5cb15d9SChris von Recklinghausen
268b5cb15d9SChris von Recklinghausen __setup("hardened_usercopy=", parse_hardened_usercopy);
269b5cb15d9SChris von Recklinghausen
set_hardened_usercopy(void)270b5cb15d9SChris von Recklinghausen static int __init set_hardened_usercopy(void)
271b5cb15d9SChris von Recklinghausen {
272b5cb15d9SChris von Recklinghausen if (enable_checks == false)
273b5cb15d9SChris von Recklinghausen static_branch_enable(&bypass_usercopy_checks);
274b5cb15d9SChris von Recklinghausen return 1;
275b5cb15d9SChris von Recklinghausen }
276b5cb15d9SChris von Recklinghausen
277b5cb15d9SChris von Recklinghausen late_initcall(set_hardened_usercopy);
278