1457c8996SThomas Gleixner // SPDX-License-Identifier: GPL-2.0-only
2c33fa9f5SIngo Molnar /*
33f0acb1eSChristoph Hellwig * Access kernel or user memory without faulting.
4c33fa9f5SIngo Molnar */
5b95f1b31SPaul Gortmaker #include <linux/export.h>
6c33fa9f5SIngo Molnar #include <linux/mm.h>
77c7fcf76SDavid Howells #include <linux/uaccess.h>
8*d319f344SAlexei Starovoitov #include <asm/tlb.h>
9c33fa9f5SIngo Molnar
copy_from_kernel_nofault_allowed(const void * unsafe_src,size_t size)10fe557319SChristoph Hellwig bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src,
11fe557319SChristoph Hellwig size_t size)
12eab0c608SChristoph Hellwig {
13eab0c608SChristoph Hellwig return true;
14eab0c608SChristoph Hellwig }
15eab0c608SChristoph Hellwig
16fe557319SChristoph Hellwig #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \
17b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \
18b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, type, err_label); \
19b58294eaSChristoph Hellwig dst += sizeof(type); \
20b58294eaSChristoph Hellwig src += sizeof(type); \
21b58294eaSChristoph Hellwig len -= sizeof(type); \
22b58294eaSChristoph Hellwig }
23b58294eaSChristoph Hellwig
copy_from_kernel_nofault(void * dst,const void * src,size_t size)24fe557319SChristoph Hellwig long copy_from_kernel_nofault(void *dst, const void *src, size_t size)
25b58294eaSChristoph Hellwig {
262423de2eSArnd Bergmann unsigned long align = 0;
272423de2eSArnd Bergmann
282423de2eSArnd Bergmann if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
292423de2eSArnd Bergmann align = (unsigned long)dst | (unsigned long)src;
302423de2eSArnd Bergmann
31fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(src, size))
322a71e81dSChristoph Hellwig return -ERANGE;
33b58294eaSChristoph Hellwig
34b58294eaSChristoph Hellwig pagefault_disable();
352423de2eSArnd Bergmann if (!(align & 7))
36fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u64, Efault);
372423de2eSArnd Bergmann if (!(align & 3))
38fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u32, Efault);
392423de2eSArnd Bergmann if (!(align & 1))
40fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u16, Efault);
41fe557319SChristoph Hellwig copy_from_kernel_nofault_loop(dst, src, size, u8, Efault);
42b58294eaSChristoph Hellwig pagefault_enable();
43b58294eaSChristoph Hellwig return 0;
44b58294eaSChristoph Hellwig Efault:
45b58294eaSChristoph Hellwig pagefault_enable();
46b58294eaSChristoph Hellwig return -EFAULT;
47b58294eaSChristoph Hellwig }
48fe557319SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_kernel_nofault);
49b58294eaSChristoph Hellwig
50fe557319SChristoph Hellwig #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \
51b58294eaSChristoph Hellwig while (len >= sizeof(type)) { \
52b58294eaSChristoph Hellwig __put_kernel_nofault(dst, src, type, err_label); \
53b58294eaSChristoph Hellwig dst += sizeof(type); \
54b58294eaSChristoph Hellwig src += sizeof(type); \
55b58294eaSChristoph Hellwig len -= sizeof(type); \
56b58294eaSChristoph Hellwig }
57b58294eaSChristoph Hellwig
copy_to_kernel_nofault(void * dst,const void * src,size_t size)58fe557319SChristoph Hellwig long copy_to_kernel_nofault(void *dst, const void *src, size_t size)
59b58294eaSChristoph Hellwig {
602423de2eSArnd Bergmann unsigned long align = 0;
612423de2eSArnd Bergmann
622423de2eSArnd Bergmann if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS))
632423de2eSArnd Bergmann align = (unsigned long)dst | (unsigned long)src;
642423de2eSArnd Bergmann
65b58294eaSChristoph Hellwig pagefault_disable();
662423de2eSArnd Bergmann if (!(align & 7))
67fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u64, Efault);
682423de2eSArnd Bergmann if (!(align & 3))
69fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u32, Efault);
702423de2eSArnd Bergmann if (!(align & 1))
71fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u16, Efault);
72fe557319SChristoph Hellwig copy_to_kernel_nofault_loop(dst, src, size, u8, Efault);
73b58294eaSChristoph Hellwig pagefault_enable();
74b58294eaSChristoph Hellwig return 0;
75b58294eaSChristoph Hellwig Efault:
76b58294eaSChristoph Hellwig pagefault_enable();
77b58294eaSChristoph Hellwig return -EFAULT;
78b58294eaSChristoph Hellwig }
79b58294eaSChristoph Hellwig
strncpy_from_kernel_nofault(char * dst,const void * unsafe_addr,long count)80b58294eaSChristoph Hellwig long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count)
81b58294eaSChristoph Hellwig {
82b58294eaSChristoph Hellwig const void *src = unsafe_addr;
83b58294eaSChristoph Hellwig
84b58294eaSChristoph Hellwig if (unlikely(count <= 0))
85b58294eaSChristoph Hellwig return 0;
86fe557319SChristoph Hellwig if (!copy_from_kernel_nofault_allowed(unsafe_addr, count))
872a71e81dSChristoph Hellwig return -ERANGE;
88b58294eaSChristoph Hellwig
89b58294eaSChristoph Hellwig pagefault_disable();
90b58294eaSChristoph Hellwig do {
91b58294eaSChristoph Hellwig __get_kernel_nofault(dst, src, u8, Efault);
92b58294eaSChristoph Hellwig dst++;
93b58294eaSChristoph Hellwig src++;
94b58294eaSChristoph Hellwig } while (dst[-1] && src - unsafe_addr < count);
95b58294eaSChristoph Hellwig pagefault_enable();
96b58294eaSChristoph Hellwig
97b58294eaSChristoph Hellwig dst[-1] = '\0';
98b58294eaSChristoph Hellwig return src - unsafe_addr;
99b58294eaSChristoph Hellwig Efault:
100b58294eaSChristoph Hellwig pagefault_enable();
1018678ea06SAlban Crequy dst[0] = '\0';
102b58294eaSChristoph Hellwig return -EFAULT;
103b58294eaSChristoph Hellwig }
1043d708182SMasami Hiramatsu
1053d708182SMasami Hiramatsu /**
106c0ee37e8SChristoph Hellwig * copy_from_user_nofault(): safely attempt to read from a user-space location
107fc3562d7SChristoph Hellwig * @dst: pointer to the buffer that shall take the data
108fc3562d7SChristoph Hellwig * @src: address to read from. This must be a user address.
109fc3562d7SChristoph Hellwig * @size: size of the data chunk
110fc3562d7SChristoph Hellwig *
111fc3562d7SChristoph Hellwig * Safely read from user address @src to the buffer at @dst. If a kernel fault
112fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT.
113fc3562d7SChristoph Hellwig */
copy_from_user_nofault(void * dst,const void __user * src,size_t size)114c0ee37e8SChristoph Hellwig long copy_from_user_nofault(void *dst, const void __user *src, size_t size)
115fc3562d7SChristoph Hellwig {
116fc3562d7SChristoph Hellwig long ret = -EFAULT;
117*d319f344SAlexei Starovoitov
118*d319f344SAlexei Starovoitov if (!__access_ok(src, size))
119*d319f344SAlexei Starovoitov return ret;
120*d319f344SAlexei Starovoitov
121*d319f344SAlexei Starovoitov if (!nmi_uaccess_okay())
122*d319f344SAlexei Starovoitov return ret;
123*d319f344SAlexei Starovoitov
124fc3562d7SChristoph Hellwig pagefault_disable();
125fc3562d7SChristoph Hellwig ret = __copy_from_user_inatomic(dst, src, size);
126fc3562d7SChristoph Hellwig pagefault_enable();
127fc3562d7SChristoph Hellwig
128fc3562d7SChristoph Hellwig if (ret)
129fc3562d7SChristoph Hellwig return -EFAULT;
130fc3562d7SChristoph Hellwig return 0;
131fc3562d7SChristoph Hellwig }
132c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_from_user_nofault);
133fc3562d7SChristoph Hellwig
134fc3562d7SChristoph Hellwig /**
135c0ee37e8SChristoph Hellwig * copy_to_user_nofault(): safely attempt to write to a user-space location
136fc3562d7SChristoph Hellwig * @dst: address to write to
137fc3562d7SChristoph Hellwig * @src: pointer to the data that shall be written
138fc3562d7SChristoph Hellwig * @size: size of the data chunk
139fc3562d7SChristoph Hellwig *
140fc3562d7SChristoph Hellwig * Safely write to address @dst from the buffer at @src. If a kernel fault
141fc3562d7SChristoph Hellwig * happens, handle that and return -EFAULT.
142fc3562d7SChristoph Hellwig */
copy_to_user_nofault(void __user * dst,const void * src,size_t size)143c0ee37e8SChristoph Hellwig long copy_to_user_nofault(void __user *dst, const void *src, size_t size)
144fc3562d7SChristoph Hellwig {
145fc3562d7SChristoph Hellwig long ret = -EFAULT;
146fc3562d7SChristoph Hellwig
147fc3562d7SChristoph Hellwig if (access_ok(dst, size)) {
148fc3562d7SChristoph Hellwig pagefault_disable();
149fc3562d7SChristoph Hellwig ret = __copy_to_user_inatomic(dst, src, size);
150fc3562d7SChristoph Hellwig pagefault_enable();
151fc3562d7SChristoph Hellwig }
152fc3562d7SChristoph Hellwig
153fc3562d7SChristoph Hellwig if (ret)
154fc3562d7SChristoph Hellwig return -EFAULT;
155fc3562d7SChristoph Hellwig return 0;
156fc3562d7SChristoph Hellwig }
157c0ee37e8SChristoph Hellwig EXPORT_SYMBOL_GPL(copy_to_user_nofault);
158fc3562d7SChristoph Hellwig
159fc3562d7SChristoph Hellwig /**
160bd88bb5dSChristoph Hellwig * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user
1613d708182SMasami Hiramatsu * address.
1623d708182SMasami Hiramatsu * @dst: Destination address, in kernel space. This buffer must be at
1633d708182SMasami Hiramatsu * least @count bytes long.
1643d708182SMasami Hiramatsu * @unsafe_addr: Unsafe user address.
1653d708182SMasami Hiramatsu * @count: Maximum number of bytes to copy, including the trailing NUL.
1663d708182SMasami Hiramatsu *
1673d708182SMasami Hiramatsu * Copies a NUL-terminated string from unsafe user address to kernel buffer.
1683d708182SMasami Hiramatsu *
1693d708182SMasami Hiramatsu * On success, returns the length of the string INCLUDING the trailing NUL.
1703d708182SMasami Hiramatsu *
1713d708182SMasami Hiramatsu * If access fails, returns -EFAULT (some data may have been copied
1723d708182SMasami Hiramatsu * and the trailing NUL added).
1733d708182SMasami Hiramatsu *
1743d708182SMasami Hiramatsu * If @count is smaller than the length of the string, copies @count-1 bytes,
1753d708182SMasami Hiramatsu * sets the last byte of @dst buffer to NUL and returns @count.
1763d708182SMasami Hiramatsu */
strncpy_from_user_nofault(char * dst,const void __user * unsafe_addr,long count)177bd88bb5dSChristoph Hellwig long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr,
1783d708182SMasami Hiramatsu long count)
1793d708182SMasami Hiramatsu {
1803d708182SMasami Hiramatsu long ret;
1813d708182SMasami Hiramatsu
1823d708182SMasami Hiramatsu if (unlikely(count <= 0))
1833d708182SMasami Hiramatsu return 0;
1843d708182SMasami Hiramatsu
1853d708182SMasami Hiramatsu pagefault_disable();
1863d708182SMasami Hiramatsu ret = strncpy_from_user(dst, unsafe_addr, count);
1873d708182SMasami Hiramatsu pagefault_enable();
1883d708182SMasami Hiramatsu
1893d708182SMasami Hiramatsu if (ret >= count) {
1903d708182SMasami Hiramatsu ret = count;
1913d708182SMasami Hiramatsu dst[ret - 1] = '\0';
1923d708182SMasami Hiramatsu } else if (ret > 0) {
1933d708182SMasami Hiramatsu ret++;
1943d708182SMasami Hiramatsu }
1953d708182SMasami Hiramatsu
1963d708182SMasami Hiramatsu return ret;
1973d708182SMasami Hiramatsu }
1983d708182SMasami Hiramatsu
1993d708182SMasami Hiramatsu /**
20002dddb16SChristoph Hellwig * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL.
2013d708182SMasami Hiramatsu * @unsafe_addr: The string to measure.
2023d708182SMasami Hiramatsu * @count: Maximum count (including NUL)
2033d708182SMasami Hiramatsu *
2043d708182SMasami Hiramatsu * Get the size of a NUL-terminated string in user space without pagefault.
2053d708182SMasami Hiramatsu *
2063d708182SMasami Hiramatsu * Returns the size of the string INCLUDING the terminating NUL.
2073d708182SMasami Hiramatsu *
2083d708182SMasami Hiramatsu * If the string is too long, returns a number larger than @count. User
2093d708182SMasami Hiramatsu * has to check the return value against "> count".
2103d708182SMasami Hiramatsu * On exception (or invalid count), returns 0.
2113d708182SMasami Hiramatsu *
2123d708182SMasami Hiramatsu * Unlike strnlen_user, this can be used from IRQ handler etc. because
2133d708182SMasami Hiramatsu * it disables pagefaults.
2143d708182SMasami Hiramatsu */
strnlen_user_nofault(const void __user * unsafe_addr,long count)21502dddb16SChristoph Hellwig long strnlen_user_nofault(const void __user *unsafe_addr, long count)
2163d708182SMasami Hiramatsu {
2173d708182SMasami Hiramatsu int ret;
2183d708182SMasami Hiramatsu
2193d708182SMasami Hiramatsu pagefault_disable();
2203d708182SMasami Hiramatsu ret = strnlen_user(unsafe_addr, count);
2213d708182SMasami Hiramatsu pagefault_enable();
2223d708182SMasami Hiramatsu
2233d708182SMasami Hiramatsu return ret;
2243d708182SMasami Hiramatsu }
225ad7489d5SChristophe Leroy
__copy_overflow(int size,unsigned long count)226ad7489d5SChristophe Leroy void __copy_overflow(int size, unsigned long count)
227ad7489d5SChristophe Leroy {
228ad7489d5SChristophe Leroy WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count);
229ad7489d5SChristophe Leroy }
230ad7489d5SChristophe Leroy EXPORT_SYMBOL(__copy_overflow);
231