1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Access kernel or user memory without faulting. 4 */ 5 #include <linux/export.h> 6 #include <linux/mm.h> 7 #include <linux/uaccess.h> 8 #include <asm/tlb.h> 9 10 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 11 size_t size) 12 { 13 return true; 14 } 15 16 /* 17 * The below only uses kmsan_check_memory() to ensure uninitialized kernel 18 * memory isn't leaked. 19 */ 20 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 21 while (len >= sizeof(type)) { \ 22 __get_kernel_nofault(dst, src, type, err_label); \ 23 kmsan_check_memory(src, sizeof(type)); \ 24 dst += sizeof(type); \ 25 src += sizeof(type); \ 26 len -= sizeof(type); \ 27 } 28 29 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 30 { 31 unsigned long align = 0; 32 33 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 34 align = (unsigned long)dst | (unsigned long)src; 35 36 if (!copy_from_kernel_nofault_allowed(src, size)) 37 return -ERANGE; 38 39 pagefault_disable(); 40 if (!(align & 7)) 41 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 42 if (!(align & 3)) 43 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 44 if (!(align & 1)) 45 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 46 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 47 pagefault_enable(); 48 return 0; 49 Efault: 50 pagefault_enable(); 51 return -EFAULT; 52 } 53 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 54 55 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 56 while (len >= sizeof(type)) { \ 57 __put_kernel_nofault(dst, src, type, err_label); \ 58 instrument_write(dst, sizeof(type)); \ 59 dst += sizeof(type); \ 60 src += sizeof(type); \ 61 len -= sizeof(type); \ 62 } 63 64 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 65 { 66 unsigned long align = 0; 67 68 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS)) 69 align = (unsigned long)dst | (unsigned long)src; 70 71 pagefault_disable(); 72 if (!(align & 7)) 73 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 74 if (!(align & 3)) 75 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 76 if (!(align & 1)) 77 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 78 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 79 pagefault_enable(); 80 return 0; 81 Efault: 82 pagefault_enable(); 83 return -EFAULT; 84 } 85 86 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 87 { 88 const void *src = unsafe_addr; 89 90 if (unlikely(count <= 0)) 91 return 0; 92 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 93 return -ERANGE; 94 95 pagefault_disable(); 96 do { 97 __get_kernel_nofault(dst, src, u8, Efault); 98 dst++; 99 src++; 100 } while (dst[-1] && src - unsafe_addr < count); 101 pagefault_enable(); 102 103 dst[-1] = '\0'; 104 return src - unsafe_addr; 105 Efault: 106 pagefault_enable(); 107 dst[0] = '\0'; 108 return -EFAULT; 109 } 110 111 /** 112 * copy_from_user_nofault(): safely attempt to read from a user-space location 113 * @dst: pointer to the buffer that shall take the data 114 * @src: address to read from. This must be a user address. 115 * @size: size of the data chunk 116 * 117 * Safely read from user address @src to the buffer at @dst. If a kernel fault 118 * happens, handle that and return -EFAULT. 119 */ 120 long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 121 { 122 long ret = -EFAULT; 123 124 if (!__access_ok(src, size)) 125 return ret; 126 127 if (!nmi_uaccess_okay()) 128 return ret; 129 130 pagefault_disable(); 131 ret = __copy_from_user_inatomic(dst, src, size); 132 pagefault_enable(); 133 134 if (ret) 135 return -EFAULT; 136 return 0; 137 } 138 EXPORT_SYMBOL_GPL(copy_from_user_nofault); 139 140 /** 141 * copy_to_user_nofault(): safely attempt to write to a user-space location 142 * @dst: address to write to 143 * @src: pointer to the data that shall be written 144 * @size: size of the data chunk 145 * 146 * Safely write to address @dst from the buffer at @src. If a kernel fault 147 * happens, handle that and return -EFAULT. 148 */ 149 long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 150 { 151 long ret = -EFAULT; 152 153 if (access_ok(dst, size)) { 154 pagefault_disable(); 155 ret = __copy_to_user_inatomic(dst, src, size); 156 pagefault_enable(); 157 } 158 159 if (ret) 160 return -EFAULT; 161 return 0; 162 } 163 EXPORT_SYMBOL_GPL(copy_to_user_nofault); 164 165 /** 166 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 167 * address. 168 * @dst: Destination address, in kernel space. This buffer must be at 169 * least @count bytes long. 170 * @unsafe_addr: Unsafe user address. 171 * @count: Maximum number of bytes to copy, including the trailing NUL. 172 * 173 * Copies a NUL-terminated string from unsafe user address to kernel buffer. 174 * 175 * On success, returns the length of the string INCLUDING the trailing NUL. 176 * 177 * If access fails, returns -EFAULT (some data may have been copied 178 * and the trailing NUL added). 179 * 180 * If @count is smaller than the length of the string, copies @count-1 bytes, 181 * sets the last byte of @dst buffer to NUL and returns @count. 182 */ 183 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 184 long count) 185 { 186 long ret; 187 188 if (unlikely(count <= 0)) 189 return 0; 190 191 pagefault_disable(); 192 ret = strncpy_from_user(dst, unsafe_addr, count); 193 pagefault_enable(); 194 195 if (ret >= count) { 196 ret = count; 197 dst[ret - 1] = '\0'; 198 } else if (ret >= 0) { 199 ret++; 200 } 201 202 return ret; 203 } 204 205 /** 206 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 207 * @unsafe_addr: The string to measure. 208 * @count: Maximum count (including NUL) 209 * 210 * Get the size of a NUL-terminated string in user space without pagefault. 211 * 212 * Returns the size of the string INCLUDING the terminating NUL. 213 * 214 * If the string is too long, returns a number larger than @count. User 215 * has to check the return value against "> count". 216 * On exception (or invalid count), returns 0. 217 * 218 * Unlike strnlen_user, this can be used from IRQ handler etc. because 219 * it disables pagefaults. 220 */ 221 long strnlen_user_nofault(const void __user *unsafe_addr, long count) 222 { 223 int ret; 224 225 pagefault_disable(); 226 ret = strnlen_user(unsafe_addr, count); 227 pagefault_enable(); 228 229 return ret; 230 } 231 232 void __copy_overflow(int size, unsigned long count) 233 { 234 WARN(1, "Buffer overflow detected (%d < %lu)!\n", size, count); 235 } 236 EXPORT_SYMBOL(__copy_overflow); 237