1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Access kernel or user memory without faulting. 4 */ 5 #include <linux/export.h> 6 #include <linux/mm.h> 7 #include <linux/uaccess.h> 8 9 bool __weak copy_from_kernel_nofault_allowed(const void *unsafe_src, 10 size_t size) 11 { 12 return true; 13 } 14 15 #ifdef HAVE_GET_KERNEL_NOFAULT 16 17 #define copy_from_kernel_nofault_loop(dst, src, len, type, err_label) \ 18 while (len >= sizeof(type)) { \ 19 __get_kernel_nofault(dst, src, type, err_label); \ 20 dst += sizeof(type); \ 21 src += sizeof(type); \ 22 len -= sizeof(type); \ 23 } 24 25 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 26 { 27 if (!copy_from_kernel_nofault_allowed(src, size)) 28 return -ERANGE; 29 30 pagefault_disable(); 31 copy_from_kernel_nofault_loop(dst, src, size, u64, Efault); 32 copy_from_kernel_nofault_loop(dst, src, size, u32, Efault); 33 copy_from_kernel_nofault_loop(dst, src, size, u16, Efault); 34 copy_from_kernel_nofault_loop(dst, src, size, u8, Efault); 35 pagefault_enable(); 36 return 0; 37 Efault: 38 pagefault_enable(); 39 return -EFAULT; 40 } 41 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 42 43 #define copy_to_kernel_nofault_loop(dst, src, len, type, err_label) \ 44 while (len >= sizeof(type)) { \ 45 __put_kernel_nofault(dst, src, type, err_label); \ 46 dst += sizeof(type); \ 47 src += sizeof(type); \ 48 len -= sizeof(type); \ 49 } 50 51 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 52 { 53 pagefault_disable(); 54 copy_to_kernel_nofault_loop(dst, src, size, u64, Efault); 55 copy_to_kernel_nofault_loop(dst, src, size, u32, Efault); 56 copy_to_kernel_nofault_loop(dst, src, size, u16, Efault); 57 copy_to_kernel_nofault_loop(dst, src, size, u8, Efault); 58 pagefault_enable(); 59 return 0; 60 Efault: 61 pagefault_enable(); 62 return -EFAULT; 63 } 64 65 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 66 { 67 const void *src = unsafe_addr; 68 69 if (unlikely(count <= 0)) 70 return 0; 71 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 72 return -ERANGE; 73 74 pagefault_disable(); 75 do { 76 __get_kernel_nofault(dst, src, u8, Efault); 77 dst++; 78 src++; 79 } while (dst[-1] && src - unsafe_addr < count); 80 pagefault_enable(); 81 82 dst[-1] = '\0'; 83 return src - unsafe_addr; 84 Efault: 85 pagefault_enable(); 86 dst[-1] = '\0'; 87 return -EFAULT; 88 } 89 #else /* HAVE_GET_KERNEL_NOFAULT */ 90 /** 91 * copy_from_kernel_nofault(): safely attempt to read from kernel-space 92 * @dst: pointer to the buffer that shall take the data 93 * @src: address to read from 94 * @size: size of the data chunk 95 * 96 * Safely read from kernel address @src to the buffer at @dst. If a kernel 97 * fault happens, handle that and return -EFAULT. If @src is not a valid kernel 98 * address, return -ERANGE. 99 * 100 * We ensure that the copy_from_user is executed in atomic context so that 101 * do_page_fault() doesn't attempt to take mmap_lock. This makes 102 * copy_from_kernel_nofault() suitable for use within regions where the caller 103 * already holds mmap_lock, or other locks which nest inside mmap_lock. 104 */ 105 long copy_from_kernel_nofault(void *dst, const void *src, size_t size) 106 { 107 long ret; 108 mm_segment_t old_fs = get_fs(); 109 110 if (!copy_from_kernel_nofault_allowed(src, size)) 111 return -ERANGE; 112 113 set_fs(KERNEL_DS); 114 pagefault_disable(); 115 ret = __copy_from_user_inatomic(dst, (__force const void __user *)src, 116 size); 117 pagefault_enable(); 118 set_fs(old_fs); 119 120 if (ret) 121 return -EFAULT; 122 return 0; 123 } 124 EXPORT_SYMBOL_GPL(copy_from_kernel_nofault); 125 126 /** 127 * copy_to_kernel_nofault(): safely attempt to write to a location 128 * @dst: address to write to 129 * @src: pointer to the data that shall be written 130 * @size: size of the data chunk 131 * 132 * Safely write to address @dst from the buffer at @src. If a kernel fault 133 * happens, handle that and return -EFAULT. 134 */ 135 long copy_to_kernel_nofault(void *dst, const void *src, size_t size) 136 { 137 long ret; 138 mm_segment_t old_fs = get_fs(); 139 140 set_fs(KERNEL_DS); 141 pagefault_disable(); 142 ret = __copy_to_user_inatomic((__force void __user *)dst, src, size); 143 pagefault_enable(); 144 set_fs(old_fs); 145 146 if (ret) 147 return -EFAULT; 148 return 0; 149 } 150 151 /** 152 * strncpy_from_kernel_nofault: - Copy a NUL terminated string from unsafe 153 * address. 154 * @dst: Destination address, in kernel space. This buffer must be at 155 * least @count bytes long. 156 * @unsafe_addr: Unsafe address. 157 * @count: Maximum number of bytes to copy, including the trailing NUL. 158 * 159 * Copies a NUL-terminated string from unsafe address to kernel buffer. 160 * 161 * On success, returns the length of the string INCLUDING the trailing NUL. 162 * 163 * If access fails, returns -EFAULT (some data may have been copied and the 164 * trailing NUL added). If @unsafe_addr is not a valid kernel address, return 165 * -ERANGE. 166 * 167 * If @count is smaller than the length of the string, copies @count-1 bytes, 168 * sets the last byte of @dst buffer to NUL and returns @count. 169 */ 170 long strncpy_from_kernel_nofault(char *dst, const void *unsafe_addr, long count) 171 { 172 mm_segment_t old_fs = get_fs(); 173 const void *src = unsafe_addr; 174 long ret; 175 176 if (unlikely(count <= 0)) 177 return 0; 178 if (!copy_from_kernel_nofault_allowed(unsafe_addr, count)) 179 return -ERANGE; 180 181 set_fs(KERNEL_DS); 182 pagefault_disable(); 183 184 do { 185 ret = __get_user(*dst++, (const char __user __force *)src++); 186 } while (dst[-1] && ret == 0 && src - unsafe_addr < count); 187 188 dst[-1] = '\0'; 189 pagefault_enable(); 190 set_fs(old_fs); 191 192 return ret ? -EFAULT : src - unsafe_addr; 193 } 194 #endif /* HAVE_GET_KERNEL_NOFAULT */ 195 196 /** 197 * copy_from_user_nofault(): safely attempt to read from a user-space location 198 * @dst: pointer to the buffer that shall take the data 199 * @src: address to read from. This must be a user address. 200 * @size: size of the data chunk 201 * 202 * Safely read from user address @src to the buffer at @dst. If a kernel fault 203 * happens, handle that and return -EFAULT. 204 */ 205 long copy_from_user_nofault(void *dst, const void __user *src, size_t size) 206 { 207 long ret = -EFAULT; 208 mm_segment_t old_fs = force_uaccess_begin(); 209 210 if (access_ok(src, size)) { 211 pagefault_disable(); 212 ret = __copy_from_user_inatomic(dst, src, size); 213 pagefault_enable(); 214 } 215 force_uaccess_end(old_fs); 216 217 if (ret) 218 return -EFAULT; 219 return 0; 220 } 221 EXPORT_SYMBOL_GPL(copy_from_user_nofault); 222 223 /** 224 * copy_to_user_nofault(): safely attempt to write to a user-space location 225 * @dst: address to write to 226 * @src: pointer to the data that shall be written 227 * @size: size of the data chunk 228 * 229 * Safely write to address @dst from the buffer at @src. If a kernel fault 230 * happens, handle that and return -EFAULT. 231 */ 232 long copy_to_user_nofault(void __user *dst, const void *src, size_t size) 233 { 234 long ret = -EFAULT; 235 mm_segment_t old_fs = force_uaccess_begin(); 236 237 if (access_ok(dst, size)) { 238 pagefault_disable(); 239 ret = __copy_to_user_inatomic(dst, src, size); 240 pagefault_enable(); 241 } 242 force_uaccess_end(old_fs); 243 244 if (ret) 245 return -EFAULT; 246 return 0; 247 } 248 EXPORT_SYMBOL_GPL(copy_to_user_nofault); 249 250 /** 251 * strncpy_from_user_nofault: - Copy a NUL terminated string from unsafe user 252 * address. 253 * @dst: Destination address, in kernel space. This buffer must be at 254 * least @count bytes long. 255 * @unsafe_addr: Unsafe user address. 256 * @count: Maximum number of bytes to copy, including the trailing NUL. 257 * 258 * Copies a NUL-terminated string from unsafe user address to kernel buffer. 259 * 260 * On success, returns the length of the string INCLUDING the trailing NUL. 261 * 262 * If access fails, returns -EFAULT (some data may have been copied 263 * and the trailing NUL added). 264 * 265 * If @count is smaller than the length of the string, copies @count-1 bytes, 266 * sets the last byte of @dst buffer to NUL and returns @count. 267 */ 268 long strncpy_from_user_nofault(char *dst, const void __user *unsafe_addr, 269 long count) 270 { 271 mm_segment_t old_fs; 272 long ret; 273 274 if (unlikely(count <= 0)) 275 return 0; 276 277 old_fs = force_uaccess_begin(); 278 pagefault_disable(); 279 ret = strncpy_from_user(dst, unsafe_addr, count); 280 pagefault_enable(); 281 force_uaccess_end(old_fs); 282 283 if (ret >= count) { 284 ret = count; 285 dst[ret - 1] = '\0'; 286 } else if (ret > 0) { 287 ret++; 288 } 289 290 return ret; 291 } 292 293 /** 294 * strnlen_user_nofault: - Get the size of a user string INCLUDING final NUL. 295 * @unsafe_addr: The string to measure. 296 * @count: Maximum count (including NUL) 297 * 298 * Get the size of a NUL-terminated string in user space without pagefault. 299 * 300 * Returns the size of the string INCLUDING the terminating NUL. 301 * 302 * If the string is too long, returns a number larger than @count. User 303 * has to check the return value against "> count". 304 * On exception (or invalid count), returns 0. 305 * 306 * Unlike strnlen_user, this can be used from IRQ handler etc. because 307 * it disables pagefaults. 308 */ 309 long strnlen_user_nofault(const void __user *unsafe_addr, long count) 310 { 311 mm_segment_t old_fs; 312 int ret; 313 314 old_fs = force_uaccess_begin(); 315 pagefault_disable(); 316 ret = strnlen_user(unsafe_addr, count); 317 pagefault_enable(); 318 force_uaccess_end(old_fs); 319 320 return ret; 321 } 322