1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 * 5 * This file was copied from include/asm-generic/uaccess.h 6 */ 7 8 #ifndef _ASM_RISCV_UACCESS_H 9 #define _ASM_RISCV_UACCESS_H 10 11 #include <asm/asm-extable.h> 12 #include <asm/cpufeature.h> 13 #include <asm/pgtable.h> /* for TASK_SIZE */ 14 15 #ifdef CONFIG_RISCV_ISA_SUPM 16 static inline unsigned long __untagged_addr_remote(struct mm_struct *mm, unsigned long addr) 17 { 18 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SUPM)) { 19 u8 pmlen = mm->context.pmlen; 20 21 /* Virtual addresses are sign-extended; physical addresses are zero-extended. */ 22 if (IS_ENABLED(CONFIG_MMU)) 23 return (long)(addr << pmlen) >> pmlen; 24 else 25 return (addr << pmlen) >> pmlen; 26 } 27 28 return addr; 29 } 30 31 #define untagged_addr(addr) ({ \ 32 unsigned long __addr = (__force unsigned long)(addr); \ 33 (__force __typeof__(addr))__untagged_addr_remote(current->mm, __addr); \ 34 }) 35 36 #define untagged_addr_remote(mm, addr) ({ \ 37 unsigned long __addr = (__force unsigned long)(addr); \ 38 mmap_assert_locked(mm); \ 39 (__force __typeof__(addr))__untagged_addr_remote(mm, __addr); \ 40 }) 41 42 #define access_ok(addr, size) likely(__access_ok(untagged_addr(addr), size)) 43 #else 44 #define untagged_addr(addr) (addr) 45 #endif 46 47 /* 48 * User space memory access functions 49 */ 50 #ifdef CONFIG_MMU 51 #include <linux/errno.h> 52 #include <linux/compiler.h> 53 #include <linux/thread_info.h> 54 #include <asm/byteorder.h> 55 #include <asm/extable.h> 56 #include <asm/asm.h> 57 #include <asm-generic/access_ok.h> 58 59 #define __enable_user_access() \ 60 __asm__ __volatile__ ("csrs sstatus, %0" : : "r" (SR_SUM) : "memory") 61 #define __disable_user_access() \ 62 __asm__ __volatile__ ("csrc sstatus, %0" : : "r" (SR_SUM) : "memory") 63 64 /* 65 * This is the smallest unsigned integer type that can fit a value 66 * (up to 'long long') 67 */ 68 #define __inttype(x) __typeof__( \ 69 __typefits(x, char, \ 70 __typefits(x, short, \ 71 __typefits(x, int, \ 72 __typefits(x, long, 0ULL))))) 73 74 #define __typefits(x, type, not) \ 75 __builtin_choose_expr(sizeof(x) <= sizeof(type), (unsigned type)0, not) 76 77 /* 78 * The exception table consists of pairs of addresses: the first is the 79 * address of an instruction that is allowed to fault, and the second is 80 * the address at which the program should continue. No registers are 81 * modified, so it is entirely up to the continuation code to figure out 82 * what to do. 83 * 84 * All the routines below use bits of fixup code that are out of line 85 * with the main instruction path. This means when everything is well, 86 * we don't even have to jump over them. Further, they do not intrude 87 * on our cache or tlb entries. 88 */ 89 90 #define __LSW 0 91 #define __MSW 1 92 93 /* 94 * The "__xxx" versions of the user access functions do not verify the address 95 * space - it must have been done previously with a separate "access_ok()" 96 * call. 97 */ 98 99 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 100 #define __get_user_asm(insn, x, ptr, label) \ 101 asm_goto_output( \ 102 "1:\n" \ 103 " " insn " %0, %1\n" \ 104 _ASM_EXTABLE_UACCESS_ERR(1b, %l2, %0) \ 105 : "=&r" (x) \ 106 : "m" (*(ptr)) : : label) 107 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 108 #define __get_user_asm(insn, x, ptr, label) \ 109 do { \ 110 long __gua_err = 0; \ 111 __asm__ __volatile__ ( \ 112 "1:\n" \ 113 " " insn " %1, %2\n" \ 114 "2:\n" \ 115 _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 2b, %0, %1) \ 116 : "+r" (__gua_err), "=&r" (x) \ 117 : "m" (*(ptr))); \ 118 if (__gua_err) \ 119 goto label; \ 120 } while (0) 121 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 122 123 #ifdef CONFIG_64BIT 124 #define __get_user_8(x, ptr, label) \ 125 __get_user_asm("ld", x, ptr, label) 126 #else /* !CONFIG_64BIT */ 127 128 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 129 #define __get_user_8(x, ptr, label) \ 130 do { \ 131 u32 __user *__ptr = (u32 __user *)(ptr); \ 132 u32 __lo, __hi; \ 133 asm_goto_output( \ 134 "1:\n" \ 135 " lw %0, %2\n" \ 136 "2:\n" \ 137 " lw %1, %3\n" \ 138 _ASM_EXTABLE_UACCESS_ERR(1b, %l4, %0) \ 139 _ASM_EXTABLE_UACCESS_ERR(2b, %l4, %0) \ 140 : "=&r" (__lo), "=r" (__hi) \ 141 : "m" (__ptr[__LSW]), "m" (__ptr[__MSW]) \ 142 : : label); \ 143 (x) = (__typeof__(x))((__typeof__((x) - (x)))( \ 144 (((u64)__hi << 32) | __lo))); \ 145 } while (0) 146 #else /* !CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 147 #define __get_user_8(x, ptr, label) \ 148 do { \ 149 u32 __user *__ptr = (u32 __user *)(ptr); \ 150 u32 __lo, __hi; \ 151 long __gu8_err = 0; \ 152 __asm__ __volatile__ ( \ 153 "1:\n" \ 154 " lw %1, %3\n" \ 155 "2:\n" \ 156 " lw %2, %4\n" \ 157 "3:\n" \ 158 _ASM_EXTABLE_UACCESS_ERR_ZERO(1b, 3b, %0, %1) \ 159 _ASM_EXTABLE_UACCESS_ERR_ZERO(2b, 3b, %0, %1) \ 160 : "+r" (__gu8_err), "=&r" (__lo), "=r" (__hi) \ 161 : "m" (__ptr[__LSW]), "m" (__ptr[__MSW])); \ 162 if (__gu8_err) { \ 163 __hi = 0; \ 164 goto label; \ 165 } \ 166 (x) = (__typeof__(x))((__typeof__((x) - (x)))( \ 167 (((u64)__hi << 32) | __lo))); \ 168 } while (0) 169 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 170 171 #endif /* CONFIG_64BIT */ 172 173 unsigned long __must_check __asm_copy_to_user_sum_enabled(void __user *to, 174 const void *from, unsigned long n); 175 unsigned long __must_check __asm_copy_from_user_sum_enabled(void *to, 176 const void __user *from, unsigned long n); 177 178 #define __get_user_nocheck(x, __gu_ptr, label) \ 179 do { \ 180 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && \ 181 !IS_ALIGNED((uintptr_t)__gu_ptr, sizeof(*__gu_ptr))) { \ 182 if (__asm_copy_from_user_sum_enabled(&(x), __gu_ptr, sizeof(*__gu_ptr))) \ 183 goto label; \ 184 break; \ 185 } \ 186 switch (sizeof(*__gu_ptr)) { \ 187 case 1: \ 188 __get_user_asm("lb", (x), __gu_ptr, label); \ 189 break; \ 190 case 2: \ 191 __get_user_asm("lh", (x), __gu_ptr, label); \ 192 break; \ 193 case 4: \ 194 __get_user_asm("lw", (x), __gu_ptr, label); \ 195 break; \ 196 case 8: \ 197 __get_user_8((x), __gu_ptr, label); \ 198 break; \ 199 default: \ 200 BUILD_BUG(); \ 201 } \ 202 } while (0) 203 204 #define __get_user_error(x, ptr, err) \ 205 do { \ 206 __label__ __gu_failed; \ 207 \ 208 __get_user_nocheck(x, ptr, __gu_failed); \ 209 err = 0; \ 210 break; \ 211 __gu_failed: \ 212 x = 0; \ 213 err = -EFAULT; \ 214 } while (0) 215 216 /** 217 * __get_user: - Get a simple variable from user space, with less checking. 218 * @x: Variable to store result. 219 * @ptr: Source address, in user space. 220 * 221 * Context: User context only. This function may sleep. 222 * 223 * This macro copies a single simple variable from user space to kernel 224 * space. It supports simple types like char and int, but not larger 225 * data types like structures or arrays. 226 * 227 * @ptr must have pointer-to-simple-variable type, and the result of 228 * dereferencing @ptr must be assignable to @x without a cast. 229 * 230 * Caller must check the pointer with access_ok() before calling this 231 * function. 232 * 233 * Returns zero on success, or -EFAULT on error. 234 * On error, the variable @x is set to zero. 235 */ 236 #define __get_user(x, ptr) \ 237 ({ \ 238 const __typeof__(*(ptr)) __user *__gu_ptr = untagged_addr(ptr); \ 239 long __gu_err = 0; \ 240 __typeof__(x) __gu_val; \ 241 \ 242 __chk_user_ptr(__gu_ptr); \ 243 \ 244 __enable_user_access(); \ 245 __get_user_error(__gu_val, __gu_ptr, __gu_err); \ 246 __disable_user_access(); \ 247 \ 248 (x) = __gu_val; \ 249 \ 250 __gu_err; \ 251 }) 252 253 /** 254 * get_user: - Get a simple variable from user space. 255 * @x: Variable to store result. 256 * @ptr: Source address, in user space. 257 * 258 * Context: User context only. This function may sleep. 259 * 260 * This macro copies a single simple variable from user space to kernel 261 * space. It supports simple types like char and int, but not larger 262 * data types like structures or arrays. 263 * 264 * @ptr must have pointer-to-simple-variable type, and the result of 265 * dereferencing @ptr must be assignable to @x without a cast. 266 * 267 * Returns zero on success, or -EFAULT on error. 268 * On error, the variable @x is set to zero. 269 */ 270 #define get_user(x, ptr) \ 271 ({ \ 272 const __typeof__(*(ptr)) __user *__p = (ptr); \ 273 might_fault(); \ 274 access_ok(__p, sizeof(*__p)) ? \ 275 __get_user((x), __p) : \ 276 ((x) = (__force __typeof__(x))0, -EFAULT); \ 277 }) 278 279 #define __put_user_asm(insn, x, ptr, label) \ 280 do { \ 281 __typeof__(*(ptr)) __x = x; \ 282 asm goto( \ 283 "1:\n" \ 284 " " insn " %z0, %1\n" \ 285 _ASM_EXTABLE(1b, %l2) \ 286 : : "rJ" (__x), "m"(*(ptr)) : : label); \ 287 } while (0) 288 289 #ifdef CONFIG_64BIT 290 #define __put_user_8(x, ptr, label) \ 291 __put_user_asm("sd", x, ptr, label) 292 #else /* !CONFIG_64BIT */ 293 #define __put_user_8(x, ptr, label) \ 294 do { \ 295 u32 __user *__ptr = (u32 __user *)(ptr); \ 296 u64 __x = (__typeof__((x)-(x)))(x); \ 297 asm goto( \ 298 "1:\n" \ 299 " sw %z0, %2\n" \ 300 "2:\n" \ 301 " sw %z1, %3\n" \ 302 _ASM_EXTABLE(1b, %l4) \ 303 _ASM_EXTABLE(2b, %l4) \ 304 : : "rJ" (__x), "rJ" (__x >> 32), \ 305 "m" (__ptr[__LSW]), \ 306 "m" (__ptr[__MSW]) : : label); \ 307 } while (0) 308 #endif /* CONFIG_64BIT */ 309 310 #define __put_user_nocheck(x, __gu_ptr, label) \ 311 do { \ 312 if (!IS_ENABLED(CONFIG_HAVE_EFFICIENT_UNALIGNED_ACCESS) && \ 313 !IS_ALIGNED((uintptr_t)__gu_ptr, sizeof(*__gu_ptr))) { \ 314 __inttype(x) val = (__inttype(x))x; \ 315 if (__asm_copy_to_user_sum_enabled(__gu_ptr, &(val), sizeof(*__gu_ptr))) \ 316 goto label; \ 317 break; \ 318 } \ 319 switch (sizeof(*__gu_ptr)) { \ 320 case 1: \ 321 __put_user_asm("sb", (x), __gu_ptr, label); \ 322 break; \ 323 case 2: \ 324 __put_user_asm("sh", (x), __gu_ptr, label); \ 325 break; \ 326 case 4: \ 327 __put_user_asm("sw", (x), __gu_ptr, label); \ 328 break; \ 329 case 8: \ 330 __put_user_8((x), __gu_ptr, label); \ 331 break; \ 332 default: \ 333 BUILD_BUG(); \ 334 } \ 335 } while (0) 336 337 #define __put_user_error(x, ptr, err) \ 338 do { \ 339 __label__ err_label; \ 340 __put_user_nocheck(x, ptr, err_label); \ 341 break; \ 342 err_label: \ 343 (err) = -EFAULT; \ 344 } while (0) 345 346 /** 347 * __put_user: - Write a simple value into user space, with less checking. 348 * @x: Value to copy to user space. 349 * @ptr: Destination address, in user space. 350 * 351 * Context: User context only. This function may sleep. 352 * 353 * This macro copies a single simple value from kernel space to user 354 * space. It supports simple types like char and int, but not larger 355 * data types like structures or arrays. 356 * 357 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 358 * to the result of dereferencing @ptr. The value of @x is copied to avoid 359 * re-ordering where @x is evaluated inside the block that enables user-space 360 * access (thus bypassing user space protection if @x is a function). 361 * 362 * Caller must check the pointer with access_ok() before calling this 363 * function. 364 * 365 * Returns zero on success, or -EFAULT on error. 366 */ 367 #define __put_user(x, ptr) \ 368 ({ \ 369 __typeof__(*(ptr)) __user *__gu_ptr = untagged_addr(ptr); \ 370 __typeof__(*__gu_ptr) __val = (x); \ 371 long __pu_err = 0; \ 372 \ 373 __chk_user_ptr(__gu_ptr); \ 374 \ 375 __enable_user_access(); \ 376 __put_user_error(__val, __gu_ptr, __pu_err); \ 377 __disable_user_access(); \ 378 \ 379 __pu_err; \ 380 }) 381 382 /** 383 * put_user: - Write a simple value into user space. 384 * @x: Value to copy to user space. 385 * @ptr: Destination address, in user space. 386 * 387 * Context: User context only. This function may sleep. 388 * 389 * This macro copies a single simple value from kernel space to user 390 * space. It supports simple types like char and int, but not larger 391 * data types like structures or arrays. 392 * 393 * @ptr must have pointer-to-simple-variable type, and @x must be assignable 394 * to the result of dereferencing @ptr. 395 * 396 * Returns zero on success, or -EFAULT on error. 397 */ 398 #define put_user(x, ptr) \ 399 ({ \ 400 __typeof__(*(ptr)) __user *__p = (ptr); \ 401 might_fault(); \ 402 access_ok(__p, sizeof(*__p)) ? \ 403 __put_user((x), __p) : \ 404 -EFAULT; \ 405 }) 406 407 408 unsigned long __must_check __asm_copy_to_user(void __user *to, 409 const void *from, unsigned long n); 410 unsigned long __must_check __asm_copy_from_user(void *to, 411 const void __user *from, unsigned long n); 412 413 static inline unsigned long 414 raw_copy_from_user(void *to, const void __user *from, unsigned long n) 415 { 416 return __asm_copy_from_user(to, untagged_addr(from), n); 417 } 418 419 static inline unsigned long 420 raw_copy_to_user(void __user *to, const void *from, unsigned long n) 421 { 422 return __asm_copy_to_user(untagged_addr(to), from, n); 423 } 424 425 extern long strncpy_from_user(char *dest, const char __user *src, long count); 426 427 extern long __must_check strnlen_user(const char __user *str, long n); 428 429 extern 430 unsigned long __must_check __clear_user(void __user *addr, unsigned long n); 431 432 static inline 433 unsigned long __must_check clear_user(void __user *to, unsigned long n) 434 { 435 might_fault(); 436 return access_ok(to, n) ? 437 __clear_user(untagged_addr(to), n) : n; 438 } 439 440 #define __get_kernel_nofault(dst, src, type, err_label) \ 441 __get_user_nocheck(*((type *)(dst)), (type *)(src), err_label) 442 443 #define __put_kernel_nofault(dst, src, type, err_label) \ 444 __put_user_nocheck(*((type *)(src)), (type *)(dst), err_label) 445 446 static __must_check __always_inline bool user_access_begin(const void __user *ptr, size_t len) 447 { 448 if (unlikely(!access_ok(ptr, len))) 449 return 0; 450 __enable_user_access(); 451 return 1; 452 } 453 #define user_access_begin user_access_begin 454 #define user_access_end __disable_user_access 455 456 static inline unsigned long user_access_save(void) { return 0UL; } 457 static inline void user_access_restore(unsigned long enabled) { } 458 459 /* 460 * We want the unsafe accessors to always be inlined and use 461 * the error labels - thus the macro games. 462 */ 463 #define unsafe_put_user(x, ptr, label) \ 464 __put_user_nocheck(x, (ptr), label) 465 466 #define unsafe_get_user(x, ptr, label) do { \ 467 __inttype(*(ptr)) __gu_val; \ 468 __get_user_nocheck(__gu_val, (ptr), label); \ 469 (x) = (__force __typeof__(*(ptr)))__gu_val; \ 470 } while (0) 471 472 #define unsafe_copy_to_user(_dst, _src, _len, label) \ 473 if (__asm_copy_to_user_sum_enabled(_dst, _src, _len)) \ 474 goto label; 475 476 #define unsafe_copy_from_user(_dst, _src, _len, label) \ 477 if (__asm_copy_from_user_sum_enabled(_dst, _src, _len)) \ 478 goto label; 479 480 #else /* CONFIG_MMU */ 481 #include <asm-generic/uaccess.h> 482 #endif /* CONFIG_MMU */ 483 #endif /* _ASM_RISCV_UACCESS_H */ 484