1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * S390 version 4 * Copyright IBM Corp. 1999, 2000 5 * Author(s): Hartmut Penner (hp@de.ibm.com), 6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * 8 * Derived from "include/asm-i386/uaccess.h" 9 */ 10 #ifndef __S390_UACCESS_H 11 #define __S390_UACCESS_H 12 13 /* 14 * User space memory access functions 15 */ 16 #include <linux/pgtable.h> 17 #include <asm/asm-extable.h> 18 #include <asm/processor.h> 19 #include <asm/extable.h> 20 #include <asm/facility.h> 21 #include <asm-generic/access_ok.h> 22 #include <asm/asce.h> 23 #include <linux/instrumented.h> 24 25 void debug_user_asce(int exit); 26 27 #ifdef CONFIG_KMSAN 28 #define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory 29 #else 30 #define uaccess_kmsan_or_inline __always_inline 31 #endif 32 33 #define INLINE_COPY_USER 34 35 static uaccess_kmsan_or_inline __must_check unsigned long 36 raw_copy_from_user(void *to, const void __user *from, unsigned long size) 37 { 38 unsigned long osize; 39 int cc; 40 41 while (1) { 42 osize = size; 43 asm_inline volatile( 44 " lhi %%r0,%[spec]\n" 45 "0: mvcos %[to],%[from],%[size]\n" 46 "1: nopr %%r7\n" 47 CC_IPM(cc) 48 EX_TABLE_UA_MVCOS_FROM(0b, 0b) 49 EX_TABLE_UA_MVCOS_FROM(1b, 0b) 50 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char *)to) 51 : [spec] "I" (0x81), [from] "Q" (*(const char __user *)from) 52 : CC_CLOBBER_LIST("memory", "0")); 53 if (__builtin_constant_p(osize) && osize <= 4096) 54 return osize - size; 55 if (likely(CC_TRANSFORM(cc) == 0)) 56 return osize - size; 57 size -= 4096; 58 to += 4096; 59 from += 4096; 60 } 61 } 62 63 static uaccess_kmsan_or_inline __must_check unsigned long 64 raw_copy_to_user(void __user *to, const void *from, unsigned long size) 65 { 66 unsigned long osize; 67 int cc; 68 69 while (1) { 70 osize = size; 71 asm_inline volatile( 72 " llilh %%r0,%[spec]\n" 73 "0: mvcos %[to],%[from],%[size]\n" 74 "1: nopr %%r7\n" 75 CC_IPM(cc) 76 EX_TABLE_UA_MVCOS_TO(0b, 0b) 77 EX_TABLE_UA_MVCOS_TO(1b, 0b) 78 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) 79 : [spec] "I" (0x81), [from] "Q" (*(const char *)from) 80 : CC_CLOBBER_LIST("memory", "0")); 81 if (__builtin_constant_p(osize) && osize <= 4096) 82 return osize - size; 83 if (likely(CC_TRANSFORM(cc) == 0)) 84 return osize - size; 85 size -= 4096; 86 to += 4096; 87 from += 4096; 88 } 89 } 90 91 unsigned long __must_check 92 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key); 93 94 static __always_inline unsigned long __must_check 95 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key) 96 { 97 if (check_copy_size(to, n, false)) 98 n = _copy_from_user_key(to, from, n, key); 99 return n; 100 } 101 102 unsigned long __must_check 103 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key); 104 105 static __always_inline unsigned long __must_check 106 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key) 107 { 108 if (check_copy_size(from, n, true)) 109 n = _copy_to_user_key(to, from, n, key); 110 return n; 111 } 112 113 int __noreturn __put_user_bad(void); 114 115 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 116 117 #define DEFINE_PUT_USER_NOINSTR(type) \ 118 static uaccess_kmsan_or_inline int \ 119 __put_user_##type##_noinstr(unsigned type __user *to, \ 120 unsigned type *from, \ 121 unsigned long size) \ 122 { \ 123 asm goto( \ 124 " llilh %%r0,%[spec]\n" \ 125 "0: mvcos %[to],%[from],%[size]\n" \ 126 "1: nopr %%r7\n" \ 127 EX_TABLE(0b, %l[Efault]) \ 128 EX_TABLE(1b, %l[Efault]) \ 129 : [to] "+Q" (*to) \ 130 : [size] "d" (size), [from] "Q" (*from), \ 131 [spec] "I" (0x81) \ 132 : "cc", "0" \ 133 : Efault \ 134 ); \ 135 return 0; \ 136 Efault: \ 137 return -EFAULT; \ 138 } 139 140 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 141 142 #define DEFINE_PUT_USER_NOINSTR(type) \ 143 static uaccess_kmsan_or_inline int \ 144 __put_user_##type##_noinstr(unsigned type __user *to, \ 145 unsigned type *from, \ 146 unsigned long size) \ 147 { \ 148 int rc; \ 149 \ 150 asm_inline volatile( \ 151 " llilh %%r0,%[spec]\n" \ 152 "0: mvcos %[to],%[from],%[size]\n" \ 153 "1: lhi %[rc],0\n" \ 154 "2:\n" \ 155 EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ 156 EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ 157 : [rc] "=d" (rc), [to] "+Q" (*to) \ 158 : [size] "d" (size), [from] "Q" (*from), \ 159 [spec] "I" (0x81) \ 160 : "cc", "0"); \ 161 return rc; \ 162 } 163 164 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 165 166 DEFINE_PUT_USER_NOINSTR(char); 167 DEFINE_PUT_USER_NOINSTR(short); 168 DEFINE_PUT_USER_NOINSTR(int); 169 DEFINE_PUT_USER_NOINSTR(long); 170 171 #define DEFINE_PUT_USER(type) \ 172 static __always_inline int \ 173 __put_user_##type(unsigned type __user *to, unsigned type *from, \ 174 unsigned long size) \ 175 { \ 176 int rc; \ 177 \ 178 rc = __put_user_##type##_noinstr(to, from, size); \ 179 instrument_put_user(*from, to, size); \ 180 return rc; \ 181 } 182 183 DEFINE_PUT_USER(char); 184 DEFINE_PUT_USER(short); 185 DEFINE_PUT_USER(int); 186 DEFINE_PUT_USER(long); 187 188 #define __put_user(x, ptr) \ 189 ({ \ 190 __typeof__(*(ptr)) __x = (x); \ 191 int __prc; \ 192 \ 193 __chk_user_ptr(ptr); \ 194 switch (sizeof(*(ptr))) { \ 195 case 1: \ 196 __prc = __put_user_char((unsigned char __user *)(ptr), \ 197 (unsigned char *)&__x, \ 198 sizeof(*(ptr))); \ 199 break; \ 200 case 2: \ 201 __prc = __put_user_short((unsigned short __user *)(ptr),\ 202 (unsigned short *)&__x, \ 203 sizeof(*(ptr))); \ 204 break; \ 205 case 4: \ 206 __prc = __put_user_int((unsigned int __user *)(ptr), \ 207 (unsigned int *)&__x, \ 208 sizeof(*(ptr))); \ 209 break; \ 210 case 8: \ 211 __prc = __put_user_long((unsigned long __user *)(ptr), \ 212 (unsigned long *)&__x, \ 213 sizeof(*(ptr))); \ 214 break; \ 215 default: \ 216 __prc = __put_user_bad(); \ 217 break; \ 218 } \ 219 __builtin_expect(__prc, 0); \ 220 }) 221 222 #define put_user(x, ptr) \ 223 ({ \ 224 might_fault(); \ 225 __put_user(x, ptr); \ 226 }) 227 228 int __noreturn __get_user_bad(void); 229 230 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 231 232 #define DEFINE_GET_USER_NOINSTR(type) \ 233 static uaccess_kmsan_or_inline int \ 234 __get_user_##type##_noinstr(unsigned type *to, \ 235 const unsigned type __user *from, \ 236 unsigned long size) \ 237 { \ 238 asm goto( \ 239 " lhi %%r0,%[spec]\n" \ 240 "0: mvcos %[to],%[from],%[size]\n" \ 241 "1: nopr %%r7\n" \ 242 EX_TABLE(0b, %l[Efault]) \ 243 EX_TABLE(1b, %l[Efault]) \ 244 : [to] "=Q" (*to) \ 245 : [size] "d" (size), [from] "Q" (*from), \ 246 [spec] "I" (0x81) \ 247 : "cc", "0" \ 248 : Efault \ 249 ); \ 250 return 0; \ 251 Efault: \ 252 *to = 0; \ 253 return -EFAULT; \ 254 } 255 256 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 257 258 #define DEFINE_GET_USER_NOINSTR(type) \ 259 static uaccess_kmsan_or_inline int \ 260 __get_user_##type##_noinstr(unsigned type *to, \ 261 const unsigned type __user *from, \ 262 unsigned long size) \ 263 { \ 264 int rc; \ 265 \ 266 asm_inline volatile( \ 267 " lhi %%r0,%[spec]\n" \ 268 "0: mvcos %[to],%[from],%[size]\n" \ 269 "1: lhi %[rc],0\n" \ 270 "2:\n" \ 271 EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ 272 EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ 273 : [rc] "=d" (rc), [to] "=Q" (*to) \ 274 : [size] "d" (size), [from] "Q" (*from), \ 275 [spec] "I" (0x81) \ 276 : "cc", "0"); \ 277 if (likely(!rc)) \ 278 return 0; \ 279 *to = 0; \ 280 return rc; \ 281 } 282 283 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 284 285 DEFINE_GET_USER_NOINSTR(char); 286 DEFINE_GET_USER_NOINSTR(short); 287 DEFINE_GET_USER_NOINSTR(int); 288 DEFINE_GET_USER_NOINSTR(long); 289 290 #define DEFINE_GET_USER(type) \ 291 static __always_inline int \ 292 __get_user_##type(unsigned type *to, const unsigned type __user *from, \ 293 unsigned long size) \ 294 { \ 295 int rc; \ 296 \ 297 rc = __get_user_##type##_noinstr(to, from, size); \ 298 instrument_get_user(*to); \ 299 return rc; \ 300 } 301 302 DEFINE_GET_USER(char); 303 DEFINE_GET_USER(short); 304 DEFINE_GET_USER(int); 305 DEFINE_GET_USER(long); 306 307 #define __get_user(x, ptr) \ 308 ({ \ 309 const __user void *____guptr = (ptr); \ 310 int __grc; \ 311 \ 312 __chk_user_ptr(ptr); \ 313 switch (sizeof(*(ptr))) { \ 314 case 1: { \ 315 const unsigned char __user *__guptr = ____guptr; \ 316 unsigned char __x; \ 317 \ 318 __grc = __get_user_char(&__x, __guptr, sizeof(*(ptr))); \ 319 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 320 break; \ 321 }; \ 322 case 2: { \ 323 const unsigned short __user *__guptr = ____guptr; \ 324 unsigned short __x; \ 325 \ 326 __grc = __get_user_short(&__x, __guptr, sizeof(*(ptr)));\ 327 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 328 break; \ 329 }; \ 330 case 4: { \ 331 const unsigned int __user *__guptr = ____guptr; \ 332 unsigned int __x; \ 333 \ 334 __grc = __get_user_int(&__x, __guptr, sizeof(*(ptr))); \ 335 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 336 break; \ 337 }; \ 338 case 8: { \ 339 const unsigned long __user *__guptr = ____guptr; \ 340 unsigned long __x; \ 341 \ 342 __grc = __get_user_long(&__x, __guptr, sizeof(*(ptr))); \ 343 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 344 break; \ 345 }; \ 346 default: \ 347 __grc = __get_user_bad(); \ 348 break; \ 349 } \ 350 __builtin_expect(__grc, 0); \ 351 }) 352 353 #define get_user(x, ptr) \ 354 ({ \ 355 might_fault(); \ 356 __get_user(x, ptr); \ 357 }) 358 359 /* 360 * Copy a null terminated string from userspace. 361 */ 362 long __must_check strncpy_from_user(char *dst, const char __user *src, long count); 363 364 long __must_check strnlen_user(const char __user *src, long count); 365 366 static uaccess_kmsan_or_inline __must_check unsigned long 367 __clear_user(void __user *to, unsigned long size) 368 { 369 unsigned long osize; 370 int cc; 371 372 while (1) { 373 osize = size; 374 asm_inline volatile( 375 " llilh %%r0,%[spec]\n" 376 "0: mvcos %[to],%[from],%[size]\n" 377 "1: nopr %%r7\n" 378 CC_IPM(cc) 379 EX_TABLE_UA_MVCOS_TO(0b, 0b) 380 EX_TABLE_UA_MVCOS_TO(1b, 0b) 381 : CC_OUT(cc, cc), [size] "+d" (size), [to] "=Q" (*(char __user *)to) 382 : [spec] "I" (0x81), [from] "Q" (*(const char *)empty_zero_page) 383 : CC_CLOBBER_LIST("memory", "0")); 384 if (__builtin_constant_p(osize) && osize <= 4096) 385 return osize - size; 386 if (CC_TRANSFORM(cc) == 0) 387 return osize - size; 388 size -= 4096; 389 to += 4096; 390 } 391 } 392 393 static __always_inline unsigned long __must_check clear_user(void __user *to, unsigned long n) 394 { 395 might_fault(); 396 return __clear_user(to, n); 397 } 398 399 void *__s390_kernel_write(void *dst, const void *src, size_t size); 400 401 static inline void *s390_kernel_write(void *dst, const void *src, size_t size) 402 { 403 if (__is_defined(__DECOMPRESSOR)) 404 return memcpy(dst, src, size); 405 return __s390_kernel_write(dst, src, size); 406 } 407 408 void __noreturn __mvc_kernel_nofault_bad(void); 409 410 #if defined(CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && defined(CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS) 411 412 #define __mvc_kernel_nofault(dst, src, type, err_label) \ 413 do { \ 414 switch (sizeof(type)) { \ 415 case 1: \ 416 case 2: \ 417 case 4: \ 418 case 8: \ 419 asm goto( \ 420 "0: mvc %O[_dst](%[_len],%R[_dst]),%[_src]\n" \ 421 "1: nopr %%r7\n" \ 422 EX_TABLE(0b, %l[err_label]) \ 423 EX_TABLE(1b, %l[err_label]) \ 424 : [_dst] "=Q" (*(type *)dst) \ 425 : [_src] "Q" (*(type *)(src)), \ 426 [_len] "I" (sizeof(type)) \ 427 : \ 428 : err_label); \ 429 break; \ 430 default: \ 431 __mvc_kernel_nofault_bad(); \ 432 break; \ 433 } \ 434 } while (0) 435 436 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ 437 438 #define __mvc_kernel_nofault(dst, src, type, err_label) \ 439 do { \ 440 type *(__dst) = (type *)(dst); \ 441 int __rc; \ 442 \ 443 switch (sizeof(type)) { \ 444 case 1: \ 445 case 2: \ 446 case 4: \ 447 case 8: \ 448 asm_inline volatile( \ 449 "0: mvc 0(%[_len],%[_dst]),%[_src]\n" \ 450 "1: lhi %[_rc],0\n" \ 451 "2:\n" \ 452 EX_TABLE_UA_FAULT(0b, 2b, %[_rc]) \ 453 EX_TABLE_UA_FAULT(1b, 2b, %[_rc]) \ 454 : [_rc] "=d" (__rc), \ 455 "=m" (*__dst) \ 456 : [_src] "Q" (*(type *)(src)), \ 457 [_dst] "a" (__dst), \ 458 [_len] "I" (sizeof(type))); \ 459 if (__rc) \ 460 goto err_label; \ 461 break; \ 462 default: \ 463 __mvc_kernel_nofault_bad(); \ 464 break; \ 465 } \ 466 } while (0) 467 468 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ 469 470 #define arch_get_kernel_nofault __mvc_kernel_nofault 471 #define arch_put_kernel_nofault __mvc_kernel_nofault 472 473 int __cmpxchg_key1(void *address, unsigned char *uval, unsigned char old, 474 unsigned char new, unsigned long key); 475 int __cmpxchg_key2(void *address, unsigned short *uval, unsigned short old, 476 unsigned short new, unsigned long key); 477 int __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old, 478 unsigned int new, unsigned long key); 479 int __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old, 480 unsigned long new, unsigned long key); 481 int __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old, 482 __uint128_t new, unsigned long key); 483 484 #endif /* __S390_UACCESS_H */ 485