1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * S390 version 4 * Copyright IBM Corp. 1999, 2000 5 * Author(s): Hartmut Penner (hp@de.ibm.com), 6 * Martin Schwidefsky (schwidefsky@de.ibm.com) 7 * 8 * Derived from "include/asm-i386/uaccess.h" 9 */ 10 #ifndef __S390_UACCESS_H 11 #define __S390_UACCESS_H 12 13 /* 14 * User space memory access functions 15 */ 16 #include <asm/asm-extable.h> 17 #include <asm/processor.h> 18 #include <asm/extable.h> 19 #include <asm/facility.h> 20 #include <asm-generic/access_ok.h> 21 #include <linux/instrumented.h> 22 23 void debug_user_asce(int exit); 24 25 union oac { 26 unsigned int val; 27 struct { 28 struct { 29 unsigned short key : 4; 30 unsigned short : 4; 31 unsigned short as : 2; 32 unsigned short : 4; 33 unsigned short k : 1; 34 unsigned short a : 1; 35 } oac1; 36 struct { 37 unsigned short key : 4; 38 unsigned short : 4; 39 unsigned short as : 2; 40 unsigned short : 4; 41 unsigned short k : 1; 42 unsigned short a : 1; 43 } oac2; 44 }; 45 }; 46 47 static __always_inline __must_check unsigned long 48 raw_copy_from_user_key(void *to, const void __user *from, unsigned long size, unsigned long key) 49 { 50 unsigned long rem; 51 union oac spec = { 52 .oac2.key = key, 53 .oac2.as = PSW_BITS_AS_SECONDARY, 54 .oac2.k = 1, 55 .oac2.a = 1, 56 }; 57 58 asm_inline volatile( 59 " lr %%r0,%[spec]\n" 60 "0: mvcos 0(%[to]),0(%[from]),%[size]\n" 61 "1: jz 5f\n" 62 " algr %[size],%[val]\n" 63 " slgr %[from],%[val]\n" 64 " slgr %[to],%[val]\n" 65 " j 0b\n" 66 "2: la %[rem],4095(%[from])\n" /* rem = from + 4095 */ 67 " nr %[rem],%[val]\n" /* rem = (from + 4095) & -4096 */ 68 " slgr %[rem],%[from]\n" 69 " clgr %[size],%[rem]\n" /* copy crosses next page boundary? */ 70 " jnh 6f\n" 71 "3: mvcos 0(%[to]),0(%[from]),%[rem]\n" 72 "4: slgr %[size],%[rem]\n" 73 " j 6f\n" 74 "5: lghi %[size],0\n" 75 "6:\n" 76 EX_TABLE(0b, 2b) 77 EX_TABLE(1b, 2b) 78 EX_TABLE(3b, 6b) 79 EX_TABLE(4b, 6b) 80 : [size] "+&a" (size), [from] "+&a" (from), [to] "+&a" (to), [rem] "=&a" (rem) 81 : [val] "a" (-4096UL), [spec] "d" (spec.val) 82 : "cc", "memory", "0"); 83 return size; 84 } 85 86 static __always_inline __must_check unsigned long 87 raw_copy_from_user(void *to, const void __user *from, unsigned long n) 88 { 89 return raw_copy_from_user_key(to, from, n, 0); 90 } 91 92 static __always_inline __must_check unsigned long 93 raw_copy_to_user_key(void __user *to, const void *from, unsigned long size, unsigned long key) 94 { 95 unsigned long rem; 96 union oac spec = { 97 .oac1.key = key, 98 .oac1.as = PSW_BITS_AS_SECONDARY, 99 .oac1.k = 1, 100 .oac1.a = 1, 101 }; 102 103 asm_inline volatile( 104 " lr %%r0,%[spec]\n" 105 "0: mvcos 0(%[to]),0(%[from]),%[size]\n" 106 "1: jz 5f\n" 107 " algr %[size],%[val]\n" 108 " slgr %[to],%[val]\n" 109 " slgr %[from],%[val]\n" 110 " j 0b\n" 111 "2: la %[rem],4095(%[to])\n" /* rem = to + 4095 */ 112 " nr %[rem],%[val]\n" /* rem = (to + 4095) & -4096 */ 113 " slgr %[rem],%[to]\n" 114 " clgr %[size],%[rem]\n" /* copy crosses next page boundary? */ 115 " jnh 6f\n" 116 "3: mvcos 0(%[to]),0(%[from]),%[rem]\n" 117 "4: slgr %[size],%[rem]\n" 118 " j 6f\n" 119 "5: lghi %[size],0\n" 120 "6:\n" 121 EX_TABLE(0b, 2b) 122 EX_TABLE(1b, 2b) 123 EX_TABLE(3b, 6b) 124 EX_TABLE(4b, 6b) 125 : [size] "+&a" (size), [to] "+&a" (to), [from] "+&a" (from), [rem] "=&a" (rem) 126 : [val] "a" (-4096UL), [spec] "d" (spec.val) 127 : "cc", "memory", "0"); 128 return size; 129 } 130 131 static __always_inline __must_check unsigned long 132 raw_copy_to_user(void __user *to, const void *from, unsigned long n) 133 { 134 return raw_copy_to_user_key(to, from, n, 0); 135 } 136 137 unsigned long __must_check 138 _copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key); 139 140 static __always_inline unsigned long __must_check 141 copy_from_user_key(void *to, const void __user *from, unsigned long n, unsigned long key) 142 { 143 if (check_copy_size(to, n, false)) 144 n = _copy_from_user_key(to, from, n, key); 145 return n; 146 } 147 148 unsigned long __must_check 149 _copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key); 150 151 static __always_inline unsigned long __must_check 152 copy_to_user_key(void __user *to, const void *from, unsigned long n, unsigned long key) 153 { 154 if (check_copy_size(from, n, true)) 155 n = _copy_to_user_key(to, from, n, key); 156 return n; 157 } 158 159 int __noreturn __put_user_bad(void); 160 161 #ifdef CONFIG_KMSAN 162 #define uaccess_kmsan_or_inline noinline __maybe_unused __no_sanitize_memory 163 #else 164 #define uaccess_kmsan_or_inline __always_inline 165 #endif 166 167 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 168 169 #define DEFINE_PUT_USER_NOINSTR(type) \ 170 static uaccess_kmsan_or_inline int \ 171 __put_user_##type##_noinstr(unsigned type __user *to, \ 172 unsigned type *from, \ 173 unsigned long size) \ 174 { \ 175 asm goto( \ 176 " llilh %%r0,%[spec]\n" \ 177 "0: mvcos %[to],%[from],%[size]\n" \ 178 "1: nopr %%r7\n" \ 179 EX_TABLE(0b, %l[Efault]) \ 180 EX_TABLE(1b, %l[Efault]) \ 181 : [to] "+Q" (*to) \ 182 : [size] "d" (size), [from] "Q" (*from), \ 183 [spec] "I" (0x81) \ 184 : "cc", "0" \ 185 : Efault \ 186 ); \ 187 return 0; \ 188 Efault: \ 189 return -EFAULT; \ 190 } 191 192 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 193 194 #define DEFINE_PUT_USER_NOINSTR(type) \ 195 static uaccess_kmsan_or_inline int \ 196 __put_user_##type##_noinstr(unsigned type __user *to, \ 197 unsigned type *from, \ 198 unsigned long size) \ 199 { \ 200 int rc; \ 201 \ 202 asm volatile( \ 203 " llilh %%r0,%[spec]\n" \ 204 "0: mvcos %[to],%[from],%[size]\n" \ 205 "1: lhi %[rc],0\n" \ 206 "2:\n" \ 207 EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ 208 EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ 209 : [rc] "=d" (rc), [to] "+Q" (*to) \ 210 : [size] "d" (size), [from] "Q" (*from), \ 211 [spec] "I" (0x81) \ 212 : "cc", "0"); \ 213 return rc; \ 214 } 215 216 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 217 218 DEFINE_PUT_USER_NOINSTR(char); 219 DEFINE_PUT_USER_NOINSTR(short); 220 DEFINE_PUT_USER_NOINSTR(int); 221 DEFINE_PUT_USER_NOINSTR(long); 222 223 #define DEFINE_PUT_USER(type) \ 224 static __always_inline int \ 225 __put_user_##type(unsigned type __user *to, unsigned type *from, \ 226 unsigned long size) \ 227 { \ 228 int rc; \ 229 \ 230 rc = __put_user_##type##_noinstr(to, from, size); \ 231 instrument_put_user(*from, to, size); \ 232 return rc; \ 233 } 234 235 DEFINE_PUT_USER(char); 236 DEFINE_PUT_USER(short); 237 DEFINE_PUT_USER(int); 238 DEFINE_PUT_USER(long); 239 240 #define __put_user(x, ptr) \ 241 ({ \ 242 __typeof__(*(ptr)) __x = (x); \ 243 int __prc; \ 244 \ 245 __chk_user_ptr(ptr); \ 246 switch (sizeof(*(ptr))) { \ 247 case 1: \ 248 __prc = __put_user_char((unsigned char __user *)(ptr), \ 249 (unsigned char *)&__x, \ 250 sizeof(*(ptr))); \ 251 break; \ 252 case 2: \ 253 __prc = __put_user_short((unsigned short __user *)(ptr),\ 254 (unsigned short *)&__x, \ 255 sizeof(*(ptr))); \ 256 break; \ 257 case 4: \ 258 __prc = __put_user_int((unsigned int __user *)(ptr), \ 259 (unsigned int *)&__x, \ 260 sizeof(*(ptr))); \ 261 break; \ 262 case 8: \ 263 __prc = __put_user_long((unsigned long __user *)(ptr), \ 264 (unsigned long *)&__x, \ 265 sizeof(*(ptr))); \ 266 break; \ 267 default: \ 268 __prc = __put_user_bad(); \ 269 break; \ 270 } \ 271 __builtin_expect(__prc, 0); \ 272 }) 273 274 #define put_user(x, ptr) \ 275 ({ \ 276 might_fault(); \ 277 __put_user(x, ptr); \ 278 }) 279 280 int __noreturn __get_user_bad(void); 281 282 #ifdef CONFIG_CC_HAS_ASM_GOTO_OUTPUT 283 284 #define DEFINE_GET_USER_NOINSTR(type) \ 285 static uaccess_kmsan_or_inline int \ 286 __get_user_##type##_noinstr(unsigned type *to, \ 287 const unsigned type __user *from, \ 288 unsigned long size) \ 289 { \ 290 asm goto( \ 291 " lhi %%r0,%[spec]\n" \ 292 "0: mvcos %[to],%[from],%[size]\n" \ 293 "1: nopr %%r7\n" \ 294 EX_TABLE(0b, %l[Efault]) \ 295 EX_TABLE(1b, %l[Efault]) \ 296 : [to] "=Q" (*to) \ 297 : [size] "d" (size), [from] "Q" (*from), \ 298 [spec] "I" (0x81) \ 299 : "cc", "0" \ 300 : Efault \ 301 ); \ 302 return 0; \ 303 Efault: \ 304 *to = 0; \ 305 return -EFAULT; \ 306 } 307 308 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 309 310 #define DEFINE_GET_USER_NOINSTR(type) \ 311 static uaccess_kmsan_or_inline int \ 312 __get_user_##type##_noinstr(unsigned type *to, \ 313 const unsigned type __user *from, \ 314 unsigned long size) \ 315 { \ 316 int rc; \ 317 \ 318 asm volatile( \ 319 " lhi %%r0,%[spec]\n" \ 320 "0: mvcos %[to],%[from],%[size]\n" \ 321 "1: lhi %[rc],0\n" \ 322 "2:\n" \ 323 EX_TABLE_UA_FAULT(0b, 2b, %[rc]) \ 324 EX_TABLE_UA_FAULT(1b, 2b, %[rc]) \ 325 : [rc] "=d" (rc), [to] "=Q" (*to) \ 326 : [size] "d" (size), [from] "Q" (*from), \ 327 [spec] "I" (0x81) \ 328 : "cc", "0"); \ 329 if (likely(!rc)) \ 330 return 0; \ 331 *to = 0; \ 332 return rc; \ 333 } 334 335 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT */ 336 337 DEFINE_GET_USER_NOINSTR(char); 338 DEFINE_GET_USER_NOINSTR(short); 339 DEFINE_GET_USER_NOINSTR(int); 340 DEFINE_GET_USER_NOINSTR(long); 341 342 #define DEFINE_GET_USER(type) \ 343 static __always_inline int \ 344 __get_user_##type(unsigned type *to, const unsigned type __user *from, \ 345 unsigned long size) \ 346 { \ 347 int rc; \ 348 \ 349 rc = __get_user_##type##_noinstr(to, from, size); \ 350 instrument_get_user(*to); \ 351 return rc; \ 352 } 353 354 DEFINE_GET_USER(char); 355 DEFINE_GET_USER(short); 356 DEFINE_GET_USER(int); 357 DEFINE_GET_USER(long); 358 359 #define __get_user(x, ptr) \ 360 ({ \ 361 const __user void *____guptr = (ptr); \ 362 int __grc; \ 363 \ 364 __chk_user_ptr(ptr); \ 365 switch (sizeof(*(ptr))) { \ 366 case 1: { \ 367 const unsigned char __user *__guptr = ____guptr; \ 368 unsigned char __x; \ 369 \ 370 __grc = __get_user_char(&__x, __guptr, sizeof(*(ptr))); \ 371 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 372 break; \ 373 }; \ 374 case 2: { \ 375 const unsigned short __user *__guptr = ____guptr; \ 376 unsigned short __x; \ 377 \ 378 __grc = __get_user_short(&__x, __guptr, sizeof(*(ptr)));\ 379 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 380 break; \ 381 }; \ 382 case 4: { \ 383 const unsigned int __user *__guptr = ____guptr; \ 384 unsigned int __x; \ 385 \ 386 __grc = __get_user_int(&__x, __guptr, sizeof(*(ptr))); \ 387 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 388 break; \ 389 }; \ 390 case 8: { \ 391 const unsigned long __user *__guptr = ____guptr; \ 392 unsigned long __x; \ 393 \ 394 __grc = __get_user_long(&__x, __guptr, sizeof(*(ptr))); \ 395 (x) = *(__force __typeof__(*(ptr)) *)&__x; \ 396 break; \ 397 }; \ 398 default: \ 399 __grc = __get_user_bad(); \ 400 break; \ 401 } \ 402 __builtin_expect(__grc, 0); \ 403 }) 404 405 #define get_user(x, ptr) \ 406 ({ \ 407 might_fault(); \ 408 __get_user(x, ptr); \ 409 }) 410 411 /* 412 * Copy a null terminated string from userspace. 413 */ 414 long __must_check strncpy_from_user(char *dst, const char __user *src, long count); 415 416 long __must_check strnlen_user(const char __user *src, long count); 417 418 /* 419 * Zero Userspace 420 */ 421 unsigned long __must_check __clear_user(void __user *to, unsigned long size); 422 423 static inline unsigned long __must_check clear_user(void __user *to, unsigned long n) 424 { 425 might_fault(); 426 return __clear_user(to, n); 427 } 428 429 void *__s390_kernel_write(void *dst, const void *src, size_t size); 430 431 static inline void *s390_kernel_write(void *dst, const void *src, size_t size) 432 { 433 if (__is_defined(__DECOMPRESSOR)) 434 return memcpy(dst, src, size); 435 return __s390_kernel_write(dst, src, size); 436 } 437 438 void __noreturn __mvc_kernel_nofault_bad(void); 439 440 #if defined(CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && defined(CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS) 441 442 #define __mvc_kernel_nofault(dst, src, type, err_label) \ 443 do { \ 444 switch (sizeof(type)) { \ 445 case 1: \ 446 case 2: \ 447 case 4: \ 448 case 8: \ 449 asm goto( \ 450 "0: mvc %O[_dst](%[_len],%R[_dst]),%[_src]\n" \ 451 "1: nopr %%r7\n" \ 452 EX_TABLE(0b, %l[err_label]) \ 453 EX_TABLE(1b, %l[err_label]) \ 454 : [_dst] "=Q" (*(type *)dst) \ 455 : [_src] "Q" (*(type *)(src)), \ 456 [_len] "I" (sizeof(type)) \ 457 : \ 458 : err_label); \ 459 break; \ 460 default: \ 461 __mvc_kernel_nofault_bad(); \ 462 break; \ 463 } \ 464 } while (0) 465 466 #else /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT) && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ 467 468 #define __mvc_kernel_nofault(dst, src, type, err_label) \ 469 do { \ 470 type *(__dst) = (type *)(dst); \ 471 int __rc; \ 472 \ 473 switch (sizeof(type)) { \ 474 case 1: \ 475 case 2: \ 476 case 4: \ 477 case 8: \ 478 asm_inline volatile( \ 479 "0: mvc 0(%[_len],%[_dst]),%[_src]\n" \ 480 "1: lhi %[_rc],0\n" \ 481 "2:\n" \ 482 EX_TABLE_UA_FAULT(0b, 2b, %[_rc]) \ 483 EX_TABLE_UA_FAULT(1b, 2b, %[_rc]) \ 484 : [_rc] "=d" (__rc), \ 485 "=m" (*__dst) \ 486 : [_src] "Q" (*(type *)(src)), \ 487 [_dst] "a" (__dst), \ 488 [_len] "I" (sizeof(type))); \ 489 if (__rc) \ 490 goto err_label; \ 491 break; \ 492 default: \ 493 __mvc_kernel_nofault_bad(); \ 494 break; \ 495 } \ 496 } while (0) 497 498 #endif /* CONFIG_CC_HAS_ASM_GOTO_OUTPUT && CONFIG_CC_HAS_ASM_AOR_FORMAT_FLAGS */ 499 500 #define __get_kernel_nofault __mvc_kernel_nofault 501 #define __put_kernel_nofault __mvc_kernel_nofault 502 503 void __cmpxchg_user_key_called_with_bad_pointer(void); 504 505 #define CMPXCHG_USER_KEY_MAX_LOOPS 128 506 507 static __always_inline int __cmpxchg_user_key(unsigned long address, void *uval, 508 __uint128_t old, __uint128_t new, 509 unsigned long key, int size) 510 { 511 int rc = 0; 512 513 switch (size) { 514 case 1: { 515 unsigned int prev, shift, mask, _old, _new; 516 unsigned long count; 517 518 shift = (3 ^ (address & 3)) << 3; 519 address ^= address & 3; 520 _old = ((unsigned int)old & 0xff) << shift; 521 _new = ((unsigned int)new & 0xff) << shift; 522 mask = ~(0xff << shift); 523 asm volatile( 524 " spka 0(%[key])\n" 525 " sacf 256\n" 526 " llill %[count],%[max_loops]\n" 527 "0: l %[prev],%[address]\n" 528 "1: nr %[prev],%[mask]\n" 529 " xilf %[mask],0xffffffff\n" 530 " or %[new],%[prev]\n" 531 " or %[prev],%[tmp]\n" 532 "2: lr %[tmp],%[prev]\n" 533 "3: cs %[prev],%[new],%[address]\n" 534 "4: jnl 5f\n" 535 " xr %[tmp],%[prev]\n" 536 " xr %[new],%[tmp]\n" 537 " nr %[tmp],%[mask]\n" 538 " jnz 5f\n" 539 " brct %[count],2b\n" 540 "5: sacf 768\n" 541 " spka %[default_key]\n" 542 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 543 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 544 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 545 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 546 : [rc] "+&d" (rc), 547 [prev] "=&d" (prev), 548 [address] "+Q" (*(int *)address), 549 [tmp] "+&d" (_old), 550 [new] "+&d" (_new), 551 [mask] "+&d" (mask), 552 [count] "=a" (count) 553 : [key] "%[count]" (key << 4), 554 [default_key] "J" (PAGE_DEFAULT_KEY), 555 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) 556 : "memory", "cc"); 557 *(unsigned char *)uval = prev >> shift; 558 if (!count) 559 rc = -EAGAIN; 560 return rc; 561 } 562 case 2: { 563 unsigned int prev, shift, mask, _old, _new; 564 unsigned long count; 565 566 shift = (2 ^ (address & 2)) << 3; 567 address ^= address & 2; 568 _old = ((unsigned int)old & 0xffff) << shift; 569 _new = ((unsigned int)new & 0xffff) << shift; 570 mask = ~(0xffff << shift); 571 asm volatile( 572 " spka 0(%[key])\n" 573 " sacf 256\n" 574 " llill %[count],%[max_loops]\n" 575 "0: l %[prev],%[address]\n" 576 "1: nr %[prev],%[mask]\n" 577 " xilf %[mask],0xffffffff\n" 578 " or %[new],%[prev]\n" 579 " or %[prev],%[tmp]\n" 580 "2: lr %[tmp],%[prev]\n" 581 "3: cs %[prev],%[new],%[address]\n" 582 "4: jnl 5f\n" 583 " xr %[tmp],%[prev]\n" 584 " xr %[new],%[tmp]\n" 585 " nr %[tmp],%[mask]\n" 586 " jnz 5f\n" 587 " brct %[count],2b\n" 588 "5: sacf 768\n" 589 " spka %[default_key]\n" 590 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev]) 591 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev]) 592 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev]) 593 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev]) 594 : [rc] "+&d" (rc), 595 [prev] "=&d" (prev), 596 [address] "+Q" (*(int *)address), 597 [tmp] "+&d" (_old), 598 [new] "+&d" (_new), 599 [mask] "+&d" (mask), 600 [count] "=a" (count) 601 : [key] "%[count]" (key << 4), 602 [default_key] "J" (PAGE_DEFAULT_KEY), 603 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS) 604 : "memory", "cc"); 605 *(unsigned short *)uval = prev >> shift; 606 if (!count) 607 rc = -EAGAIN; 608 return rc; 609 } 610 case 4: { 611 unsigned int prev = old; 612 613 asm volatile( 614 " spka 0(%[key])\n" 615 " sacf 256\n" 616 "0: cs %[prev],%[new],%[address]\n" 617 "1: sacf 768\n" 618 " spka %[default_key]\n" 619 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 620 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 621 : [rc] "+&d" (rc), 622 [prev] "+&d" (prev), 623 [address] "+Q" (*(int *)address) 624 : [new] "d" ((unsigned int)new), 625 [key] "a" (key << 4), 626 [default_key] "J" (PAGE_DEFAULT_KEY) 627 : "memory", "cc"); 628 *(unsigned int *)uval = prev; 629 return rc; 630 } 631 case 8: { 632 unsigned long prev = old; 633 634 asm volatile( 635 " spka 0(%[key])\n" 636 " sacf 256\n" 637 "0: csg %[prev],%[new],%[address]\n" 638 "1: sacf 768\n" 639 " spka %[default_key]\n" 640 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev]) 641 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev]) 642 : [rc] "+&d" (rc), 643 [prev] "+&d" (prev), 644 [address] "+QS" (*(long *)address) 645 : [new] "d" ((unsigned long)new), 646 [key] "a" (key << 4), 647 [default_key] "J" (PAGE_DEFAULT_KEY) 648 : "memory", "cc"); 649 *(unsigned long *)uval = prev; 650 return rc; 651 } 652 case 16: { 653 __uint128_t prev = old; 654 655 asm volatile( 656 " spka 0(%[key])\n" 657 " sacf 256\n" 658 "0: cdsg %[prev],%[new],%[address]\n" 659 "1: sacf 768\n" 660 " spka %[default_key]\n" 661 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev]) 662 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev]) 663 : [rc] "+&d" (rc), 664 [prev] "+&d" (prev), 665 [address] "+QS" (*(__int128_t *)address) 666 : [new] "d" (new), 667 [key] "a" (key << 4), 668 [default_key] "J" (PAGE_DEFAULT_KEY) 669 : "memory", "cc"); 670 *(__uint128_t *)uval = prev; 671 return rc; 672 } 673 } 674 __cmpxchg_user_key_called_with_bad_pointer(); 675 return rc; 676 } 677 678 /** 679 * cmpxchg_user_key() - cmpxchg with user space target, honoring storage keys 680 * @ptr: User space address of value to compare to @old and exchange with 681 * @new. Must be aligned to sizeof(*@ptr). 682 * @uval: Address where the old value of *@ptr is written to. 683 * @old: Old value. Compared to the content pointed to by @ptr in order to 684 * determine if the exchange occurs. The old value read from *@ptr is 685 * written to *@uval. 686 * @new: New value to place at *@ptr. 687 * @key: Access key to use for checking storage key protection. 688 * 689 * Perform a cmpxchg on a user space target, honoring storage key protection. 690 * @key alone determines how key checking is performed, neither 691 * storage-protection-override nor fetch-protection-override apply. 692 * The caller must compare *@uval and @old to determine if values have been 693 * exchanged. In case of an exception *@uval is set to zero. 694 * 695 * Return: 0: cmpxchg executed 696 * -EFAULT: an exception happened when trying to access *@ptr 697 * -EAGAIN: maxed out number of retries (byte and short only) 698 */ 699 #define cmpxchg_user_key(ptr, uval, old, new, key) \ 700 ({ \ 701 __typeof__(ptr) __ptr = (ptr); \ 702 __typeof__(uval) __uval = (uval); \ 703 \ 704 BUILD_BUG_ON(sizeof(*(__ptr)) != sizeof(*(__uval))); \ 705 might_fault(); \ 706 __chk_user_ptr(__ptr); \ 707 __cmpxchg_user_key((unsigned long)(__ptr), (void *)(__uval), \ 708 (old), (new), (key), sizeof(*(__ptr))); \ 709 }) 710 711 #endif /* __S390_UACCESS_H */ 712