1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020 Google LLC. 4 */ 5 #ifndef __ASM_RWONCE_H 6 #define __ASM_RWONCE_H 7 8 #if defined(CONFIG_LTO) && !defined(__ASSEMBLER__) 9 10 #include <linux/compiler_types.h> 11 #include <asm/alternative-macros.h> 12 13 #ifndef BUILD_VDSO 14 15 #define __LOAD_RCPC(sfx, regs...) \ 16 ALTERNATIVE( \ 17 "ldar" #sfx "\t" #regs, \ 18 ".arch_extension rcpc\n" \ 19 "ldapr" #sfx "\t" #regs, \ 20 ARM64_HAS_LDAPR) 21 22 /* 23 * Replace this with typeof_unqual() when minimum compiler versions are 24 * increased to GCC 14 and Clang 19. For the time being, we need this 25 * workaround, which relies on function return values dropping qualifiers. 26 */ 27 #define __rwonce_typeof_unqual(x) typeof(({ \ 28 __diag_push() \ 29 __diag_ignore_all("-Wignored-qualifiers", "") \ 30 ((typeof(x)(*)(void))0)(); \ 31 __diag_pop() })) 32 33 /* 34 * When building with LTO, there is an increased risk of the compiler 35 * converting an address dependency headed by a READ_ONCE() invocation 36 * into a control dependency and consequently allowing for harmful 37 * reordering by the CPU. 38 * 39 * Ensure that such transformations are harmless by overriding the generic 40 * READ_ONCE() definition with one that provides RCpc acquire semantics 41 * when building with LTO. 42 */ 43 #define __READ_ONCE(x) \ 44 ({ \ 45 auto __x = &(x); \ 46 auto __ret = (__rwonce_typeof_unqual(*__x) *)__x; \ 47 /* Hides alias reassignment from Clang's -Wthread-safety. */ \ 48 auto __retp = &__ret; \ 49 union { typeof(*__ret) __val; char __c[1]; } __u; \ 50 *__retp = &__u.__val; \ 51 switch (sizeof(x)) { \ 52 case 1: \ 53 asm volatile(__LOAD_RCPC(b, %w0, %1) \ 54 : "=r" (*(__u8 *)__u.__c) \ 55 : "Q" (*__x) : "memory"); \ 56 break; \ 57 case 2: \ 58 asm volatile(__LOAD_RCPC(h, %w0, %1) \ 59 : "=r" (*(__u16 *)__u.__c) \ 60 : "Q" (*__x) : "memory"); \ 61 break; \ 62 case 4: \ 63 asm volatile(__LOAD_RCPC(, %w0, %1) \ 64 : "=r" (*(__u32 *)__u.__c) \ 65 : "Q" (*__x) : "memory"); \ 66 break; \ 67 case 8: \ 68 asm volatile(__LOAD_RCPC(, %0, %1) \ 69 : "=r" (*(__u64 *)__u.__c) \ 70 : "Q" (*__x) : "memory"); \ 71 break; \ 72 default: \ 73 __u.__val = *(volatile typeof(*__x) *)__x; \ 74 } \ 75 *__ret; \ 76 }) 77 78 #endif /* !BUILD_VDSO */ 79 #endif /* CONFIG_LTO && !__ASSEMBLER__ */ 80 81 #include <asm-generic/rwonce.h> 82 83 #endif /* __ASM_RWONCE_H */ 84