1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020 Google LLC. 4 */ 5 #ifndef __ASM_RWONCE_H 6 #define __ASM_RWONCE_H 7 8 #if defined(CONFIG_LTO) && !defined(__ASSEMBLY__) 9 10 #include <linux/compiler_types.h> 11 #include <asm/alternative-macros.h> 12 13 #ifndef BUILD_VDSO 14 15 #define __LOAD_RCPC(sfx, regs...) \ 16 ALTERNATIVE( \ 17 "ldar" #sfx "\t" #regs, \ 18 ".arch_extension rcpc\n" \ 19 "ldapr" #sfx "\t" #regs, \ 20 ARM64_HAS_LDAPR) 21 22 /* 23 * When building with LTO, there is an increased risk of the compiler 24 * converting an address dependency headed by a READ_ONCE() invocation 25 * into a control dependency and consequently allowing for harmful 26 * reordering by the CPU. 27 * 28 * Ensure that such transformations are harmless by overriding the generic 29 * READ_ONCE() definition with one that provides RCpc acquire semantics 30 * when building with LTO. 31 */ 32 #define __READ_ONCE(x) \ 33 ({ \ 34 typeof(&(x)) __x = &(x); \ 35 int atomic = 1; \ 36 union { __unqual_scalar_typeof(*__x) __val; char __c[1]; } __u; \ 37 switch (sizeof(x)) { \ 38 case 1: \ 39 asm volatile(__LOAD_RCPC(b, %w0, %1) \ 40 : "=r" (*(__u8 *)__u.__c) \ 41 : "Q" (*__x) : "memory"); \ 42 break; \ 43 case 2: \ 44 asm volatile(__LOAD_RCPC(h, %w0, %1) \ 45 : "=r" (*(__u16 *)__u.__c) \ 46 : "Q" (*__x) : "memory"); \ 47 break; \ 48 case 4: \ 49 asm volatile(__LOAD_RCPC(, %w0, %1) \ 50 : "=r" (*(__u32 *)__u.__c) \ 51 : "Q" (*__x) : "memory"); \ 52 break; \ 53 case 8: \ 54 asm volatile(__LOAD_RCPC(, %0, %1) \ 55 : "=r" (*(__u64 *)__u.__c) \ 56 : "Q" (*__x) : "memory"); \ 57 break; \ 58 default: \ 59 atomic = 0; \ 60 } \ 61 atomic ? (typeof(*__x))__u.__val : (*(volatile typeof(__x))__x);\ 62 }) 63 64 #endif /* !BUILD_VDSO */ 65 #endif /* CONFIG_LTO && !__ASSEMBLY__ */ 66 67 #include <asm-generic/rwonce.h> 68 69 #endif /* __ASM_RWONCE_H */ 70