1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _TOOLS_LINUX_ASM_AARCH64_BARRIER_H 3 #define _TOOLS_LINUX_ASM_AARCH64_BARRIER_H 4 5 /* 6 * From tools/perf/perf-sys.h, last modified in: 7 * f428ebd184c82a7914b2aa7e9f868918aaf7ea78 perf tools: Fix AAAAARGH64 memory barriers 8 * 9 * XXX: arch/arm64/include/asm/barrier.h in the kernel sources use dsb, is this 10 * a case like for arm32 where we do things differently in userspace? 11 */ 12 13 #define mb() asm volatile("dmb ish" ::: "memory") 14 #define wmb() asm volatile("dmb ishst" ::: "memory") 15 #define rmb() asm volatile("dmb ishld" ::: "memory") 16 17 /* 18 * Kernel uses dmb variants on arm64 for smp_*() barriers. Pretty much the same 19 * implementation as above mb()/wmb()/rmb(), though for the latter kernel uses 20 * dsb. In any case, should above mb()/wmb()/rmb() change, make sure the below 21 * smp_*() don't. 22 */ 23 #define smp_mb() asm volatile("dmb ish" ::: "memory") 24 #define smp_wmb() asm volatile("dmb ishst" ::: "memory") 25 #define smp_rmb() asm volatile("dmb ishld" ::: "memory") 26 27 #define smp_store_release(p, v) \ 28 do { \ 29 union { typeof(*p) __val; char __c[1]; } __u = \ 30 { .__val = (v) }; \ 31 \ 32 switch (sizeof(*p)) { \ 33 case 1: \ 34 asm volatile ("stlrb %w1, %0" \ 35 : "=Q" (*p) \ 36 : "r" (*(__u8_alias_t *)__u.__c) \ 37 : "memory"); \ 38 break; \ 39 case 2: \ 40 asm volatile ("stlrh %w1, %0" \ 41 : "=Q" (*p) \ 42 : "r" (*(__u16_alias_t *)__u.__c) \ 43 : "memory"); \ 44 break; \ 45 case 4: \ 46 asm volatile ("stlr %w1, %0" \ 47 : "=Q" (*p) \ 48 : "r" (*(__u32_alias_t *)__u.__c) \ 49 : "memory"); \ 50 break; \ 51 case 8: \ 52 asm volatile ("stlr %1, %0" \ 53 : "=Q" (*p) \ 54 : "r" (*(__u64_alias_t *)__u.__c) \ 55 : "memory"); \ 56 break; \ 57 default: \ 58 /* Only to shut up gcc ... */ \ 59 mb(); \ 60 break; \ 61 } \ 62 } while (0) 63 64 #define smp_load_acquire(p) \ 65 ({ \ 66 union { typeof(*p) __val; char __c[1]; } __u = \ 67 { .__c = { 0 } }; \ 68 \ 69 switch (sizeof(*p)) { \ 70 case 1: \ 71 asm volatile ("ldarb %w0, %1" \ 72 : "=r" (*(__u8_alias_t *)__u.__c) \ 73 : "Q" (*p) : "memory"); \ 74 break; \ 75 case 2: \ 76 asm volatile ("ldarh %w0, %1" \ 77 : "=r" (*(__u16_alias_t *)__u.__c) \ 78 : "Q" (*p) : "memory"); \ 79 break; \ 80 case 4: \ 81 asm volatile ("ldar %w0, %1" \ 82 : "=r" (*(__u32_alias_t *)__u.__c) \ 83 : "Q" (*p) : "memory"); \ 84 break; \ 85 case 8: \ 86 asm volatile ("ldar %0, %1" \ 87 : "=r" (*(__u64_alias_t *)__u.__c) \ 88 : "Q" (*p) : "memory"); \ 89 break; \ 90 default: \ 91 /* Only to shut up gcc ... */ \ 92 mb(); \ 93 break; \ 94 } \ 95 __u.__val; \ 96 }) 97 98 #endif /* _TOOLS_LINUX_ASM_AARCH64_BARRIER_H */ 99