1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Based on arch/arm/include/asm/barrier.h 4 * 5 * Copyright (C) 2012 ARM Ltd. 6 */ 7 #ifndef __ASM_BARRIER_H 8 #define __ASM_BARRIER_H 9 10 #ifndef __ASSEMBLY__ 11 12 #include <linux/kasan-checks.h> 13 14 #define __nops(n) ".rept " #n "\nnop\n.endr\n" 15 #define nops(n) asm volatile(__nops(n)) 16 17 #define sev() asm volatile("sev" : : : "memory") 18 #define wfe() asm volatile("wfe" : : : "memory") 19 #define wfi() asm volatile("wfi" : : : "memory") 20 21 #define isb() asm volatile("isb" : : : "memory") 22 #define dmb(opt) asm volatile("dmb " #opt : : : "memory") 23 #define dsb(opt) asm volatile("dsb " #opt : : : "memory") 24 25 #define psb_csync() asm volatile("hint #17" : : : "memory") 26 #define csdb() asm volatile("hint #20" : : : "memory") 27 28 #ifdef CONFIG_ARM64_PSEUDO_NMI 29 #define pmr_sync() \ 30 do { \ 31 extern struct static_key_false gic_pmr_sync; \ 32 \ 33 if (static_branch_unlikely(&gic_pmr_sync)) \ 34 dsb(sy); \ 35 } while(0) 36 #else 37 #define pmr_sync() do {} while (0) 38 #endif 39 40 #define mb() dsb(sy) 41 #define rmb() dsb(ld) 42 #define wmb() dsb(st) 43 44 #define dma_mb() dmb(osh) 45 #define dma_rmb() dmb(oshld) 46 #define dma_wmb() dmb(oshst) 47 48 /* 49 * Generate a mask for array_index__nospec() that is ~0UL when 0 <= idx < sz 50 * and 0 otherwise. 51 */ 52 #define array_index_mask_nospec array_index_mask_nospec 53 static inline unsigned long array_index_mask_nospec(unsigned long idx, 54 unsigned long sz) 55 { 56 unsigned long mask; 57 58 asm volatile( 59 " cmp %1, %2\n" 60 " sbc %0, xzr, xzr\n" 61 : "=r" (mask) 62 : "r" (idx), "Ir" (sz) 63 : "cc"); 64 65 csdb(); 66 return mask; 67 } 68 69 /* 70 * Ensure that reads of the counter are treated the same as memory reads 71 * for the purposes of ordering by subsequent memory barriers. 72 * 73 * This insanity brought to you by speculative system register reads, 74 * out-of-order memory accesses, sequence locks and Thomas Gleixner. 75 * 76 * http://lists.infradead.org/pipermail/linux-arm-kernel/2019-February/631195.html 77 */ 78 #define arch_counter_enforce_ordering(val) do { \ 79 u64 tmp, _val = (val); \ 80 \ 81 asm volatile( \ 82 " eor %0, %1, %1\n" \ 83 " add %0, sp, %0\n" \ 84 " ldr xzr, [%0]" \ 85 : "=r" (tmp) : "r" (_val)); \ 86 } while (0) 87 88 #define __smp_mb() dmb(ish) 89 #define __smp_rmb() dmb(ishld) 90 #define __smp_wmb() dmb(ishst) 91 92 #define __smp_store_release(p, v) \ 93 do { \ 94 typeof(p) __p = (p); \ 95 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u = \ 96 { .__val = (__force __unqual_scalar_typeof(*p)) (v) }; \ 97 compiletime_assert_atomic_type(*p); \ 98 kasan_check_write(__p, sizeof(*p)); \ 99 switch (sizeof(*p)) { \ 100 case 1: \ 101 asm volatile ("stlrb %w1, %0" \ 102 : "=Q" (*__p) \ 103 : "r" (*(__u8 *)__u.__c) \ 104 : "memory"); \ 105 break; \ 106 case 2: \ 107 asm volatile ("stlrh %w1, %0" \ 108 : "=Q" (*__p) \ 109 : "r" (*(__u16 *)__u.__c) \ 110 : "memory"); \ 111 break; \ 112 case 4: \ 113 asm volatile ("stlr %w1, %0" \ 114 : "=Q" (*__p) \ 115 : "r" (*(__u32 *)__u.__c) \ 116 : "memory"); \ 117 break; \ 118 case 8: \ 119 asm volatile ("stlr %1, %0" \ 120 : "=Q" (*__p) \ 121 : "r" (*(__u64 *)__u.__c) \ 122 : "memory"); \ 123 break; \ 124 } \ 125 } while (0) 126 127 #define __smp_load_acquire(p) \ 128 ({ \ 129 union { __unqual_scalar_typeof(*p) __val; char __c[1]; } __u; \ 130 typeof(p) __p = (p); \ 131 compiletime_assert_atomic_type(*p); \ 132 kasan_check_read(__p, sizeof(*p)); \ 133 switch (sizeof(*p)) { \ 134 case 1: \ 135 asm volatile ("ldarb %w0, %1" \ 136 : "=r" (*(__u8 *)__u.__c) \ 137 : "Q" (*__p) : "memory"); \ 138 break; \ 139 case 2: \ 140 asm volatile ("ldarh %w0, %1" \ 141 : "=r" (*(__u16 *)__u.__c) \ 142 : "Q" (*__p) : "memory"); \ 143 break; \ 144 case 4: \ 145 asm volatile ("ldar %w0, %1" \ 146 : "=r" (*(__u32 *)__u.__c) \ 147 : "Q" (*__p) : "memory"); \ 148 break; \ 149 case 8: \ 150 asm volatile ("ldar %0, %1" \ 151 : "=r" (*(__u64 *)__u.__c) \ 152 : "Q" (*__p) : "memory"); \ 153 break; \ 154 } \ 155 (typeof(*p))__u.__val; \ 156 }) 157 158 #define smp_cond_load_relaxed(ptr, cond_expr) \ 159 ({ \ 160 typeof(ptr) __PTR = (ptr); \ 161 __unqual_scalar_typeof(*ptr) VAL; \ 162 for (;;) { \ 163 VAL = READ_ONCE(*__PTR); \ 164 if (cond_expr) \ 165 break; \ 166 __cmpwait_relaxed(__PTR, VAL); \ 167 } \ 168 (typeof(*ptr))VAL; \ 169 }) 170 171 #define smp_cond_load_acquire(ptr, cond_expr) \ 172 ({ \ 173 typeof(ptr) __PTR = (ptr); \ 174 __unqual_scalar_typeof(*ptr) VAL; \ 175 for (;;) { \ 176 VAL = smp_load_acquire(__PTR); \ 177 if (cond_expr) \ 178 break; \ 179 __cmpwait_relaxed(__PTR, VAL); \ 180 } \ 181 (typeof(*ptr))VAL; \ 182 }) 183 184 #include <asm-generic/barrier.h> 185 186 #endif /* __ASSEMBLY__ */ 187 188 #endif /* __ASM_BARRIER_H */ 189