1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 /* 4 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 5 * - The address HAS to be 64-bit aligned 6 */ 7 8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H 9 #define _ASM_ARC_ATOMIC64_ARCV2_H 10 11 typedef struct { 12 s64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(a) { (a) } 16 17 static inline s64 arch_atomic64_read(const atomic64_t *v) 18 { 19 s64 val; 20 21 __asm__ __volatile__( 22 " ldd %0, [%1] \n" 23 : "=r"(val) 24 : "r"(&v->counter)); 25 26 return val; 27 } 28 29 static inline void arch_atomic64_set(atomic64_t *v, s64 a) 30 { 31 /* 32 * This could have been a simple assignment in "C" but would need 33 * explicit volatile. Otherwise gcc optimizers could elide the store 34 * which borked atomic64 self-test 35 * In the inline asm version, memory clobber needed for exact same 36 * reason, to tell gcc about the store. 37 * 38 * This however is not needed for sibling atomic64_add() etc since both 39 * load/store are explicitly done in inline asm. As long as API is used 40 * for each access, gcc has no way to optimize away any load/store 41 */ 42 __asm__ __volatile__( 43 " std %0, [%1] \n" 44 : 45 : "r"(a), "r"(&v->counter) 46 : "memory"); 47 } 48 49 #define ATOMIC64_OP(op, op1, op2) \ 50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ 51 { \ 52 s64 val; \ 53 \ 54 __asm__ __volatile__( \ 55 "1: \n" \ 56 " llockd %0, [%1] \n" \ 57 " " #op1 " %L0, %L0, %L2 \n" \ 58 " " #op2 " %H0, %H0, %H2 \n" \ 59 " scondd %0, [%1] \n" \ 60 " bnz 1b \n" \ 61 : "=&r"(val) \ 62 : "r"(&v->counter), "ir"(a) \ 63 : "cc", "memory"); \ 64 } \ 65 66 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 67 static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 68 { \ 69 s64 val; \ 70 \ 71 __asm__ __volatile__( \ 72 "1: \n" \ 73 " llockd %0, [%1] \n" \ 74 " " #op1 " %L0, %L0, %L2 \n" \ 75 " " #op2 " %H0, %H0, %H2 \n" \ 76 " scondd %0, [%1] \n" \ 77 " bnz 1b \n" \ 78 : [val] "=&r"(val) \ 79 : "r"(&v->counter), "ir"(a) \ 80 : "cc", "memory"); \ 81 \ 82 return val; \ 83 } 84 85 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 86 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 87 88 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 89 static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 90 { \ 91 s64 val, orig; \ 92 \ 93 __asm__ __volatile__( \ 94 "1: \n" \ 95 " llockd %0, [%2] \n" \ 96 " " #op1 " %L1, %L0, %L3 \n" \ 97 " " #op2 " %H1, %H0, %H3 \n" \ 98 " scondd %1, [%2] \n" \ 99 " bnz 1b \n" \ 100 : "=&r"(orig), "=&r"(val) \ 101 : "r"(&v->counter), "ir"(a) \ 102 : "cc", "memory"); \ 103 \ 104 return orig; \ 105 } 106 107 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 108 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 109 110 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 111 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 112 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 113 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 114 115 #define ATOMIC64_OPS(op, op1, op2) \ 116 ATOMIC64_OP(op, op1, op2) \ 117 ATOMIC64_OP_RETURN(op, op1, op2) \ 118 ATOMIC64_FETCH_OP(op, op1, op2) 119 120 ATOMIC64_OPS(add, add.f, adc) 121 ATOMIC64_OPS(sub, sub.f, sbc) 122 123 #undef ATOMIC64_OPS 124 #define ATOMIC64_OPS(op, op1, op2) \ 125 ATOMIC64_OP(op, op1, op2) \ 126 ATOMIC64_FETCH_OP(op, op1, op2) 127 128 ATOMIC64_OPS(and, and, and) 129 ATOMIC64_OPS(andnot, bic, bic) 130 ATOMIC64_OPS(or, or, or) 131 ATOMIC64_OPS(xor, xor, xor) 132 133 #define arch_atomic64_andnot arch_atomic64_andnot 134 135 #undef ATOMIC64_OPS 136 #undef ATOMIC64_FETCH_OP 137 #undef ATOMIC64_OP_RETURN 138 #undef ATOMIC64_OP 139 140 static inline u64 __arch_cmpxchg64_relaxed(volatile void *ptr, u64 old, u64 new) 141 { 142 u64 prev; 143 144 __asm__ __volatile__( 145 "1: llockd %0, [%1] \n" 146 " brne %L0, %L2, 2f \n" 147 " brne %H0, %H2, 2f \n" 148 " scondd %3, [%1] \n" 149 " bnz 1b \n" 150 "2: \n" 151 : "=&r"(prev) 152 : "r"(ptr), "ir"(old), "r"(new) 153 : "memory", "cc"); 154 155 return prev; 156 } 157 #define arch_cmpxchg64_relaxed __arch_cmpxchg64_relaxed 158 159 static inline s64 arch_atomic64_cmpxchg_relaxed(atomic64_t *v, s64 old, s64 new) 160 { 161 return arch_cmpxchg64_relaxed(&v->counter, old, new); 162 } 163 #define arch_atomic64_cmpxchg_relaxed arch_atomic64_cmpxchg_relaxed 164 165 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) 166 { 167 s64 prev; 168 169 smp_mb(); 170 171 __asm__ __volatile__( 172 "1: llockd %0, [%1] \n" 173 " scondd %2, [%1] \n" 174 " bnz 1b \n" 175 "2: \n" 176 : "=&r"(prev) 177 : "r"(ptr), "r"(new) 178 : "cc"); /* memory clobber comes from smp_mb() */ 179 180 smp_mb(); 181 182 return prev; 183 } 184 #define arch_atomic64_xchg arch_atomic64_xchg 185 186 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 187 { 188 s64 val; 189 190 smp_mb(); 191 192 __asm__ __volatile__( 193 "1: llockd %0, [%1] \n" 194 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 195 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 196 " brlt %H0, 0, 2f \n" 197 " scondd %0, [%1] \n" 198 " bnz 1b \n" 199 "2: \n" 200 : "=&r"(val) 201 : "r"(&v->counter) 202 : "cc"); /* memory clobber comes from smp_mb() */ 203 204 smp_mb(); 205 206 return val; 207 } 208 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 209 210 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 211 { 212 s64 old, temp; 213 214 smp_mb(); 215 216 __asm__ __volatile__( 217 "1: llockd %0, [%2] \n" 218 " brne %L0, %L4, 2f # continue to add since v != u \n" 219 " breq.d %H0, %H4, 3f # return since v == u \n" 220 "2: \n" 221 " add.f %L1, %L0, %L3 \n" 222 " adc %H1, %H0, %H3 \n" 223 " scondd %1, [%2] \n" 224 " bnz 1b \n" 225 "3: \n" 226 : "=&r"(old), "=&r" (temp) 227 : "r"(&v->counter), "r"(a), "r"(u) 228 : "cc"); /* memory clobber comes from smp_mb() */ 229 230 smp_mb(); 231 232 return old; 233 } 234 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 235 236 #endif 237