1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 /* 4 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 5 * - The address HAS to be 64-bit aligned 6 */ 7 8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H 9 #define _ASM_ARC_ATOMIC64_ARCV2_H 10 11 typedef struct { 12 s64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(a) { (a) } 16 17 static inline s64 arch_atomic64_read(const atomic64_t *v) 18 { 19 s64 val; 20 21 __asm__ __volatile__( 22 " ldd %0, [%1] \n" 23 : "=r"(val) 24 : "r"(&v->counter)); 25 26 return val; 27 } 28 29 static inline void arch_atomic64_set(atomic64_t *v, s64 a) 30 { 31 /* 32 * This could have been a simple assignment in "C" but would need 33 * explicit volatile. Otherwise gcc optimizers could elide the store 34 * which borked atomic64 self-test 35 * In the inline asm version, memory clobber needed for exact same 36 * reason, to tell gcc about the store. 37 * 38 * This however is not needed for sibling atomic64_add() etc since both 39 * load/store are explicitly done in inline asm. As long as API is used 40 * for each access, gcc has no way to optimize away any load/store 41 */ 42 __asm__ __volatile__( 43 " std %0, [%1] \n" 44 : 45 : "r"(a), "r"(&v->counter) 46 : "memory"); 47 } 48 49 #define ATOMIC64_OP(op, op1, op2) \ 50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ 51 { \ 52 s64 val; \ 53 \ 54 __asm__ __volatile__( \ 55 "1: \n" \ 56 " llockd %0, [%1] \n" \ 57 " " #op1 " %L0, %L0, %L2 \n" \ 58 " " #op2 " %H0, %H0, %H2 \n" \ 59 " scondd %0, [%1] \n" \ 60 " bnz 1b \n" \ 61 : "=&r"(val) \ 62 : "r"(&v->counter), "ir"(a) \ 63 : "cc", "memory"); \ 64 } \ 65 66 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 67 static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 68 { \ 69 s64 val; \ 70 \ 71 __asm__ __volatile__( \ 72 "1: \n" \ 73 " llockd %0, [%1] \n" \ 74 " " #op1 " %L0, %L0, %L2 \n" \ 75 " " #op2 " %H0, %H0, %H2 \n" \ 76 " scondd %0, [%1] \n" \ 77 " bnz 1b \n" \ 78 : [val] "=&r"(val) \ 79 : "r"(&v->counter), "ir"(a) \ 80 : "cc", "memory"); \ 81 \ 82 return val; \ 83 } 84 85 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 86 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 87 88 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 89 static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 90 { \ 91 s64 val, orig; \ 92 \ 93 __asm__ __volatile__( \ 94 "1: \n" \ 95 " llockd %0, [%2] \n" \ 96 " " #op1 " %L1, %L0, %L3 \n" \ 97 " " #op2 " %H1, %H0, %H3 \n" \ 98 " scondd %1, [%2] \n" \ 99 " bnz 1b \n" \ 100 : "=&r"(orig), "=&r"(val) \ 101 : "r"(&v->counter), "ir"(a) \ 102 : "cc", "memory"); \ 103 \ 104 return orig; \ 105 } 106 107 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 108 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 109 110 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 111 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 112 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 113 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 114 115 #define ATOMIC64_OPS(op, op1, op2) \ 116 ATOMIC64_OP(op, op1, op2) \ 117 ATOMIC64_OP_RETURN(op, op1, op2) \ 118 ATOMIC64_FETCH_OP(op, op1, op2) 119 120 ATOMIC64_OPS(add, add.f, adc) 121 ATOMIC64_OPS(sub, sub.f, sbc) 122 123 #undef ATOMIC64_OPS 124 #define ATOMIC64_OPS(op, op1, op2) \ 125 ATOMIC64_OP(op, op1, op2) \ 126 ATOMIC64_FETCH_OP(op, op1, op2) 127 128 ATOMIC64_OPS(and, and, and) 129 ATOMIC64_OPS(andnot, bic, bic) 130 ATOMIC64_OPS(or, or, or) 131 ATOMIC64_OPS(xor, xor, xor) 132 133 #define arch_atomic64_andnot arch_atomic64_andnot 134 135 #undef ATOMIC64_OPS 136 #undef ATOMIC64_FETCH_OP 137 #undef ATOMIC64_OP_RETURN 138 #undef ATOMIC64_OP 139 140 static inline u64 __arch_cmpxchg64_relaxed(volatile void *ptr, u64 old, u64 new) 141 { 142 u64 prev; 143 144 __asm__ __volatile__( 145 "1: llockd %0, [%1] \n" 146 " brne %L0, %L2, 2f \n" 147 " brne %H0, %H2, 2f \n" 148 " scondd %3, [%1] \n" 149 " bnz 1b \n" 150 "2: \n" 151 : "=&r"(prev) 152 : "r"(ptr), "ir"(old), "r"(new) 153 : "memory", "cc"); 154 155 return prev; 156 } 157 #define arch_cmpxchg64_relaxed __arch_cmpxchg64_relaxed 158 159 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) 160 { 161 s64 prev; 162 163 smp_mb(); 164 165 __asm__ __volatile__( 166 "1: llockd %0, [%1] \n" 167 " scondd %2, [%1] \n" 168 " bnz 1b \n" 169 "2: \n" 170 : "=&r"(prev) 171 : "r"(ptr), "r"(new) 172 : "cc"); /* memory clobber comes from smp_mb() */ 173 174 smp_mb(); 175 176 return prev; 177 } 178 #define arch_atomic64_xchg arch_atomic64_xchg 179 180 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 181 { 182 s64 val; 183 184 smp_mb(); 185 186 __asm__ __volatile__( 187 "1: llockd %0, [%1] \n" 188 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 189 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 190 " brlt %H0, 0, 2f \n" 191 " scondd %0, [%1] \n" 192 " bnz 1b \n" 193 "2: \n" 194 : "=&r"(val) 195 : "r"(&v->counter) 196 : "cc"); /* memory clobber comes from smp_mb() */ 197 198 smp_mb(); 199 200 return val; 201 } 202 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 203 204 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 205 { 206 s64 old, temp; 207 208 smp_mb(); 209 210 __asm__ __volatile__( 211 "1: llockd %0, [%2] \n" 212 " brne %L0, %L4, 2f # continue to add since v != u \n" 213 " breq.d %H0, %H4, 3f # return since v == u \n" 214 "2: \n" 215 " add.f %L1, %L0, %L3 \n" 216 " adc %H1, %H0, %H3 \n" 217 " scondd %1, [%2] \n" 218 " bnz 1b \n" 219 "3: \n" 220 : "=&r"(old), "=&r" (temp) 221 : "r"(&v->counter), "r"(a), "r"(u) 222 : "cc"); /* memory clobber comes from smp_mb() */ 223 224 smp_mb(); 225 226 return old; 227 } 228 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 229 230 #endif 231