1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 3 /* 4 * ARCv2 supports 64-bit exclusive load (LLOCKD) / store (SCONDD) 5 * - The address HAS to be 64-bit aligned 6 */ 7 8 #ifndef _ASM_ARC_ATOMIC64_ARCV2_H 9 #define _ASM_ARC_ATOMIC64_ARCV2_H 10 11 typedef struct { 12 s64 __aligned(8) counter; 13 } atomic64_t; 14 15 #define ATOMIC64_INIT(a) { (a) } 16 17 static inline s64 arch_atomic64_read(const atomic64_t *v) 18 { 19 s64 val; 20 21 __asm__ __volatile__( 22 " ldd %0, [%1] \n" 23 : "=r"(val) 24 : "r"(&v->counter)); 25 26 return val; 27 } 28 29 static inline void arch_atomic64_set(atomic64_t *v, s64 a) 30 { 31 /* 32 * This could have been a simple assignment in "C" but would need 33 * explicit volatile. Otherwise gcc optimizers could elide the store 34 * which borked atomic64 self-test 35 * In the inline asm version, memory clobber needed for exact same 36 * reason, to tell gcc about the store. 37 * 38 * This however is not needed for sibling atomic64_add() etc since both 39 * load/store are explicitly done in inline asm. As long as API is used 40 * for each access, gcc has no way to optimize away any load/store 41 */ 42 __asm__ __volatile__( 43 " std %0, [%1] \n" 44 : 45 : "r"(a), "r"(&v->counter) 46 : "memory"); 47 } 48 49 #define ATOMIC64_OP(op, op1, op2) \ 50 static inline void arch_atomic64_##op(s64 a, atomic64_t *v) \ 51 { \ 52 s64 val; \ 53 \ 54 __asm__ __volatile__( \ 55 "1: \n" \ 56 " llockd %0, [%1] \n" \ 57 " " #op1 " %L0, %L0, %L2 \n" \ 58 " " #op2 " %H0, %H0, %H2 \n" \ 59 " scondd %0, [%1] \n" \ 60 " bnz 1b \n" \ 61 : "=&r"(val) \ 62 : "r"(&v->counter), "ir"(a) \ 63 : "cc"); \ 64 } \ 65 66 #define ATOMIC64_OP_RETURN(op, op1, op2) \ 67 static inline s64 arch_atomic64_##op##_return_relaxed(s64 a, atomic64_t *v) \ 68 { \ 69 s64 val; \ 70 \ 71 __asm__ __volatile__( \ 72 "1: \n" \ 73 " llockd %0, [%1] \n" \ 74 " " #op1 " %L0, %L0, %L2 \n" \ 75 " " #op2 " %H0, %H0, %H2 \n" \ 76 " scondd %0, [%1] \n" \ 77 " bnz 1b \n" \ 78 : [val] "=&r"(val) \ 79 : "r"(&v->counter), "ir"(a) \ 80 : "cc"); /* memory clobber comes from smp_mb() */ \ 81 \ 82 return val; \ 83 } 84 85 #define arch_atomic64_add_return_relaxed arch_atomic64_add_return_relaxed 86 #define arch_atomic64_sub_return_relaxed arch_atomic64_sub_return_relaxed 87 88 #define ATOMIC64_FETCH_OP(op, op1, op2) \ 89 static inline s64 arch_atomic64_fetch_##op##_relaxed(s64 a, atomic64_t *v) \ 90 { \ 91 s64 val, orig; \ 92 \ 93 __asm__ __volatile__( \ 94 "1: \n" \ 95 " llockd %0, [%2] \n" \ 96 " " #op1 " %L1, %L0, %L3 \n" \ 97 " " #op2 " %H1, %H0, %H3 \n" \ 98 " scondd %1, [%2] \n" \ 99 " bnz 1b \n" \ 100 : "=&r"(orig), "=&r"(val) \ 101 : "r"(&v->counter), "ir"(a) \ 102 : "cc"); /* memory clobber comes from smp_mb() */ \ 103 \ 104 return orig; \ 105 } 106 107 #define arch_atomic64_fetch_add_relaxed arch_atomic64_fetch_add_relaxed 108 #define arch_atomic64_fetch_sub_relaxed arch_atomic64_fetch_sub_relaxed 109 110 #define arch_atomic64_fetch_and_relaxed arch_atomic64_fetch_and_relaxed 111 #define arch_atomic64_fetch_andnot_relaxed arch_atomic64_fetch_andnot_relaxed 112 #define arch_atomic64_fetch_or_relaxed arch_atomic64_fetch_or_relaxed 113 #define arch_atomic64_fetch_xor_relaxed arch_atomic64_fetch_xor_relaxed 114 115 #define ATOMIC64_OPS(op, op1, op2) \ 116 ATOMIC64_OP(op, op1, op2) \ 117 ATOMIC64_OP_RETURN(op, op1, op2) \ 118 ATOMIC64_FETCH_OP(op, op1, op2) 119 120 ATOMIC64_OPS(add, add.f, adc) 121 ATOMIC64_OPS(sub, sub.f, sbc) 122 123 #undef ATOMIC64_OPS 124 #define ATOMIC64_OPS(op, op1, op2) \ 125 ATOMIC64_OP(op, op1, op2) \ 126 ATOMIC64_FETCH_OP(op, op1, op2) 127 128 ATOMIC64_OPS(and, and, and) 129 ATOMIC64_OPS(andnot, bic, bic) 130 ATOMIC64_OPS(or, or, or) 131 ATOMIC64_OPS(xor, xor, xor) 132 133 #define arch_atomic64_andnot arch_atomic64_andnot 134 135 #undef ATOMIC64_OPS 136 #undef ATOMIC64_FETCH_OP 137 #undef ATOMIC64_OP_RETURN 138 #undef ATOMIC64_OP 139 140 static inline s64 141 arch_atomic64_cmpxchg(atomic64_t *ptr, s64 expected, s64 new) 142 { 143 s64 prev; 144 145 smp_mb(); 146 147 __asm__ __volatile__( 148 "1: llockd %0, [%1] \n" 149 " brne %L0, %L2, 2f \n" 150 " brne %H0, %H2, 2f \n" 151 " scondd %3, [%1] \n" 152 " bnz 1b \n" 153 "2: \n" 154 : "=&r"(prev) 155 : "r"(ptr), "ir"(expected), "r"(new) 156 : "cc"); /* memory clobber comes from smp_mb() */ 157 158 smp_mb(); 159 160 return prev; 161 } 162 163 static inline s64 arch_atomic64_xchg(atomic64_t *ptr, s64 new) 164 { 165 s64 prev; 166 167 smp_mb(); 168 169 __asm__ __volatile__( 170 "1: llockd %0, [%1] \n" 171 " scondd %2, [%1] \n" 172 " bnz 1b \n" 173 "2: \n" 174 : "=&r"(prev) 175 : "r"(ptr), "r"(new) 176 : "cc"); /* memory clobber comes from smp_mb() */ 177 178 smp_mb(); 179 180 return prev; 181 } 182 183 /** 184 * arch_atomic64_dec_if_positive - decrement by 1 if old value positive 185 * @v: pointer of type atomic64_t 186 * 187 * The function returns the old value of *v minus 1, even if 188 * the atomic variable, v, was not decremented. 189 */ 190 191 static inline s64 arch_atomic64_dec_if_positive(atomic64_t *v) 192 { 193 s64 val; 194 195 smp_mb(); 196 197 __asm__ __volatile__( 198 "1: llockd %0, [%1] \n" 199 " sub.f %L0, %L0, 1 # w0 - 1, set C on borrow\n" 200 " sub.c %H0, %H0, 1 # if C set, w1 - 1\n" 201 " brlt %H0, 0, 2f \n" 202 " scondd %0, [%1] \n" 203 " bnz 1b \n" 204 "2: \n" 205 : "=&r"(val) 206 : "r"(&v->counter) 207 : "cc"); /* memory clobber comes from smp_mb() */ 208 209 smp_mb(); 210 211 return val; 212 } 213 #define arch_atomic64_dec_if_positive arch_atomic64_dec_if_positive 214 215 /** 216 * arch_atomic64_fetch_add_unless - add unless the number is a given value 217 * @v: pointer of type atomic64_t 218 * @a: the amount to add to v... 219 * @u: ...unless v is equal to u. 220 * 221 * Atomically adds @a to @v, if it was not @u. 222 * Returns the old value of @v 223 */ 224 static inline s64 arch_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 225 { 226 s64 old, temp; 227 228 smp_mb(); 229 230 __asm__ __volatile__( 231 "1: llockd %0, [%2] \n" 232 " brne %L0, %L4, 2f # continue to add since v != u \n" 233 " breq.d %H0, %H4, 3f # return since v == u \n" 234 "2: \n" 235 " add.f %L1, %L0, %L3 \n" 236 " adc %H1, %H0, %H3 \n" 237 " scondd %1, [%2] \n" 238 " bnz 1b \n" 239 "3: \n" 240 : "=&r"(old), "=&r" (temp) 241 : "r"(&v->counter), "r"(a), "r"(u) 242 : "cc"); /* memory clobber comes from smp_mb() */ 243 244 smp_mb(); 245 246 return old; 247 } 248 #define arch_atomic64_fetch_add_unless arch_atomic64_fetch_add_unless 249 250 #endif 251