1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Generic implementation of 64-bit atomics using spinlocks, 4 * useful on processors that don't have 64-bit atomic instructions. 5 * 6 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com> 7 */ 8 #include <linux/types.h> 9 #include <linux/cache.h> 10 #include <linux/spinlock.h> 11 #include <linux/init.h> 12 #include <linux/export.h> 13 #include <linux/atomic.h> 14 15 /* 16 * We use a hashed array of spinlocks to provide exclusive access 17 * to each atomic64_t variable. Since this is expected to used on 18 * systems with small numbers of CPUs (<= 4 or so), we use a 19 * relatively small array of 16 spinlocks to avoid wasting too much 20 * memory on the spinlock array. 21 */ 22 #define NR_LOCKS 16 23 24 /* 25 * Ensure each lock is in a separate cacheline. 26 */ 27 static union { 28 raw_spinlock_t lock; 29 char pad[L1_CACHE_BYTES]; 30 } atomic64_lock[NR_LOCKS] __cacheline_aligned_in_smp = { 31 [0 ... (NR_LOCKS - 1)] = { 32 .lock = __RAW_SPIN_LOCK_UNLOCKED(atomic64_lock.lock), 33 }, 34 }; 35 36 static inline raw_spinlock_t *lock_addr(const atomic64_t *v) 37 { 38 unsigned long addr = (unsigned long) v; 39 40 addr >>= L1_CACHE_SHIFT; 41 addr ^= (addr >> 8) ^ (addr >> 16); 42 return &atomic64_lock[addr & (NR_LOCKS - 1)].lock; 43 } 44 45 s64 generic_atomic64_read(const atomic64_t *v) 46 { 47 unsigned long flags; 48 raw_spinlock_t *lock = lock_addr(v); 49 s64 val; 50 51 raw_spin_lock_irqsave(lock, flags); 52 val = v->counter; 53 raw_spin_unlock_irqrestore(lock, flags); 54 return val; 55 } 56 EXPORT_SYMBOL(generic_atomic64_read); 57 58 void generic_atomic64_set(atomic64_t *v, s64 i) 59 { 60 unsigned long flags; 61 raw_spinlock_t *lock = lock_addr(v); 62 63 raw_spin_lock_irqsave(lock, flags); 64 v->counter = i; 65 raw_spin_unlock_irqrestore(lock, flags); 66 } 67 EXPORT_SYMBOL(generic_atomic64_set); 68 69 #define ATOMIC64_OP(op, c_op) \ 70 void generic_atomic64_##op(s64 a, atomic64_t *v) \ 71 { \ 72 unsigned long flags; \ 73 raw_spinlock_t *lock = lock_addr(v); \ 74 \ 75 raw_spin_lock_irqsave(lock, flags); \ 76 v->counter c_op a; \ 77 raw_spin_unlock_irqrestore(lock, flags); \ 78 } \ 79 EXPORT_SYMBOL(generic_atomic64_##op); 80 81 #define ATOMIC64_OP_RETURN(op, c_op) \ 82 s64 generic_atomic64_##op##_return(s64 a, atomic64_t *v) \ 83 { \ 84 unsigned long flags; \ 85 raw_spinlock_t *lock = lock_addr(v); \ 86 s64 val; \ 87 \ 88 raw_spin_lock_irqsave(lock, flags); \ 89 val = (v->counter c_op a); \ 90 raw_spin_unlock_irqrestore(lock, flags); \ 91 return val; \ 92 } \ 93 EXPORT_SYMBOL(generic_atomic64_##op##_return); 94 95 #define ATOMIC64_FETCH_OP(op, c_op) \ 96 s64 generic_atomic64_fetch_##op(s64 a, atomic64_t *v) \ 97 { \ 98 unsigned long flags; \ 99 raw_spinlock_t *lock = lock_addr(v); \ 100 s64 val; \ 101 \ 102 raw_spin_lock_irqsave(lock, flags); \ 103 val = v->counter; \ 104 v->counter c_op a; \ 105 raw_spin_unlock_irqrestore(lock, flags); \ 106 return val; \ 107 } \ 108 EXPORT_SYMBOL(generic_atomic64_fetch_##op); 109 110 #define ATOMIC64_OPS(op, c_op) \ 111 ATOMIC64_OP(op, c_op) \ 112 ATOMIC64_OP_RETURN(op, c_op) \ 113 ATOMIC64_FETCH_OP(op, c_op) 114 115 ATOMIC64_OPS(add, +=) 116 ATOMIC64_OPS(sub, -=) 117 118 #undef ATOMIC64_OPS 119 #define ATOMIC64_OPS(op, c_op) \ 120 ATOMIC64_OP(op, c_op) \ 121 ATOMIC64_FETCH_OP(op, c_op) 122 123 ATOMIC64_OPS(and, &=) 124 ATOMIC64_OPS(or, |=) 125 ATOMIC64_OPS(xor, ^=) 126 127 #undef ATOMIC64_OPS 128 #undef ATOMIC64_FETCH_OP 129 #undef ATOMIC64_OP 130 131 s64 generic_atomic64_dec_if_positive(atomic64_t *v) 132 { 133 unsigned long flags; 134 raw_spinlock_t *lock = lock_addr(v); 135 s64 val; 136 137 raw_spin_lock_irqsave(lock, flags); 138 val = v->counter - 1; 139 if (val >= 0) 140 v->counter = val; 141 raw_spin_unlock_irqrestore(lock, flags); 142 return val; 143 } 144 EXPORT_SYMBOL(generic_atomic64_dec_if_positive); 145 146 s64 generic_atomic64_cmpxchg(atomic64_t *v, s64 o, s64 n) 147 { 148 unsigned long flags; 149 raw_spinlock_t *lock = lock_addr(v); 150 s64 val; 151 152 raw_spin_lock_irqsave(lock, flags); 153 val = v->counter; 154 if (val == o) 155 v->counter = n; 156 raw_spin_unlock_irqrestore(lock, flags); 157 return val; 158 } 159 EXPORT_SYMBOL(generic_atomic64_cmpxchg); 160 161 s64 generic_atomic64_xchg(atomic64_t *v, s64 new) 162 { 163 unsigned long flags; 164 raw_spinlock_t *lock = lock_addr(v); 165 s64 val; 166 167 raw_spin_lock_irqsave(lock, flags); 168 val = v->counter; 169 v->counter = new; 170 raw_spin_unlock_irqrestore(lock, flags); 171 return val; 172 } 173 EXPORT_SYMBOL(generic_atomic64_xchg); 174 175 s64 generic_atomic64_fetch_add_unless(atomic64_t *v, s64 a, s64 u) 176 { 177 unsigned long flags; 178 raw_spinlock_t *lock = lock_addr(v); 179 s64 val; 180 181 raw_spin_lock_irqsave(lock, flags); 182 val = v->counter; 183 if (val != u) 184 v->counter += a; 185 raw_spin_unlock_irqrestore(lock, flags); 186 187 return val; 188 } 189 EXPORT_SYMBOL(generic_atomic64_fetch_add_unless); 190