1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 64-bit atomic xchg() and cmpxchg() definitions. 3 * 4 * Copyright (C) 1996, 1997, 2000 David S. Miller (davem@redhat.com) 5 */ 6 7 #ifndef __ARCH_SPARC64_CMPXCHG__ 8 #define __ARCH_SPARC64_CMPXCHG__ 9 10 static inline unsigned long 11 __cmpxchg_u32(volatile int *m, int old, int new) 12 { 13 __asm__ __volatile__("cas [%2], %3, %0" 14 : "=&r" (new) 15 : "0" (new), "r" (m), "r" (old) 16 : "memory"); 17 18 return new; 19 } 20 21 static inline unsigned long xchg32(__volatile__ unsigned int *m, unsigned int val) 22 { 23 unsigned long tmp1, tmp2; 24 25 __asm__ __volatile__( 26 " mov %0, %1\n" 27 "1: lduw [%4], %2\n" 28 " cas [%4], %2, %0\n" 29 " cmp %2, %0\n" 30 " bne,a,pn %%icc, 1b\n" 31 " mov %1, %0\n" 32 : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) 33 : "0" (val), "r" (m) 34 : "cc", "memory"); 35 return val; 36 } 37 38 static inline unsigned long xchg64(__volatile__ unsigned long *m, unsigned long val) 39 { 40 unsigned long tmp1, tmp2; 41 42 __asm__ __volatile__( 43 " mov %0, %1\n" 44 "1: ldx [%4], %2\n" 45 " casx [%4], %2, %0\n" 46 " cmp %2, %0\n" 47 " bne,a,pn %%xcc, 1b\n" 48 " mov %1, %0\n" 49 : "=&r" (val), "=&r" (tmp1), "=&r" (tmp2) 50 : "0" (val), "r" (m) 51 : "cc", "memory"); 52 return val; 53 } 54 55 #define arch_xchg(ptr,x) \ 56 ({ __typeof__(*(ptr)) __ret; \ 57 __ret = (__typeof__(*(ptr))) \ 58 __arch_xchg((unsigned long)(x), (ptr), sizeof(*(ptr))); \ 59 __ret; \ 60 }) 61 62 void __xchg_called_with_bad_pointer(void); 63 64 /* 65 * Use 4 byte cas instruction to achieve 2 byte xchg. Main logic 66 * here is to get the bit shift of the byte we are interested in. 67 * The XOR is handy for reversing the bits for big-endian byte order. 68 */ 69 static inline unsigned long 70 xchg16(__volatile__ unsigned short *m, unsigned short val) 71 { 72 unsigned long maddr = (unsigned long)m; 73 int bit_shift = (((unsigned long)m & 2) ^ 2) << 3; 74 unsigned int mask = 0xffff << bit_shift; 75 unsigned int *ptr = (unsigned int *) (maddr & ~2); 76 unsigned int old32, new32, load32; 77 78 /* Read the old value */ 79 load32 = *ptr; 80 81 do { 82 old32 = load32; 83 new32 = (load32 & (~mask)) | val << bit_shift; 84 load32 = __cmpxchg_u32(ptr, old32, new32); 85 } while (load32 != old32); 86 87 return (load32 & mask) >> bit_shift; 88 } 89 90 static __always_inline unsigned long 91 __arch_xchg(unsigned long x, __volatile__ void * ptr, int size) 92 { 93 switch (size) { 94 case 2: 95 return xchg16(ptr, x); 96 case 4: 97 return xchg32(ptr, x); 98 case 8: 99 return xchg64(ptr, x); 100 } 101 __xchg_called_with_bad_pointer(); 102 return x; 103 } 104 105 /* 106 * Atomic compare and exchange. Compare OLD with MEM, if identical, 107 * store NEW in MEM. Return the initial value in MEM. Success is 108 * indicated by comparing RETURN with OLD. 109 */ 110 111 #include <asm-generic/cmpxchg-local.h> 112 113 114 static inline unsigned long 115 __cmpxchg_u64(volatile long *m, unsigned long old, unsigned long new) 116 { 117 __asm__ __volatile__("casx [%2], %3, %0" 118 : "=&r" (new) 119 : "0" (new), "r" (m), "r" (old) 120 : "memory"); 121 122 return new; 123 } 124 125 /* 126 * Use 4 byte cas instruction to achieve 1 byte cmpxchg. Main logic 127 * here is to get the bit shift of the byte we are interested in. 128 * The XOR is handy for reversing the bits for big-endian byte order 129 */ 130 static inline unsigned long 131 __cmpxchg_u8(volatile unsigned char *m, unsigned char old, unsigned char new) 132 { 133 unsigned long maddr = (unsigned long)m; 134 int bit_shift = (((unsigned long)m & 3) ^ 3) << 3; 135 unsigned int mask = 0xff << bit_shift; 136 unsigned int *ptr = (unsigned int *) (maddr & ~3); 137 unsigned int old32, new32, load; 138 unsigned int load32 = *ptr; 139 140 do { 141 new32 = (load32 & ~mask) | (new << bit_shift); 142 old32 = (load32 & ~mask) | (old << bit_shift); 143 load32 = __cmpxchg_u32(ptr, old32, new32); 144 if (load32 == old32) 145 return old; 146 load = (load32 & mask) >> bit_shift; 147 } while (load == old); 148 149 return load; 150 } 151 152 /* This function doesn't exist, so you'll get a linker error 153 if something tries to do an invalid cmpxchg(). */ 154 void __cmpxchg_called_with_bad_pointer(void); 155 156 static inline unsigned long 157 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) 158 { 159 switch (size) { 160 case 1: 161 return __cmpxchg_u8(ptr, old, new); 162 case 4: 163 return __cmpxchg_u32(ptr, old, new); 164 case 8: 165 return __cmpxchg_u64(ptr, old, new); 166 } 167 __cmpxchg_called_with_bad_pointer(); 168 return old; 169 } 170 171 #define arch_cmpxchg(ptr,o,n) \ 172 ({ \ 173 __typeof__(*(ptr)) _o_ = (o); \ 174 __typeof__(*(ptr)) _n_ = (n); \ 175 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ 176 (unsigned long)_n_, sizeof(*(ptr))); \ 177 }) 178 179 /* 180 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 181 * them available. 182 */ 183 184 static inline unsigned long __cmpxchg_local(volatile void *ptr, 185 unsigned long old, 186 unsigned long new, int size) 187 { 188 switch (size) { 189 case 4: 190 case 8: return __cmpxchg(ptr, old, new, size); 191 default: 192 return __generic_cmpxchg_local(ptr, old, new, size); 193 } 194 195 return old; 196 } 197 198 #define arch_cmpxchg_local(ptr, o, n) \ 199 ((__typeof__(*(ptr)))__cmpxchg_local((ptr), (unsigned long)(o), \ 200 (unsigned long)(n), sizeof(*(ptr)))) 201 #define arch_cmpxchg64_local(ptr, o, n) \ 202 ({ \ 203 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 204 arch_cmpxchg_local((ptr), (o), (n)); \ 205 }) 206 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) 207 208 #endif /* __ARCH_SPARC64_CMPXCHG__ */ 209