1 /* 2 * Atomic xchg and cmpxchg operations. 3 * 4 * This file is subject to the terms and conditions of the GNU General Public 5 * License. See the file "COPYING" in the main directory of this archive 6 * for more details. 7 * 8 * Copyright (C) 2001 - 2005 Tensilica Inc. 9 */ 10 11 #ifndef _XTENSA_CMPXCHG_H 12 #define _XTENSA_CMPXCHG_H 13 14 #ifndef __ASSEMBLER__ 15 16 #include <linux/bits.h> 17 #include <linux/stringify.h> 18 #include <linux/cmpxchg-emu.h> 19 20 /* 21 * cmpxchg 22 */ 23 24 static inline unsigned long 25 __cmpxchg_u32(volatile int *p, int old, int new) 26 { 27 #if XCHAL_HAVE_EXCLUSIVE 28 unsigned long tmp, result; 29 30 __asm__ __volatile__( 31 "1: l32ex %[result], %[addr]\n" 32 " bne %[result], %[cmp], 2f\n" 33 " mov %[tmp], %[new]\n" 34 " s32ex %[tmp], %[addr]\n" 35 " getex %[tmp]\n" 36 " beqz %[tmp], 1b\n" 37 "2:\n" 38 : [result] "=&a" (result), [tmp] "=&a" (tmp) 39 : [new] "a" (new), [addr] "a" (p), [cmp] "a" (old) 40 : "memory" 41 ); 42 43 return result; 44 #elif XCHAL_HAVE_S32C1I 45 __asm__ __volatile__( 46 " wsr %[cmp], scompare1\n" 47 " s32c1i %[new], %[mem]\n" 48 : [new] "+a" (new), [mem] "+m" (*p) 49 : [cmp] "a" (old) 50 : "memory" 51 ); 52 53 return new; 54 #else 55 __asm__ __volatile__( 56 " rsil a14, "__stringify(TOPLEVEL)"\n" 57 " l32i %[old], %[mem]\n" 58 " bne %[old], %[cmp], 1f\n" 59 " s32i %[new], %[mem]\n" 60 "1:\n" 61 " wsr a14, ps\n" 62 " rsync\n" 63 : [old] "=&a" (old), [mem] "+m" (*p) 64 : [cmp] "a" (old), [new] "r" (new) 65 : "a14", "memory"); 66 return old; 67 #endif 68 } 69 /* This function doesn't exist, so you'll get a linker error 70 * if something tries to do an invalid cmpxchg(). */ 71 72 extern void __cmpxchg_called_with_bad_pointer(void); 73 74 static __inline__ unsigned long 75 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, int size) 76 { 77 switch (size) { 78 case 1: return cmpxchg_emu_u8(ptr, old, new); 79 case 2: return cmpxchg_emu_u16(ptr, old, new); 80 case 4: return __cmpxchg_u32(ptr, old, new); 81 default: __cmpxchg_called_with_bad_pointer(); 82 return old; 83 } 84 } 85 86 #define arch_cmpxchg(ptr,o,n) \ 87 ({ __typeof__(*(ptr)) _o_ = (o); \ 88 __typeof__(*(ptr)) _n_ = (n); \ 89 (__typeof__(*(ptr))) __cmpxchg((ptr), (unsigned long)_o_, \ 90 (unsigned long)_n_, sizeof (*(ptr))); \ 91 }) 92 93 #include <asm-generic/cmpxchg-local.h> 94 95 static inline unsigned long __cmpxchg_local(volatile void *ptr, 96 unsigned long old, 97 unsigned long new, int size) 98 { 99 switch (size) { 100 case 4: 101 return __cmpxchg_u32(ptr, old, new); 102 default: 103 return __generic_cmpxchg_local(ptr, old, new, size); 104 } 105 106 return old; 107 } 108 109 /* 110 * cmpxchg_local and cmpxchg64_local are atomic wrt current CPU. Always make 111 * them available. 112 */ 113 #define arch_cmpxchg_local(ptr, o, n) \ 114 ((__typeof__(*(ptr)))__generic_cmpxchg_local((ptr), (unsigned long)(o),\ 115 (unsigned long)(n), sizeof(*(ptr)))) 116 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) 117 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) 118 119 /* 120 * xchg_u32 121 * 122 * Note that a14 is used here because the register allocation 123 * done by the compiler is not guaranteed and a window overflow 124 * may not occur between the rsil and wsr instructions. By using 125 * a14 in the rsil, the machine is guaranteed to be in a state 126 * where no register reference will cause an overflow. 127 */ 128 129 static inline unsigned long xchg_u32(volatile int * m, unsigned long val) 130 { 131 #if XCHAL_HAVE_EXCLUSIVE 132 unsigned long tmp, result; 133 134 __asm__ __volatile__( 135 "1: l32ex %[result], %[addr]\n" 136 " mov %[tmp], %[val]\n" 137 " s32ex %[tmp], %[addr]\n" 138 " getex %[tmp]\n" 139 " beqz %[tmp], 1b\n" 140 : [result] "=&a" (result), [tmp] "=&a" (tmp) 141 : [val] "a" (val), [addr] "a" (m) 142 : "memory" 143 ); 144 145 return result; 146 #elif XCHAL_HAVE_S32C1I 147 unsigned long tmp, result; 148 __asm__ __volatile__( 149 "1: l32i %[tmp], %[mem]\n" 150 " mov %[result], %[val]\n" 151 " wsr %[tmp], scompare1\n" 152 " s32c1i %[result], %[mem]\n" 153 " bne %[result], %[tmp], 1b\n" 154 : [result] "=&a" (result), [tmp] "=&a" (tmp), 155 [mem] "+m" (*m) 156 : [val] "a" (val) 157 : "memory" 158 ); 159 return result; 160 #else 161 unsigned long tmp; 162 __asm__ __volatile__( 163 " rsil a14, "__stringify(TOPLEVEL)"\n" 164 " l32i %[tmp], %[mem]\n" 165 " s32i %[val], %[mem]\n" 166 " wsr a14, ps\n" 167 " rsync\n" 168 : [tmp] "=&a" (tmp), [mem] "+m" (*m) 169 : [val] "a" (val) 170 : "a14", "memory"); 171 return tmp; 172 #endif 173 } 174 175 #define arch_xchg(ptr,x) \ 176 ((__typeof__(*(ptr)))__arch_xchg((unsigned long)(x),(ptr),sizeof(*(ptr)))) 177 178 static inline u32 xchg_small(volatile void *ptr, u32 x, int size) 179 { 180 int off = (unsigned long)ptr % sizeof(u32); 181 volatile u32 *p = ptr - off; 182 #ifdef __BIG_ENDIAN 183 int bitoff = (sizeof(u32) - size - off) * BITS_PER_BYTE; 184 #else 185 int bitoff = off * BITS_PER_BYTE; 186 #endif 187 u32 bitmask = ((0x1 << size * BITS_PER_BYTE) - 1) << bitoff; 188 u32 oldv, newv; 189 u32 ret; 190 191 do { 192 oldv = READ_ONCE(*p); 193 ret = (oldv & bitmask) >> bitoff; 194 newv = (oldv & ~bitmask) | (x << bitoff); 195 } while (__cmpxchg_u32(p, oldv, newv) != oldv); 196 197 return ret; 198 } 199 200 /* 201 * This only works if the compiler isn't horribly bad at optimizing. 202 * gcc-2.5.8 reportedly can't handle this, but I define that one to 203 * be dead anyway. 204 */ 205 206 extern void __xchg_called_with_bad_pointer(void); 207 208 static __inline__ unsigned long 209 __arch_xchg(unsigned long x, volatile void * ptr, int size) 210 { 211 switch (size) { 212 case 1: 213 return xchg_small(ptr, x, 1); 214 case 2: 215 return xchg_small(ptr, x, 2); 216 case 4: 217 return xchg_u32(ptr, x); 218 default: 219 __xchg_called_with_bad_pointer(); 220 return x; 221 } 222 } 223 224 #endif /* __ASSEMBLER__ */ 225 226 #endif /* _XTENSA_CMPXCHG_H */ 227