1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * Copyright (C) 2003, 06, 07 by Ralf Baechle (ralf@linux-mips.org) 7 */ 8 #ifndef __ASM_CMPXCHG_H 9 #define __ASM_CMPXCHG_H 10 11 #include <linux/bug.h> 12 #include <linux/irqflags.h> 13 #include <asm/compiler.h> 14 #include <asm/war.h> 15 16 static inline unsigned long __xchg_u32(volatile int * m, unsigned int val) 17 { 18 __u32 retval; 19 20 smp_mb__before_llsc(); 21 22 if (kernel_uses_llsc && R10000_LLSC_WAR) { 23 unsigned long dummy; 24 25 __asm__ __volatile__( 26 " .set arch=r4000 \n" 27 "1: ll %0, %3 # xchg_u32 \n" 28 " .set mips0 \n" 29 " move %2, %z4 \n" 30 " .set arch=r4000 \n" 31 " sc %2, %1 \n" 32 " beqzl %2, 1b \n" 33 " .set mips0 \n" 34 : "=&r" (retval), "=" GCC_OFF_SMALL_ASM() (*m), "=&r" (dummy) 35 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) 36 : "memory"); 37 } else if (kernel_uses_llsc) { 38 unsigned long dummy; 39 40 do { 41 __asm__ __volatile__( 42 " .set "MIPS_ISA_ARCH_LEVEL" \n" 43 " ll %0, %3 # xchg_u32 \n" 44 " .set mips0 \n" 45 " move %2, %z4 \n" 46 " .set "MIPS_ISA_ARCH_LEVEL" \n" 47 " sc %2, %1 \n" 48 " .set mips0 \n" 49 : "=&r" (retval), "=" GCC_OFF_SMALL_ASM() (*m), 50 "=&r" (dummy) 51 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) 52 : "memory"); 53 } while (unlikely(!dummy)); 54 } else { 55 unsigned long flags; 56 57 raw_local_irq_save(flags); 58 retval = *m; 59 *m = val; 60 raw_local_irq_restore(flags); /* implies memory barrier */ 61 } 62 63 smp_llsc_mb(); 64 65 return retval; 66 } 67 68 #ifdef CONFIG_64BIT 69 static inline __u64 __xchg_u64(volatile __u64 * m, __u64 val) 70 { 71 __u64 retval; 72 73 smp_mb__before_llsc(); 74 75 if (kernel_uses_llsc && R10000_LLSC_WAR) { 76 unsigned long dummy; 77 78 __asm__ __volatile__( 79 " .set arch=r4000 \n" 80 "1: lld %0, %3 # xchg_u64 \n" 81 " move %2, %z4 \n" 82 " scd %2, %1 \n" 83 " beqzl %2, 1b \n" 84 " .set mips0 \n" 85 : "=&r" (retval), "=" GCC_OFF_SMALL_ASM() (*m), "=&r" (dummy) 86 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) 87 : "memory"); 88 } else if (kernel_uses_llsc) { 89 unsigned long dummy; 90 91 do { 92 __asm__ __volatile__( 93 " .set "MIPS_ISA_ARCH_LEVEL" \n" 94 " lld %0, %3 # xchg_u64 \n" 95 " move %2, %z4 \n" 96 " scd %2, %1 \n" 97 " .set mips0 \n" 98 : "=&r" (retval), "=" GCC_OFF_SMALL_ASM() (*m), 99 "=&r" (dummy) 100 : GCC_OFF_SMALL_ASM() (*m), "Jr" (val) 101 : "memory"); 102 } while (unlikely(!dummy)); 103 } else { 104 unsigned long flags; 105 106 raw_local_irq_save(flags); 107 retval = *m; 108 *m = val; 109 raw_local_irq_restore(flags); /* implies memory barrier */ 110 } 111 112 smp_llsc_mb(); 113 114 return retval; 115 } 116 #else 117 extern __u64 __xchg_u64_unsupported_on_32bit_kernels(volatile __u64 * m, __u64 val); 118 #define __xchg_u64 __xchg_u64_unsupported_on_32bit_kernels 119 #endif 120 121 static inline unsigned long __xchg(unsigned long x, volatile void * ptr, int size) 122 { 123 switch (size) { 124 case 4: 125 return __xchg_u32(ptr, x); 126 case 8: 127 return __xchg_u64(ptr, x); 128 } 129 130 return x; 131 } 132 133 #define xchg(ptr, x) \ 134 ({ \ 135 BUILD_BUG_ON(sizeof(*(ptr)) & ~0xc); \ 136 \ 137 ((__typeof__(*(ptr))) \ 138 __xchg((unsigned long)(x), (ptr), sizeof(*(ptr)))); \ 139 }) 140 141 #define __cmpxchg_asm(ld, st, m, old, new) \ 142 ({ \ 143 __typeof(*(m)) __ret; \ 144 \ 145 if (kernel_uses_llsc && R10000_LLSC_WAR) { \ 146 __asm__ __volatile__( \ 147 " .set push \n" \ 148 " .set noat \n" \ 149 " .set arch=r4000 \n" \ 150 "1: " ld " %0, %2 # __cmpxchg_asm \n" \ 151 " bne %0, %z3, 2f \n" \ 152 " .set mips0 \n" \ 153 " move $1, %z4 \n" \ 154 " .set arch=r4000 \n" \ 155 " " st " $1, %1 \n" \ 156 " beqzl $1, 1b \n" \ 157 "2: \n" \ 158 " .set pop \n" \ 159 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 160 : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ 161 : "memory"); \ 162 } else if (kernel_uses_llsc) { \ 163 __asm__ __volatile__( \ 164 " .set push \n" \ 165 " .set noat \n" \ 166 " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 167 "1: " ld " %0, %2 # __cmpxchg_asm \n" \ 168 " bne %0, %z3, 2f \n" \ 169 " .set mips0 \n" \ 170 " move $1, %z4 \n" \ 171 " .set "MIPS_ISA_ARCH_LEVEL" \n" \ 172 " " st " $1, %1 \n" \ 173 " beqz $1, 1b \n" \ 174 " .set pop \n" \ 175 "2: \n" \ 176 : "=&r" (__ret), "=" GCC_OFF_SMALL_ASM() (*m) \ 177 : GCC_OFF_SMALL_ASM() (*m), "Jr" (old), "Jr" (new) \ 178 : "memory"); \ 179 } else { \ 180 unsigned long __flags; \ 181 \ 182 raw_local_irq_save(__flags); \ 183 __ret = *m; \ 184 if (__ret == old) \ 185 *m = new; \ 186 raw_local_irq_restore(__flags); \ 187 } \ 188 \ 189 __ret; \ 190 }) 191 192 /* 193 * This function doesn't exist, so you'll get a linker error 194 * if something tries to do an invalid cmpxchg(). 195 */ 196 extern void __cmpxchg_called_with_bad_pointer(void); 197 198 #define __cmpxchg(ptr, old, new, pre_barrier, post_barrier) \ 199 ({ \ 200 __typeof__(ptr) __ptr = (ptr); \ 201 __typeof__(*(ptr)) __old = (old); \ 202 __typeof__(*(ptr)) __new = (new); \ 203 __typeof__(*(ptr)) __res = 0; \ 204 \ 205 pre_barrier; \ 206 \ 207 switch (sizeof(*(__ptr))) { \ 208 case 4: \ 209 __res = __cmpxchg_asm("ll", "sc", __ptr, __old, __new); \ 210 break; \ 211 case 8: \ 212 if (sizeof(long) == 8) { \ 213 __res = __cmpxchg_asm("lld", "scd", __ptr, \ 214 __old, __new); \ 215 break; \ 216 } \ 217 default: \ 218 __cmpxchg_called_with_bad_pointer(); \ 219 break; \ 220 } \ 221 \ 222 post_barrier; \ 223 \ 224 __res; \ 225 }) 226 227 #define cmpxchg(ptr, old, new) __cmpxchg(ptr, old, new, smp_mb__before_llsc(), smp_llsc_mb()) 228 #define cmpxchg_local(ptr, old, new) __cmpxchg(ptr, old, new, , ) 229 230 #ifdef CONFIG_64BIT 231 #define cmpxchg64_local(ptr, o, n) \ 232 ({ \ 233 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 234 cmpxchg_local((ptr), (o), (n)); \ 235 }) 236 237 #define cmpxchg64(ptr, o, n) \ 238 ({ \ 239 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 240 cmpxchg((ptr), (o), (n)); \ 241 }) 242 #else 243 #include <asm-generic/cmpxchg-local.h> 244 #define cmpxchg64_local(ptr, o, n) __cmpxchg64_local_generic((ptr), (o), (n)) 245 #define cmpxchg64(ptr, o, n) cmpxchg64_local((ptr), (o), (n)) 246 #endif 247 248 #endif /* __ASM_CMPXCHG_H */ 249