1 #ifndef _ASM_X86_CMPXCHG_32_H 2 #define _ASM_X86_CMPXCHG_32_H 3 4 /* 5 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, you 6 * you need to test for the feature in boot_cpu_data. 7 */ 8 9 /* 10 * CMPXCHG8B only writes to the target if we had the previous 11 * value in registers, otherwise it acts as a read and gives us the 12 * "new previous" value. That is why there is a loop. Preloading 13 * EDX:EAX is a performance optimization: in the common case it means 14 * we need only one locked operation. 15 * 16 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very 17 * least an FPU save and/or %cr0.ts manipulation. 18 * 19 * cmpxchg8b must be used with the lock prefix here to allow the 20 * instruction to be executed atomically. We need to have the reader 21 * side to see the coherent 64bit value. 22 */ 23 static inline void set_64bit(volatile u64 *ptr, u64 value) 24 { 25 u32 low = value; 26 u32 high = value >> 32; 27 u64 prev = *ptr; 28 29 asm volatile("\n1:\t" 30 LOCK_PREFIX "cmpxchg8b %0\n\t" 31 "jnz 1b" 32 : "=m" (*ptr), "+A" (prev) 33 : "b" (low), "c" (high) 34 : "memory"); 35 } 36 37 #ifdef CONFIG_X86_CMPXCHG 38 #define __HAVE_ARCH_CMPXCHG 1 39 #endif 40 41 #ifdef CONFIG_X86_CMPXCHG64 42 #define cmpxchg64(ptr, o, n) \ 43 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ 44 (unsigned long long)(n))) 45 #define cmpxchg64_local(ptr, o, n) \ 46 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ 47 (unsigned long long)(n))) 48 #endif 49 50 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 51 { 52 u64 prev; 53 asm volatile(LOCK_PREFIX "cmpxchg8b %1" 54 : "=A" (prev), 55 "+m" (*ptr) 56 : "b" ((u32)new), 57 "c" ((u32)(new >> 32)), 58 "0" (old) 59 : "memory"); 60 return prev; 61 } 62 63 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 64 { 65 u64 prev; 66 asm volatile("cmpxchg8b %1" 67 : "=A" (prev), 68 "+m" (*ptr) 69 : "b" ((u32)new), 70 "c" ((u32)(new >> 32)), 71 "0" (old) 72 : "memory"); 73 return prev; 74 } 75 76 #ifndef CONFIG_X86_CMPXCHG 77 /* 78 * Building a kernel capable running on 80386. It may be necessary to 79 * simulate the cmpxchg on the 80386 CPU. For that purpose we define 80 * a function for each of the sizes we support. 81 */ 82 83 extern unsigned long cmpxchg_386_u8(volatile void *, u8, u8); 84 extern unsigned long cmpxchg_386_u16(volatile void *, u16, u16); 85 extern unsigned long cmpxchg_386_u32(volatile void *, u32, u32); 86 87 static inline unsigned long cmpxchg_386(volatile void *ptr, unsigned long old, 88 unsigned long new, int size) 89 { 90 switch (size) { 91 case 1: 92 return cmpxchg_386_u8(ptr, old, new); 93 case 2: 94 return cmpxchg_386_u16(ptr, old, new); 95 case 4: 96 return cmpxchg_386_u32(ptr, old, new); 97 } 98 return old; 99 } 100 101 #define cmpxchg(ptr, o, n) \ 102 ({ \ 103 __typeof__(*(ptr)) __ret; \ 104 if (likely(boot_cpu_data.x86 > 3)) \ 105 __ret = (__typeof__(*(ptr)))__cmpxchg((ptr), \ 106 (unsigned long)(o), (unsigned long)(n), \ 107 sizeof(*(ptr))); \ 108 else \ 109 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 110 (unsigned long)(o), (unsigned long)(n), \ 111 sizeof(*(ptr))); \ 112 __ret; \ 113 }) 114 #define cmpxchg_local(ptr, o, n) \ 115 ({ \ 116 __typeof__(*(ptr)) __ret; \ 117 if (likely(boot_cpu_data.x86 > 3)) \ 118 __ret = (__typeof__(*(ptr)))__cmpxchg_local((ptr), \ 119 (unsigned long)(o), (unsigned long)(n), \ 120 sizeof(*(ptr))); \ 121 else \ 122 __ret = (__typeof__(*(ptr)))cmpxchg_386((ptr), \ 123 (unsigned long)(o), (unsigned long)(n), \ 124 sizeof(*(ptr))); \ 125 __ret; \ 126 }) 127 #endif 128 129 #ifndef CONFIG_X86_CMPXCHG64 130 /* 131 * Building a kernel capable running on 80386 and 80486. It may be necessary 132 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 133 */ 134 135 #define cmpxchg64(ptr, o, n) \ 136 ({ \ 137 __typeof__(*(ptr)) __ret; \ 138 __typeof__(*(ptr)) __old = (o); \ 139 __typeof__(*(ptr)) __new = (n); \ 140 alternative_io(LOCK_PREFIX_HERE \ 141 "call cmpxchg8b_emu", \ 142 "lock; cmpxchg8b (%%esi)" , \ 143 X86_FEATURE_CX8, \ 144 "=A" (__ret), \ 145 "S" ((ptr)), "0" (__old), \ 146 "b" ((unsigned int)__new), \ 147 "c" ((unsigned int)(__new>>32)) \ 148 : "memory"); \ 149 __ret; }) 150 151 152 #define cmpxchg64_local(ptr, o, n) \ 153 ({ \ 154 __typeof__(*(ptr)) __ret; \ 155 __typeof__(*(ptr)) __old = (o); \ 156 __typeof__(*(ptr)) __new = (n); \ 157 alternative_io("call cmpxchg8b_emu", \ 158 "cmpxchg8b (%%esi)" , \ 159 X86_FEATURE_CX8, \ 160 "=A" (__ret), \ 161 "S" ((ptr)), "0" (__old), \ 162 "b" ((unsigned int)__new), \ 163 "c" ((unsigned int)(__new>>32)) \ 164 : "memory"); \ 165 __ret; }) 166 167 #endif 168 169 #define cmpxchg8b(ptr, o1, o2, n1, n2) \ 170 ({ \ 171 char __ret; \ 172 __typeof__(o2) __dummy; \ 173 __typeof__(*(ptr)) __old1 = (o1); \ 174 __typeof__(o2) __old2 = (o2); \ 175 __typeof__(*(ptr)) __new1 = (n1); \ 176 __typeof__(o2) __new2 = (n2); \ 177 asm volatile(LOCK_PREFIX "cmpxchg8b %2; setz %1" \ 178 : "=d"(__dummy), "=a" (__ret), "+m" (*ptr)\ 179 : "a" (__old1), "d"(__old2), \ 180 "b" (__new1), "c" (__new2) \ 181 : "memory"); \ 182 __ret; }) 183 184 185 #define cmpxchg8b_local(ptr, o1, o2, n1, n2) \ 186 ({ \ 187 char __ret; \ 188 __typeof__(o2) __dummy; \ 189 __typeof__(*(ptr)) __old1 = (o1); \ 190 __typeof__(o2) __old2 = (o2); \ 191 __typeof__(*(ptr)) __new1 = (n1); \ 192 __typeof__(o2) __new2 = (n2); \ 193 asm volatile("cmpxchg8b %2; setz %1" \ 194 : "=d"(__dummy), "=a"(__ret), "+m" (*ptr)\ 195 : "a" (__old), "d"(__old2), \ 196 "b" (__new1), "c" (__new2), \ 197 : "memory"); \ 198 __ret; }) 199 200 201 #define cmpxchg_double(ptr, o1, o2, n1, n2) \ 202 ({ \ 203 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 204 VM_BUG_ON((unsigned long)(ptr) % 8); \ 205 cmpxchg8b((ptr), (o1), (o2), (n1), (n2)); \ 206 }) 207 208 #define cmpxchg_double_local(ptr, o1, o2, n1, n2) \ 209 ({ \ 210 BUILD_BUG_ON(sizeof(*(ptr)) != 4); \ 211 VM_BUG_ON((unsigned long)(ptr) % 8); \ 212 cmpxchg16b_local((ptr), (o1), (o2), (n1), (n2)); \ 213 }) 214 215 #define system_has_cmpxchg_double() cpu_has_cx8 216 217 #endif /* _ASM_X86_CMPXCHG_32_H */ 218