1 #ifndef ASM_X86_CMPXCHG_H 2 #define ASM_X86_CMPXCHG_H 3 4 #include <linux/compiler.h> 5 #include <asm/alternative.h> /* Provides LOCK_PREFIX */ 6 7 /* 8 * Non-existant functions to indicate usage errors at link time 9 * (or compile-time if the compiler implements __compiletime_error(). 10 */ 11 extern void __xchg_wrong_size(void) 12 __compiletime_error("Bad argument size for xchg"); 13 extern void __cmpxchg_wrong_size(void) 14 __compiletime_error("Bad argument size for cmpxchg"); 15 extern void __xadd_wrong_size(void) 16 __compiletime_error("Bad argument size for xadd"); 17 18 /* 19 * Constants for operation sizes. On 32-bit, the 64-bit size it set to 20 * -1 because sizeof will never return -1, thereby making those switch 21 * case statements guaranteeed dead code which the compiler will 22 * eliminate, and allowing the "missing symbol in the default case" to 23 * indicate a usage error. 24 */ 25 #define __X86_CASE_B 1 26 #define __X86_CASE_W 2 27 #define __X86_CASE_L 4 28 #ifdef CONFIG_64BIT 29 #define __X86_CASE_Q 8 30 #else 31 #define __X86_CASE_Q -1 /* sizeof will never return -1 */ 32 #endif 33 34 /* 35 * Note: no "lock" prefix even on SMP: xchg always implies lock anyway. 36 * Since this is generally used to protect other memory information, we 37 * use "asm volatile" and "memory" clobbers to prevent gcc from moving 38 * information around. 39 */ 40 #define __xchg(x, ptr, size) \ 41 ({ \ 42 __typeof(*(ptr)) __x = (x); \ 43 switch (size) { \ 44 case __X86_CASE_B: \ 45 { \ 46 volatile u8 *__ptr = (volatile u8 *)(ptr); \ 47 asm volatile("xchgb %0,%1" \ 48 : "=q" (__x), "+m" (*__ptr) \ 49 : "0" (__x) \ 50 : "memory"); \ 51 break; \ 52 } \ 53 case __X86_CASE_W: \ 54 { \ 55 volatile u16 *__ptr = (volatile u16 *)(ptr); \ 56 asm volatile("xchgw %0,%1" \ 57 : "=r" (__x), "+m" (*__ptr) \ 58 : "0" (__x) \ 59 : "memory"); \ 60 break; \ 61 } \ 62 case __X86_CASE_L: \ 63 { \ 64 volatile u32 *__ptr = (volatile u32 *)(ptr); \ 65 asm volatile("xchgl %0,%1" \ 66 : "=r" (__x), "+m" (*__ptr) \ 67 : "0" (__x) \ 68 : "memory"); \ 69 break; \ 70 } \ 71 case __X86_CASE_Q: \ 72 { \ 73 volatile u64 *__ptr = (volatile u64 *)(ptr); \ 74 asm volatile("xchgq %0,%1" \ 75 : "=r" (__x), "+m" (*__ptr) \ 76 : "0" (__x) \ 77 : "memory"); \ 78 break; \ 79 } \ 80 default: \ 81 __xchg_wrong_size(); \ 82 } \ 83 __x; \ 84 }) 85 86 #define xchg(ptr, v) \ 87 __xchg((v), (ptr), sizeof(*ptr)) 88 89 /* 90 * Atomic compare and exchange. Compare OLD with MEM, if identical, 91 * store NEW in MEM. Return the initial value in MEM. Success is 92 * indicated by comparing RETURN with OLD. 93 */ 94 #define __raw_cmpxchg(ptr, old, new, size, lock) \ 95 ({ \ 96 __typeof__(*(ptr)) __ret; \ 97 __typeof__(*(ptr)) __old = (old); \ 98 __typeof__(*(ptr)) __new = (new); \ 99 switch (size) { \ 100 case __X86_CASE_B: \ 101 { \ 102 volatile u8 *__ptr = (volatile u8 *)(ptr); \ 103 asm volatile(lock "cmpxchgb %2,%1" \ 104 : "=a" (__ret), "+m" (*__ptr) \ 105 : "q" (__new), "0" (__old) \ 106 : "memory"); \ 107 break; \ 108 } \ 109 case __X86_CASE_W: \ 110 { \ 111 volatile u16 *__ptr = (volatile u16 *)(ptr); \ 112 asm volatile(lock "cmpxchgw %2,%1" \ 113 : "=a" (__ret), "+m" (*__ptr) \ 114 : "r" (__new), "0" (__old) \ 115 : "memory"); \ 116 break; \ 117 } \ 118 case __X86_CASE_L: \ 119 { \ 120 volatile u32 *__ptr = (volatile u32 *)(ptr); \ 121 asm volatile(lock "cmpxchgl %2,%1" \ 122 : "=a" (__ret), "+m" (*__ptr) \ 123 : "r" (__new), "0" (__old) \ 124 : "memory"); \ 125 break; \ 126 } \ 127 case __X86_CASE_Q: \ 128 { \ 129 volatile u64 *__ptr = (volatile u64 *)(ptr); \ 130 asm volatile(lock "cmpxchgq %2,%1" \ 131 : "=a" (__ret), "+m" (*__ptr) \ 132 : "r" (__new), "0" (__old) \ 133 : "memory"); \ 134 break; \ 135 } \ 136 default: \ 137 __cmpxchg_wrong_size(); \ 138 } \ 139 __ret; \ 140 }) 141 142 #define __cmpxchg(ptr, old, new, size) \ 143 __raw_cmpxchg((ptr), (old), (new), (size), LOCK_PREFIX) 144 145 #define __sync_cmpxchg(ptr, old, new, size) \ 146 __raw_cmpxchg((ptr), (old), (new), (size), "lock; ") 147 148 #define __cmpxchg_local(ptr, old, new, size) \ 149 __raw_cmpxchg((ptr), (old), (new), (size), "") 150 151 #ifdef CONFIG_X86_32 152 # include "cmpxchg_32.h" 153 #else 154 # include "cmpxchg_64.h" 155 #endif 156 157 #ifdef __HAVE_ARCH_CMPXCHG 158 #define cmpxchg(ptr, old, new) \ 159 __cmpxchg((ptr), (old), (new), sizeof(*ptr)) 160 161 #define sync_cmpxchg(ptr, old, new) \ 162 __sync_cmpxchg((ptr), (old), (new), sizeof(*ptr)) 163 164 #define cmpxchg_local(ptr, old, new) \ 165 __cmpxchg_local((ptr), (old), (new), sizeof(*ptr)) 166 #endif 167 168 #define __xadd(ptr, inc, lock) \ 169 ({ \ 170 __typeof__ (*(ptr)) __ret = (inc); \ 171 switch (sizeof(*(ptr))) { \ 172 case __X86_CASE_B: \ 173 asm volatile (lock "xaddb %b0, %1\n" \ 174 : "+r" (__ret), "+m" (*(ptr)) \ 175 : : "memory", "cc"); \ 176 break; \ 177 case __X86_CASE_W: \ 178 asm volatile (lock "xaddw %w0, %1\n" \ 179 : "+r" (__ret), "+m" (*(ptr)) \ 180 : : "memory", "cc"); \ 181 break; \ 182 case __X86_CASE_L: \ 183 asm volatile (lock "xaddl %0, %1\n" \ 184 : "+r" (__ret), "+m" (*(ptr)) \ 185 : : "memory", "cc"); \ 186 break; \ 187 case __X86_CASE_Q: \ 188 asm volatile (lock "xaddq %q0, %1\n" \ 189 : "+r" (__ret), "+m" (*(ptr)) \ 190 : : "memory", "cc"); \ 191 break; \ 192 default: \ 193 __xadd_wrong_size(); \ 194 } \ 195 __ret; \ 196 }) 197 198 /* 199 * xadd() adds "inc" to "*ptr" and atomically returns the previous 200 * value of "*ptr". 201 * 202 * xadd() is locked when multiple CPUs are online 203 * xadd_sync() is always locked 204 * xadd_local() is never locked 205 */ 206 #define xadd(ptr, inc) __xadd((ptr), (inc), LOCK_PREFIX) 207 #define xadd_sync(ptr, inc) __xadd((ptr), (inc), "lock; ") 208 #define xadd_local(ptr, inc) __xadd((ptr), (inc), "") 209 210 #endif /* ASM_X86_CMPXCHG_H */ 211