1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_CMPXCHG_32_H 3 #define _ASM_X86_CMPXCHG_32_H 4 5 /* 6 * Note: if you use set64_bit(), __cmpxchg64(), or their variants, 7 * you need to test for the feature in boot_cpu_data. 8 */ 9 10 /* 11 * CMPXCHG8B only writes to the target if we had the previous 12 * value in registers, otherwise it acts as a read and gives us the 13 * "new previous" value. That is why there is a loop. Preloading 14 * EDX:EAX is a performance optimization: in the common case it means 15 * we need only one locked operation. 16 * 17 * A SIMD/3DNOW!/MMX/FPU 64-bit store here would require at the very 18 * least an FPU save and/or %cr0.ts manipulation. 19 * 20 * cmpxchg8b must be used with the lock prefix here to allow the 21 * instruction to be executed atomically. We need to have the reader 22 * side to see the coherent 64bit value. 23 */ 24 static inline void set_64bit(volatile u64 *ptr, u64 value) 25 { 26 u32 low = value; 27 u32 high = value >> 32; 28 u64 prev = *ptr; 29 30 asm volatile("\n1:\t" 31 LOCK_PREFIX "cmpxchg8b %0\n\t" 32 "jnz 1b" 33 : "=m" (*ptr), "+A" (prev) 34 : "b" (low), "c" (high) 35 : "memory"); 36 } 37 38 #ifdef CONFIG_X86_CMPXCHG64 39 #define arch_cmpxchg64(ptr, o, n) \ 40 ((__typeof__(*(ptr)))__cmpxchg64((ptr), (unsigned long long)(o), \ 41 (unsigned long long)(n))) 42 #define arch_cmpxchg64_local(ptr, o, n) \ 43 ((__typeof__(*(ptr)))__cmpxchg64_local((ptr), (unsigned long long)(o), \ 44 (unsigned long long)(n))) 45 #endif 46 47 static inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 48 { 49 u64 prev; 50 asm volatile(LOCK_PREFIX "cmpxchg8b %1" 51 : "=A" (prev), 52 "+m" (*ptr) 53 : "b" ((u32)new), 54 "c" ((u32)(new >> 32)), 55 "0" (old) 56 : "memory"); 57 return prev; 58 } 59 60 static inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 61 { 62 u64 prev; 63 asm volatile("cmpxchg8b %1" 64 : "=A" (prev), 65 "+m" (*ptr) 66 : "b" ((u32)new), 67 "c" ((u32)(new >> 32)), 68 "0" (old) 69 : "memory"); 70 return prev; 71 } 72 73 #ifndef CONFIG_X86_CMPXCHG64 74 /* 75 * Building a kernel capable running on 80386 and 80486. It may be necessary 76 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 77 */ 78 79 #define arch_cmpxchg64(ptr, o, n) \ 80 ({ \ 81 __typeof__(*(ptr)) __ret; \ 82 __typeof__(*(ptr)) __old = (o); \ 83 __typeof__(*(ptr)) __new = (n); \ 84 alternative_io(LOCK_PREFIX_HERE \ 85 "call cmpxchg8b_emu", \ 86 "lock; cmpxchg8b (%%esi)" , \ 87 X86_FEATURE_CX8, \ 88 "=A" (__ret), \ 89 "S" ((ptr)), "0" (__old), \ 90 "b" ((unsigned int)__new), \ 91 "c" ((unsigned int)(__new>>32)) \ 92 : "memory"); \ 93 __ret; }) 94 95 96 #define arch_cmpxchg64_local(ptr, o, n) \ 97 ({ \ 98 __typeof__(*(ptr)) __ret; \ 99 __typeof__(*(ptr)) __old = (o); \ 100 __typeof__(*(ptr)) __new = (n); \ 101 alternative_io("call cmpxchg8b_emu", \ 102 "cmpxchg8b (%%esi)" , \ 103 X86_FEATURE_CX8, \ 104 "=A" (__ret), \ 105 "S" ((ptr)), "0" (__old), \ 106 "b" ((unsigned int)__new), \ 107 "c" ((unsigned int)(__new>>32)) \ 108 : "memory"); \ 109 __ret; }) 110 111 #endif 112 113 #define system_has_cmpxchg_double() boot_cpu_has(X86_FEATURE_CX8) 114 115 #endif /* _ASM_X86_CMPXCHG_32_H */ 116