1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _ASM_X86_CMPXCHG_32_H 3 #define _ASM_X86_CMPXCHG_32_H 4 5 /* 6 * Note: if you use __cmpxchg64(), or their variants, 7 * you need to test for the feature in boot_cpu_data. 8 */ 9 10 union __u64_halves { 11 u64 full; 12 struct { 13 u32 low, high; 14 }; 15 }; 16 17 #define __arch_cmpxchg64(_ptr, _old, _new, _lock) \ 18 ({ \ 19 union __u64_halves o = { .full = (_old), }, \ 20 n = { .full = (_new), }; \ 21 \ 22 asm volatile(_lock "cmpxchg8b %[ptr]" \ 23 : [ptr] "+m" (*(_ptr)), \ 24 "+a" (o.low), "+d" (o.high) \ 25 : "b" (n.low), "c" (n.high) \ 26 : "memory"); \ 27 \ 28 o.full; \ 29 }) 30 31 32 static __always_inline u64 __cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 33 { 34 return __arch_cmpxchg64(ptr, old, new, LOCK_PREFIX); 35 } 36 37 static __always_inline u64 __cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 38 { 39 return __arch_cmpxchg64(ptr, old, new,); 40 } 41 42 #define __arch_try_cmpxchg64(_ptr, _oldp, _new, _lock) \ 43 ({ \ 44 union __u64_halves o = { .full = *(_oldp), }, \ 45 n = { .full = (_new), }; \ 46 bool ret; \ 47 \ 48 asm volatile(_lock "cmpxchg8b %[ptr]" \ 49 CC_SET(e) \ 50 : CC_OUT(e) (ret), \ 51 [ptr] "+m" (*(_ptr)), \ 52 "+a" (o.low), "+d" (o.high) \ 53 : "b" (n.low), "c" (n.high) \ 54 : "memory"); \ 55 \ 56 if (unlikely(!ret)) \ 57 *(_oldp) = o.full; \ 58 \ 59 likely(ret); \ 60 }) 61 62 static __always_inline bool __try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new) 63 { 64 return __arch_try_cmpxchg64(ptr, oldp, new, LOCK_PREFIX); 65 } 66 67 static __always_inline bool __try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new) 68 { 69 return __arch_try_cmpxchg64(ptr, oldp, new,); 70 } 71 72 #ifdef CONFIG_X86_CMPXCHG64 73 74 #define arch_cmpxchg64 __cmpxchg64 75 76 #define arch_cmpxchg64_local __cmpxchg64_local 77 78 #define arch_try_cmpxchg64 __try_cmpxchg64 79 80 #define arch_try_cmpxchg64_local __try_cmpxchg64_local 81 82 #else 83 84 /* 85 * Building a kernel capable running on 80386 and 80486. It may be necessary 86 * to simulate the cmpxchg8b on the 80386 and 80486 CPU. 87 */ 88 89 #define __arch_cmpxchg64_emu(_ptr, _old, _new, _lock_loc, _lock) \ 90 ({ \ 91 union __u64_halves o = { .full = (_old), }, \ 92 n = { .full = (_new), }; \ 93 \ 94 asm volatile(ALTERNATIVE(_lock_loc \ 95 "call cmpxchg8b_emu", \ 96 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \ 97 : ALT_OUTPUT_SP("+a" (o.low), "+d" (o.high)) \ 98 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \ 99 : "memory"); \ 100 \ 101 o.full; \ 102 }) 103 104 static __always_inline u64 arch_cmpxchg64(volatile u64 *ptr, u64 old, u64 new) 105 { 106 return __arch_cmpxchg64_emu(ptr, old, new, LOCK_PREFIX_HERE, "lock; "); 107 } 108 #define arch_cmpxchg64 arch_cmpxchg64 109 110 static __always_inline u64 arch_cmpxchg64_local(volatile u64 *ptr, u64 old, u64 new) 111 { 112 return __arch_cmpxchg64_emu(ptr, old, new, ,); 113 } 114 #define arch_cmpxchg64_local arch_cmpxchg64_local 115 116 #define __arch_try_cmpxchg64_emu(_ptr, _oldp, _new, _lock_loc, _lock) \ 117 ({ \ 118 union __u64_halves o = { .full = *(_oldp), }, \ 119 n = { .full = (_new), }; \ 120 bool ret; \ 121 \ 122 asm volatile(ALTERNATIVE(_lock_loc \ 123 "call cmpxchg8b_emu", \ 124 _lock "cmpxchg8b %a[ptr]", X86_FEATURE_CX8) \ 125 CC_SET(e) \ 126 : ALT_OUTPUT_SP(CC_OUT(e) (ret), \ 127 "+a" (o.low), "+d" (o.high)) \ 128 : "b" (n.low), "c" (n.high), [ptr] "S" (_ptr) \ 129 : "memory"); \ 130 \ 131 if (unlikely(!ret)) \ 132 *(_oldp) = o.full; \ 133 \ 134 likely(ret); \ 135 }) 136 137 static __always_inline bool arch_try_cmpxchg64(volatile u64 *ptr, u64 *oldp, u64 new) 138 { 139 return __arch_try_cmpxchg64_emu(ptr, oldp, new, LOCK_PREFIX_HERE, "lock; "); 140 } 141 #define arch_try_cmpxchg64 arch_try_cmpxchg64 142 143 static __always_inline bool arch_try_cmpxchg64_local(volatile u64 *ptr, u64 *oldp, u64 new) 144 { 145 return __arch_try_cmpxchg64_emu(ptr, oldp, new, ,); 146 } 147 #define arch_try_cmpxchg64_local arch_try_cmpxchg64_local 148 149 #endif 150 151 #define system_has_cmpxchg64() boot_cpu_has(X86_FEATURE_CX8) 152 153 #endif /* _ASM_X86_CMPXCHG_32_H */ 154