1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited 4 */ 5 #ifndef __ASM_CMPXCHG_H 6 #define __ASM_CMPXCHG_H 7 8 #include <linux/bits.h> 9 #include <linux/build_bug.h> 10 #include <asm/barrier.h> 11 #include <asm/cpu-features.h> 12 13 #define __xchg_amo_asm(amswap_db, m, val) \ 14 ({ \ 15 __typeof(val) __ret; \ 16 \ 17 __asm__ __volatile__ ( \ 18 " "amswap_db" %1, %z2, %0 \n" \ 19 : "+ZB" (*m), "=&r" (__ret) \ 20 : "Jr" (val) \ 21 : "memory"); \ 22 \ 23 __ret; \ 24 }) 25 26 #define __xchg_llsc_asm(ld, st, m, val) \ 27 ({ \ 28 __typeof(val) __ret, __tmp; \ 29 \ 30 asm volatile ( \ 31 "1: ll.w %0, %3 \n" \ 32 " move %1, %z4 \n" \ 33 " sc.w %1, %2 \n" \ 34 " beqz %1, 1b \n" \ 35 : "=&r" (__ret), "=&r" (__tmp), "=ZC" (*m) \ 36 : "ZC" (*m), "Jr" (val) \ 37 : "memory"); \ 38 \ 39 __ret; \ 40 }) 41 42 static inline unsigned int __xchg_small(volatile void *ptr, unsigned int val, 43 unsigned int size) 44 { 45 unsigned int shift; 46 u32 old32, mask, temp; 47 volatile u32 *ptr32; 48 49 /* Mask value to the correct size. */ 50 mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); 51 val &= mask; 52 53 /* 54 * Calculate a shift & mask that correspond to the value we wish to 55 * exchange within the naturally aligned 4 byte integerthat includes 56 * it. 57 */ 58 shift = (unsigned long)ptr & 0x3; 59 shift *= BITS_PER_BYTE; 60 mask <<= shift; 61 62 /* 63 * Calculate a pointer to the naturally aligned 4 byte integer that 64 * includes our byte of interest, and load its value. 65 */ 66 ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); 67 68 asm volatile ( 69 "1: ll.w %0, %3 \n" 70 " andn %1, %0, %z4 \n" 71 " or %1, %1, %z5 \n" 72 " sc.w %1, %2 \n" 73 " beqz %1, 1b \n" 74 : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) 75 : "ZC" (*ptr32), "Jr" (mask), "Jr" (val << shift) 76 : "memory"); 77 78 return (old32 & mask) >> shift; 79 } 80 81 static __always_inline unsigned long 82 __arch_xchg(volatile void *ptr, unsigned long x, int size) 83 { 84 switch (size) { 85 case 1: 86 case 2: 87 return __xchg_small((volatile void *)ptr, x, size); 88 89 case 4: 90 #ifdef CONFIG_CPU_HAS_AMO 91 return __xchg_amo_asm("amswap_db.w", (volatile u32 *)ptr, (u32)x); 92 #else 93 return __xchg_llsc_asm("ll.w", "sc.w", (volatile u32 *)ptr, (u32)x); 94 #endif /* CONFIG_CPU_HAS_AMO */ 95 96 #ifdef CONFIG_64BIT 97 case 8: 98 #ifdef CONFIG_CPU_HAS_AMO 99 return __xchg_amo_asm("amswap_db.d", (volatile u64 *)ptr, (u64)x); 100 #else 101 return __xchg_llsc_asm("ll.d", "sc.d", (volatile u64 *)ptr, (u64)x); 102 #endif /* CONFIG_CPU_HAS_AMO */ 103 #endif /* CONFIG_64BIT */ 104 105 default: 106 BUILD_BUG(); 107 } 108 109 return 0; 110 } 111 112 #define arch_xchg(ptr, x) \ 113 ({ \ 114 __typeof__(*(ptr)) __res; \ 115 \ 116 __res = (__typeof__(*(ptr))) \ 117 __arch_xchg((ptr), (unsigned long)(x), sizeof(*(ptr))); \ 118 \ 119 __res; \ 120 }) 121 122 #define __cmpxchg_asm(ld, st, m, old, new) \ 123 ({ \ 124 __typeof(old) __ret; \ 125 \ 126 __asm__ __volatile__( \ 127 "1: " ld " %0, %2 # __cmpxchg_asm \n" \ 128 " bne %0, %z3, 2f \n" \ 129 " move $t0, %z4 \n" \ 130 " " st " $t0, %1 \n" \ 131 " beqz $t0, 1b \n" \ 132 "2: \n" \ 133 __WEAK_LLSC_MB \ 134 : "=&r" (__ret), "=ZB"(*m) \ 135 : "ZB"(*m), "Jr" (old), "Jr" (new) \ 136 : "t0", "memory"); \ 137 \ 138 __ret; \ 139 }) 140 141 static inline unsigned int __cmpxchg_small(volatile void *ptr, unsigned int old, 142 unsigned int new, unsigned int size) 143 { 144 unsigned int shift; 145 u32 old32, mask, temp; 146 volatile u32 *ptr32; 147 148 /* Mask inputs to the correct size. */ 149 mask = GENMASK((size * BITS_PER_BYTE) - 1, 0); 150 old &= mask; 151 new &= mask; 152 153 /* 154 * Calculate a shift & mask that correspond to the value we wish to 155 * compare & exchange within the naturally aligned 4 byte integer 156 * that includes it. 157 */ 158 shift = (unsigned long)ptr & 0x3; 159 shift *= BITS_PER_BYTE; 160 old <<= shift; 161 new <<= shift; 162 mask <<= shift; 163 164 /* 165 * Calculate a pointer to the naturally aligned 4 byte integer that 166 * includes our byte of interest, and load its value. 167 */ 168 ptr32 = (volatile u32 *)((unsigned long)ptr & ~0x3); 169 170 asm volatile ( 171 "1: ll.w %0, %3 \n" 172 " and %1, %0, %z4 \n" 173 " bne %1, %z5, 2f \n" 174 " andn %1, %0, %z4 \n" 175 " or %1, %1, %z6 \n" 176 " sc.w %1, %2 \n" 177 " beqz %1, 1b \n" 178 " b 3f \n" 179 "2: \n" 180 __WEAK_LLSC_MB 181 "3: \n" 182 : "=&r" (old32), "=&r" (temp), "=ZC" (*ptr32) 183 : "ZC" (*ptr32), "Jr" (mask), "Jr" (old), "Jr" (new) 184 : "memory"); 185 186 return (old32 & mask) >> shift; 187 } 188 189 static __always_inline unsigned long 190 __cmpxchg(volatile void *ptr, unsigned long old, unsigned long new, unsigned int size) 191 { 192 switch (size) { 193 case 1: 194 case 2: 195 return __cmpxchg_small(ptr, old, new, size); 196 197 case 4: 198 return __cmpxchg_asm("ll.w", "sc.w", (volatile u32 *)ptr, 199 (u32)old, new); 200 201 case 8: 202 return __cmpxchg_asm("ll.d", "sc.d", (volatile u64 *)ptr, 203 (u64)old, new); 204 205 default: 206 BUILD_BUG(); 207 } 208 209 return 0; 210 } 211 212 #define arch_cmpxchg_local(ptr, old, new) \ 213 ((__typeof__(*(ptr))) \ 214 __cmpxchg((ptr), \ 215 (unsigned long)(__typeof__(*(ptr)))(old), \ 216 (unsigned long)(__typeof__(*(ptr)))(new), \ 217 sizeof(*(ptr)))) 218 219 #define arch_cmpxchg(ptr, old, new) \ 220 ({ \ 221 __typeof__(*(ptr)) __res; \ 222 \ 223 __res = arch_cmpxchg_local((ptr), (old), (new)); \ 224 \ 225 __res; \ 226 }) 227 228 #ifdef CONFIG_64BIT 229 #define arch_cmpxchg64_local(ptr, o, n) \ 230 ({ \ 231 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 232 arch_cmpxchg_local((ptr), (o), (n)); \ 233 }) 234 235 #define arch_cmpxchg64(ptr, o, n) \ 236 ({ \ 237 BUILD_BUG_ON(sizeof(*(ptr)) != 8); \ 238 arch_cmpxchg((ptr), (o), (n)); \ 239 }) 240 241 union __u128_halves { 242 u128 full; 243 struct { 244 u64 low; 245 u64 high; 246 }; 247 }; 248 249 #define system_has_cmpxchg128() cpu_opt(LOONGARCH_CPU_SCQ) 250 251 #define __arch_cmpxchg128(ptr, old, new, llsc_mb) \ 252 ({ \ 253 union __u128_halves __old, __new, __ret; \ 254 volatile u64 *__ptr = (volatile u64 *)(ptr); \ 255 \ 256 __old.full = (old); \ 257 __new.full = (new); \ 258 \ 259 __asm__ __volatile__( \ 260 "1: ll.d %0, %3 # 128-bit cmpxchg low \n" \ 261 llsc_mb \ 262 " ld.d %1, %4 # 128-bit cmpxchg high \n" \ 263 " move $t0, %0 \n" \ 264 " move $t1, %1 \n" \ 265 " bne %0, %z5, 2f \n" \ 266 " bne %1, %z6, 2f \n" \ 267 " move $t0, %z7 \n" \ 268 " move $t1, %z8 \n" \ 269 "2: sc.q $t0, $t1, %2 \n" \ 270 " beqz $t0, 1b \n" \ 271 llsc_mb \ 272 : "=&r" (__ret.low), "=&r" (__ret.high) \ 273 : "r" (__ptr), \ 274 "ZC" (__ptr[0]), "m" (__ptr[1]), \ 275 "Jr" (__old.low), "Jr" (__old.high), \ 276 "Jr" (__new.low), "Jr" (__new.high) \ 277 : "t0", "t1", "memory"); \ 278 \ 279 __ret.full; \ 280 }) 281 282 #define arch_cmpxchg128(ptr, o, n) \ 283 ({ \ 284 BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ 285 __arch_cmpxchg128(ptr, o, n, __WEAK_LLSC_MB); \ 286 }) 287 288 #define arch_cmpxchg128_local(ptr, o, n) \ 289 ({ \ 290 BUILD_BUG_ON(sizeof(*(ptr)) != 16); \ 291 __arch_cmpxchg128(ptr, o, n, ""); \ 292 }) 293 #else 294 #include <asm-generic/cmpxchg-local.h> 295 #define arch_cmpxchg64_local(ptr, o, n) __generic_cmpxchg64_local((ptr), (o), (n)) 296 #define arch_cmpxchg64(ptr, o, n) arch_cmpxchg64_local((ptr), (o), (n)) 297 #endif 298 299 #endif /* __ASM_CMPXCHG_H */ 300