1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_BITOPS_H 7 #define _ASM_RISCV_BITOPS_H 8 9 #ifndef _LINUX_BITOPS_H 10 #error "Only <linux/bitops.h> can be included directly" 11 #endif /* _LINUX_BITOPS_H */ 12 13 #include <linux/compiler.h> 14 #include <asm/barrier.h> 15 #include <asm/bitsperlong.h> 16 17 #if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) 18 #include <asm-generic/bitops/__ffs.h> 19 #include <asm-generic/bitops/__fls.h> 20 #include <asm-generic/bitops/ffs.h> 21 #include <asm-generic/bitops/fls.h> 22 23 #else 24 #define __HAVE_ARCH___FFS 25 #define __HAVE_ARCH___FLS 26 #define __HAVE_ARCH_FFS 27 #define __HAVE_ARCH_FLS 28 29 #include <asm-generic/bitops/__ffs.h> 30 #include <asm-generic/bitops/__fls.h> 31 #include <asm-generic/bitops/ffs.h> 32 #include <asm-generic/bitops/fls.h> 33 34 #include <asm/alternative-macros.h> 35 #include <asm/hwcap.h> 36 37 #if (BITS_PER_LONG == 64) 38 #define CTZW "ctzw " 39 #define CLZW "clzw " 40 #elif (BITS_PER_LONG == 32) 41 #define CTZW "ctz " 42 #define CLZW "clz " 43 #else 44 #error "Unexpected BITS_PER_LONG" 45 #endif 46 47 static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word) 48 { 49 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 50 return generic___ffs(word); 51 52 asm volatile (".option push\n" 53 ".option arch,+zbb\n" 54 "ctz %0, %1\n" 55 ".option pop\n" 56 : "=r" (word) : "r" (word) :); 57 58 return word; 59 } 60 61 /** 62 * __ffs - find first set bit in a long word 63 * @word: The word to search 64 * 65 * Undefined if no set bit exists, so code should check against 0 first. 66 */ 67 #define __ffs(word) \ 68 (__builtin_constant_p(word) ? \ 69 (unsigned long)__builtin_ctzl(word) : \ 70 variable__ffs(word)) 71 72 static __always_inline __attribute_const__ unsigned long variable__fls(unsigned long word) 73 { 74 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 75 return generic___fls(word); 76 77 asm volatile (".option push\n" 78 ".option arch,+zbb\n" 79 "clz %0, %1\n" 80 ".option pop\n" 81 : "=r" (word) : "r" (word) :); 82 83 return BITS_PER_LONG - 1 - word; 84 } 85 86 /** 87 * __fls - find last set bit in a long word 88 * @word: the word to search 89 * 90 * Undefined if no set bit exists, so code should check against 0 first. 91 */ 92 #define __fls(word) \ 93 (__builtin_constant_p(word) ? \ 94 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \ 95 variable__fls(word)) 96 97 static __always_inline __attribute_const__ int variable_ffs(int x) 98 { 99 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 100 return generic_ffs(x); 101 102 if (!x) 103 return 0; 104 105 asm volatile (".option push\n" 106 ".option arch,+zbb\n" 107 CTZW "%0, %1\n" 108 ".option pop\n" 109 : "=r" (x) : "r" (x) :); 110 111 return x + 1; 112 } 113 114 /** 115 * ffs - find first set bit in a word 116 * @x: the word to search 117 * 118 * This is defined the same way as the libc and compiler builtin ffs routines. 119 * 120 * ffs(value) returns 0 if value is 0 or the position of the first set bit if 121 * value is nonzero. The first (least significant) bit is at position 1. 122 */ 123 #define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x)) 124 125 static __always_inline int variable_fls(unsigned int x) 126 { 127 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 128 return generic_fls(x); 129 130 if (!x) 131 return 0; 132 133 asm volatile (".option push\n" 134 ".option arch,+zbb\n" 135 CLZW "%0, %1\n" 136 ".option pop\n" 137 : "=r" (x) : "r" (x) :); 138 139 return 32 - x; 140 } 141 142 /** 143 * fls - find last set bit in a word 144 * @x: the word to search 145 * 146 * This is defined in a similar way as ffs, but returns the position of the most 147 * significant set bit. 148 * 149 * fls(value) returns 0 if value is 0 or the position of the last set bit if 150 * value is nonzero. The last (most significant) bit is at position 32. 151 */ 152 #define fls(x) \ 153 ({ \ 154 typeof(x) x_ = (x); \ 155 __builtin_constant_p(x_) ? \ 156 ((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \ 157 : \ 158 variable_fls(x_); \ 159 }) 160 161 #endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */ 162 163 #include <asm-generic/bitops/ffz.h> 164 #include <asm-generic/bitops/fls64.h> 165 #include <asm-generic/bitops/sched.h> 166 167 #include <asm/arch_hweight.h> 168 169 #include <asm-generic/bitops/const_hweight.h> 170 171 #if (BITS_PER_LONG == 64) 172 #define __AMO(op) "amo" #op ".d" 173 #elif (BITS_PER_LONG == 32) 174 #define __AMO(op) "amo" #op ".w" 175 #else 176 #error "Unexpected BITS_PER_LONG" 177 #endif 178 179 #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \ 180 ({ \ 181 unsigned long __res, __mask; \ 182 __mask = BIT_MASK(nr); \ 183 __asm__ __volatile__ ( \ 184 __AMO(op) #ord " %0, %2, %1" \ 185 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \ 186 : "r" (mod(__mask)) \ 187 : "memory"); \ 188 ((__res & __mask) != 0); \ 189 }) 190 191 #define __op_bit_ord(op, mod, nr, addr, ord) \ 192 __asm__ __volatile__ ( \ 193 __AMO(op) #ord " zero, %1, %0" \ 194 : "+A" (addr[BIT_WORD(nr)]) \ 195 : "r" (mod(BIT_MASK(nr))) \ 196 : "memory"); 197 198 #define __test_and_op_bit(op, mod, nr, addr) \ 199 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl) 200 #define __op_bit(op, mod, nr, addr) \ 201 __op_bit_ord(op, mod, nr, addr, ) 202 203 /* Bitmask modifiers */ 204 #define __NOP(x) (x) 205 #define __NOT(x) (~(x)) 206 207 /** 208 * arch_test_and_set_bit - Set a bit and return its old value 209 * @nr: Bit to set 210 * @addr: Address to count from 211 * 212 * This is an atomic fully-ordered operation (implied full memory barrier). 213 */ 214 static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr) 215 { 216 return __test_and_op_bit(or, __NOP, nr, addr); 217 } 218 219 /** 220 * arch_test_and_clear_bit - Clear a bit and return its old value 221 * @nr: Bit to clear 222 * @addr: Address to count from 223 * 224 * This is an atomic fully-ordered operation (implied full memory barrier). 225 */ 226 static __always_inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr) 227 { 228 return __test_and_op_bit(and, __NOT, nr, addr); 229 } 230 231 /** 232 * arch_test_and_change_bit - Change a bit and return its old value 233 * @nr: Bit to change 234 * @addr: Address to count from 235 * 236 * This operation is atomic and cannot be reordered. 237 * It also implies a memory barrier. 238 */ 239 static __always_inline int arch_test_and_change_bit(int nr, volatile unsigned long *addr) 240 { 241 return __test_and_op_bit(xor, __NOP, nr, addr); 242 } 243 244 /** 245 * arch_set_bit - Atomically set a bit in memory 246 * @nr: the bit to set 247 * @addr: the address to start counting from 248 * 249 * Note: there are no guarantees that this function will not be reordered 250 * on non x86 architectures, so if you are writing portable code, 251 * make sure not to rely on its reordering guarantees. 252 * 253 * Note that @nr may be almost arbitrarily large; this function is not 254 * restricted to acting on a single-word quantity. 255 */ 256 static __always_inline void arch_set_bit(int nr, volatile unsigned long *addr) 257 { 258 __op_bit(or, __NOP, nr, addr); 259 } 260 261 /** 262 * arch_clear_bit - Clears a bit in memory 263 * @nr: Bit to clear 264 * @addr: Address to start counting from 265 * 266 * Note: there are no guarantees that this function will not be reordered 267 * on non x86 architectures, so if you are writing portable code, 268 * make sure not to rely on its reordering guarantees. 269 */ 270 static __always_inline void arch_clear_bit(int nr, volatile unsigned long *addr) 271 { 272 __op_bit(and, __NOT, nr, addr); 273 } 274 275 /** 276 * arch_change_bit - Toggle a bit in memory 277 * @nr: Bit to change 278 * @addr: Address to start counting from 279 * 280 * change_bit() may be reordered on other architectures than x86. 281 * Note that @nr may be almost arbitrarily large; this function is not 282 * restricted to acting on a single-word quantity. 283 */ 284 static __always_inline void arch_change_bit(int nr, volatile unsigned long *addr) 285 { 286 __op_bit(xor, __NOP, nr, addr); 287 } 288 289 /** 290 * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock 291 * @nr: Bit to set 292 * @addr: Address to count from 293 * 294 * This operation is atomic and provides acquire barrier semantics. 295 * It can be used to implement bit locks. 296 */ 297 static __always_inline int arch_test_and_set_bit_lock( 298 unsigned long nr, volatile unsigned long *addr) 299 { 300 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq); 301 } 302 303 /** 304 * arch_clear_bit_unlock - Clear a bit in memory, for unlock 305 * @nr: the bit to set 306 * @addr: the address to start counting from 307 * 308 * This operation is atomic and provides release barrier semantics. 309 */ 310 static __always_inline void arch_clear_bit_unlock( 311 unsigned long nr, volatile unsigned long *addr) 312 { 313 __op_bit_ord(and, __NOT, nr, addr, .rl); 314 } 315 316 /** 317 * arch___clear_bit_unlock - Clear a bit in memory, for unlock 318 * @nr: the bit to set 319 * @addr: the address to start counting from 320 * 321 * This operation is like clear_bit_unlock, however it is not atomic. 322 * It does provide release barrier semantics so it can be used to unlock 323 * a bit lock, however it would only be used if no other CPU can modify 324 * any bits in the memory until the lock is released (a good example is 325 * if the bit lock itself protects access to the other bits in the word). 326 * 327 * On RISC-V systems there seems to be no benefit to taking advantage of the 328 * non-atomic property here: it's a lot more instructions and we still have to 329 * provide release semantics anyway. 330 */ 331 static __always_inline void arch___clear_bit_unlock( 332 unsigned long nr, volatile unsigned long *addr) 333 { 334 arch_clear_bit_unlock(nr, addr); 335 } 336 337 static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask, 338 volatile unsigned long *addr) 339 { 340 unsigned long res; 341 __asm__ __volatile__ ( 342 __AMO(xor) ".rl %0, %2, %1" 343 : "=r" (res), "+A" (*addr) 344 : "r" (__NOP(mask)) 345 : "memory"); 346 return (res & BIT(7)) != 0; 347 } 348 349 #undef __test_and_op_bit 350 #undef __op_bit 351 #undef __NOP 352 #undef __NOT 353 #undef __AMO 354 355 #include <asm-generic/bitops/instrumented-atomic.h> 356 #include <asm-generic/bitops/instrumented-lock.h> 357 358 #include <asm-generic/bitops/non-atomic.h> 359 #include <asm-generic/bitops/le.h> 360 #include <asm-generic/bitops/ext2-atomic.h> 361 362 #endif /* _ASM_RISCV_BITOPS_H */ 363