1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_BITOPS_H 7 #define _ASM_RISCV_BITOPS_H 8 9 #ifndef _LINUX_BITOPS_H 10 #error "Only <linux/bitops.h> can be included directly" 11 #endif /* _LINUX_BITOPS_H */ 12 13 #include <linux/compiler.h> 14 #include <linux/irqflags.h> 15 #include <asm/barrier.h> 16 #include <asm/bitsperlong.h> 17 18 #if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) 19 #include <asm-generic/bitops/__ffs.h> 20 #include <asm-generic/bitops/__fls.h> 21 #include <asm-generic/bitops/ffs.h> 22 #include <asm-generic/bitops/fls.h> 23 24 #else 25 #define __HAVE_ARCH___FFS 26 #define __HAVE_ARCH___FLS 27 #define __HAVE_ARCH_FFS 28 #define __HAVE_ARCH_FLS 29 30 #include <asm-generic/bitops/__ffs.h> 31 #include <asm-generic/bitops/__fls.h> 32 #include <asm-generic/bitops/ffs.h> 33 #include <asm-generic/bitops/fls.h> 34 35 #include <asm/alternative-macros.h> 36 #include <asm/hwcap.h> 37 38 #if (BITS_PER_LONG == 64) 39 #define CTZW "ctzw " 40 #define CLZW "clzw " 41 #elif (BITS_PER_LONG == 32) 42 #define CTZW "ctz " 43 #define CLZW "clz " 44 #else 45 #error "Unexpected BITS_PER_LONG" 46 #endif 47 48 static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word) 49 { 50 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 51 return generic___ffs(word); 52 53 asm volatile (".option push\n" 54 ".option arch,+zbb\n" 55 "ctz %0, %1\n" 56 ".option pop\n" 57 : "=r" (word) : "r" (word) :); 58 59 return word; 60 } 61 62 /** 63 * __ffs - find first set bit in a long word 64 * @word: The word to search 65 * 66 * Undefined if no set bit exists, so code should check against 0 first. 67 */ 68 #define __ffs(word) \ 69 (__builtin_constant_p(word) ? \ 70 (unsigned long)__builtin_ctzl(word) : \ 71 variable__ffs(word)) 72 73 static __always_inline __attribute_const__ unsigned long variable__fls(unsigned long word) 74 { 75 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 76 return generic___fls(word); 77 78 asm volatile (".option push\n" 79 ".option arch,+zbb\n" 80 "clz %0, %1\n" 81 ".option pop\n" 82 : "=r" (word) : "r" (word) :); 83 84 return BITS_PER_LONG - 1 - word; 85 } 86 87 /** 88 * __fls - find last set bit in a long word 89 * @word: the word to search 90 * 91 * Undefined if no set bit exists, so code should check against 0 first. 92 */ 93 #define __fls(word) \ 94 (__builtin_constant_p(word) ? \ 95 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \ 96 variable__fls(word)) 97 98 static __always_inline __attribute_const__ int variable_ffs(int x) 99 { 100 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 101 return generic_ffs(x); 102 103 if (!x) 104 return 0; 105 106 asm volatile (".option push\n" 107 ".option arch,+zbb\n" 108 CTZW "%0, %1\n" 109 ".option pop\n" 110 : "=r" (x) : "r" (x) :); 111 112 return x + 1; 113 } 114 115 /** 116 * ffs - find first set bit in a word 117 * @x: the word to search 118 * 119 * This is defined the same way as the libc and compiler builtin ffs routines. 120 * 121 * ffs(value) returns 0 if value is 0 or the position of the first set bit if 122 * value is nonzero. The first (least significant) bit is at position 1. 123 */ 124 #define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x)) 125 126 static __always_inline int variable_fls(unsigned int x) 127 { 128 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB)) 129 return generic_fls(x); 130 131 if (!x) 132 return 0; 133 134 asm volatile (".option push\n" 135 ".option arch,+zbb\n" 136 CLZW "%0, %1\n" 137 ".option pop\n" 138 : "=r" (x) : "r" (x) :); 139 140 return 32 - x; 141 } 142 143 /** 144 * fls - find last set bit in a word 145 * @x: the word to search 146 * 147 * This is defined in a similar way as ffs, but returns the position of the most 148 * significant set bit. 149 * 150 * fls(value) returns 0 if value is 0 or the position of the last set bit if 151 * value is nonzero. The last (most significant) bit is at position 32. 152 */ 153 #define fls(x) \ 154 ({ \ 155 typeof(x) x_ = (x); \ 156 __builtin_constant_p(x_) ? \ 157 ((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \ 158 : \ 159 variable_fls(x_); \ 160 }) 161 162 #endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */ 163 164 #include <asm-generic/bitops/ffz.h> 165 #include <asm-generic/bitops/fls64.h> 166 #include <asm-generic/bitops/sched.h> 167 168 #include <asm/arch_hweight.h> 169 170 #include <asm-generic/bitops/const_hweight.h> 171 172 #if (BITS_PER_LONG == 64) 173 #define __AMO(op) "amo" #op ".d" 174 #elif (BITS_PER_LONG == 32) 175 #define __AMO(op) "amo" #op ".w" 176 #else 177 #error "Unexpected BITS_PER_LONG" 178 #endif 179 180 #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \ 181 ({ \ 182 unsigned long __res, __mask; \ 183 __mask = BIT_MASK(nr); \ 184 __asm__ __volatile__ ( \ 185 __AMO(op) #ord " %0, %2, %1" \ 186 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \ 187 : "r" (mod(__mask)) \ 188 : "memory"); \ 189 ((__res & __mask) != 0); \ 190 }) 191 192 #define __op_bit_ord(op, mod, nr, addr, ord) \ 193 __asm__ __volatile__ ( \ 194 __AMO(op) #ord " zero, %1, %0" \ 195 : "+A" (addr[BIT_WORD(nr)]) \ 196 : "r" (mod(BIT_MASK(nr))) \ 197 : "memory"); 198 199 #define __test_and_op_bit(op, mod, nr, addr) \ 200 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl) 201 #define __op_bit(op, mod, nr, addr) \ 202 __op_bit_ord(op, mod, nr, addr, ) 203 204 /* Bitmask modifiers */ 205 #define __NOP(x) (x) 206 #define __NOT(x) (~(x)) 207 208 /** 209 * arch_test_and_set_bit - Set a bit and return its old value 210 * @nr: Bit to set 211 * @addr: Address to count from 212 * 213 * This is an atomic fully-ordered operation (implied full memory barrier). 214 */ 215 static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr) 216 { 217 return __test_and_op_bit(or, __NOP, nr, addr); 218 } 219 220 /** 221 * arch_test_and_clear_bit - Clear a bit and return its old value 222 * @nr: Bit to clear 223 * @addr: Address to count from 224 * 225 * This is an atomic fully-ordered operation (implied full memory barrier). 226 */ 227 static __always_inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr) 228 { 229 return __test_and_op_bit(and, __NOT, nr, addr); 230 } 231 232 /** 233 * arch_test_and_change_bit - Change a bit and return its old value 234 * @nr: Bit to change 235 * @addr: Address to count from 236 * 237 * This operation is atomic and cannot be reordered. 238 * It also implies a memory barrier. 239 */ 240 static __always_inline int arch_test_and_change_bit(int nr, volatile unsigned long *addr) 241 { 242 return __test_and_op_bit(xor, __NOP, nr, addr); 243 } 244 245 /** 246 * arch_set_bit - Atomically set a bit in memory 247 * @nr: the bit to set 248 * @addr: the address to start counting from 249 * 250 * Note: there are no guarantees that this function will not be reordered 251 * on non x86 architectures, so if you are writing portable code, 252 * make sure not to rely on its reordering guarantees. 253 * 254 * Note that @nr may be almost arbitrarily large; this function is not 255 * restricted to acting on a single-word quantity. 256 */ 257 static __always_inline void arch_set_bit(int nr, volatile unsigned long *addr) 258 { 259 __op_bit(or, __NOP, nr, addr); 260 } 261 262 /** 263 * arch_clear_bit - Clears a bit in memory 264 * @nr: Bit to clear 265 * @addr: Address to start counting from 266 * 267 * Note: there are no guarantees that this function will not be reordered 268 * on non x86 architectures, so if you are writing portable code, 269 * make sure not to rely on its reordering guarantees. 270 */ 271 static __always_inline void arch_clear_bit(int nr, volatile unsigned long *addr) 272 { 273 __op_bit(and, __NOT, nr, addr); 274 } 275 276 /** 277 * arch_change_bit - Toggle a bit in memory 278 * @nr: Bit to change 279 * @addr: Address to start counting from 280 * 281 * change_bit() may be reordered on other architectures than x86. 282 * Note that @nr may be almost arbitrarily large; this function is not 283 * restricted to acting on a single-word quantity. 284 */ 285 static __always_inline void arch_change_bit(int nr, volatile unsigned long *addr) 286 { 287 __op_bit(xor, __NOP, nr, addr); 288 } 289 290 /** 291 * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock 292 * @nr: Bit to set 293 * @addr: Address to count from 294 * 295 * This operation is atomic and provides acquire barrier semantics. 296 * It can be used to implement bit locks. 297 */ 298 static __always_inline int arch_test_and_set_bit_lock( 299 unsigned long nr, volatile unsigned long *addr) 300 { 301 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq); 302 } 303 304 /** 305 * arch_clear_bit_unlock - Clear a bit in memory, for unlock 306 * @nr: the bit to set 307 * @addr: the address to start counting from 308 * 309 * This operation is atomic and provides release barrier semantics. 310 */ 311 static __always_inline void arch_clear_bit_unlock( 312 unsigned long nr, volatile unsigned long *addr) 313 { 314 __op_bit_ord(and, __NOT, nr, addr, .rl); 315 } 316 317 /** 318 * arch___clear_bit_unlock - Clear a bit in memory, for unlock 319 * @nr: the bit to set 320 * @addr: the address to start counting from 321 * 322 * This operation is like clear_bit_unlock, however it is not atomic. 323 * It does provide release barrier semantics so it can be used to unlock 324 * a bit lock, however it would only be used if no other CPU can modify 325 * any bits in the memory until the lock is released (a good example is 326 * if the bit lock itself protects access to the other bits in the word). 327 * 328 * On RISC-V systems there seems to be no benefit to taking advantage of the 329 * non-atomic property here: it's a lot more instructions and we still have to 330 * provide release semantics anyway. 331 */ 332 static __always_inline void arch___clear_bit_unlock( 333 unsigned long nr, volatile unsigned long *addr) 334 { 335 arch_clear_bit_unlock(nr, addr); 336 } 337 338 static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask, 339 volatile unsigned long *addr) 340 { 341 unsigned long res; 342 __asm__ __volatile__ ( 343 __AMO(xor) ".rl %0, %2, %1" 344 : "=r" (res), "+A" (*addr) 345 : "r" (__NOP(mask)) 346 : "memory"); 347 return (res & BIT(7)) != 0; 348 } 349 350 #undef __test_and_op_bit 351 #undef __op_bit 352 #undef __NOP 353 #undef __NOT 354 #undef __AMO 355 356 #include <asm-generic/bitops/instrumented-atomic.h> 357 #include <asm-generic/bitops/instrumented-lock.h> 358 359 #include <asm-generic/bitops/non-atomic.h> 360 #include <asm-generic/bitops/le.h> 361 #include <asm-generic/bitops/ext2-atomic.h> 362 363 #endif /* _ASM_RISCV_BITOPS_H */ 364