1 /* SPDX-License-Identifier: GPL-2.0-only */ 2 /* 3 * Copyright (C) 2012 Regents of the University of California 4 */ 5 6 #ifndef _ASM_RISCV_BITOPS_H 7 #define _ASM_RISCV_BITOPS_H 8 9 #ifndef _LINUX_BITOPS_H 10 #error "Only <linux/bitops.h> can be included directly" 11 #endif /* _LINUX_BITOPS_H */ 12 13 #include <linux/compiler.h> 14 #include <linux/irqflags.h> 15 #include <asm/barrier.h> 16 #include <asm/bitsperlong.h> 17 18 #if !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) 19 #include <asm-generic/bitops/__ffs.h> 20 #include <asm-generic/bitops/__fls.h> 21 #include <asm-generic/bitops/ffs.h> 22 #include <asm-generic/bitops/fls.h> 23 24 #else 25 #define __HAVE_ARCH___FFS 26 #define __HAVE_ARCH___FLS 27 #define __HAVE_ARCH_FFS 28 #define __HAVE_ARCH_FLS 29 30 #include <asm-generic/bitops/__ffs.h> 31 #include <asm-generic/bitops/__fls.h> 32 #include <asm-generic/bitops/ffs.h> 33 #include <asm-generic/bitops/fls.h> 34 35 #include <asm/alternative-macros.h> 36 #include <asm/hwcap.h> 37 38 #if (BITS_PER_LONG == 64) 39 #define CTZW "ctzw " 40 #define CLZW "clzw " 41 #elif (BITS_PER_LONG == 32) 42 #define CTZW "ctz " 43 #define CLZW "clz " 44 #else 45 #error "Unexpected BITS_PER_LONG" 46 #endif 47 48 static __always_inline unsigned long variable__ffs(unsigned long word) 49 { 50 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 51 RISCV_ISA_EXT_ZBB, 1) 52 : : : : legacy); 53 54 asm volatile (".option push\n" 55 ".option arch,+zbb\n" 56 "ctz %0, %1\n" 57 ".option pop\n" 58 : "=r" (word) : "r" (word) :); 59 60 return word; 61 62 legacy: 63 return generic___ffs(word); 64 } 65 66 /** 67 * __ffs - find first set bit in a long word 68 * @word: The word to search 69 * 70 * Undefined if no set bit exists, so code should check against 0 first. 71 */ 72 #define __ffs(word) \ 73 (__builtin_constant_p(word) ? \ 74 (unsigned long)__builtin_ctzl(word) : \ 75 variable__ffs(word)) 76 77 static __always_inline unsigned long variable__fls(unsigned long word) 78 { 79 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 80 RISCV_ISA_EXT_ZBB, 1) 81 : : : : legacy); 82 83 asm volatile (".option push\n" 84 ".option arch,+zbb\n" 85 "clz %0, %1\n" 86 ".option pop\n" 87 : "=r" (word) : "r" (word) :); 88 89 return BITS_PER_LONG - 1 - word; 90 91 legacy: 92 return generic___fls(word); 93 } 94 95 /** 96 * __fls - find last set bit in a long word 97 * @word: the word to search 98 * 99 * Undefined if no set bit exists, so code should check against 0 first. 100 */ 101 #define __fls(word) \ 102 (__builtin_constant_p(word) ? \ 103 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \ 104 variable__fls(word)) 105 106 static __always_inline int variable_ffs(int x) 107 { 108 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 109 RISCV_ISA_EXT_ZBB, 1) 110 : : : : legacy); 111 112 if (!x) 113 return 0; 114 115 asm volatile (".option push\n" 116 ".option arch,+zbb\n" 117 CTZW "%0, %1\n" 118 ".option pop\n" 119 : "=r" (x) : "r" (x) :); 120 121 return x + 1; 122 123 legacy: 124 return generic_ffs(x); 125 } 126 127 /** 128 * ffs - find first set bit in a word 129 * @x: the word to search 130 * 131 * This is defined the same way as the libc and compiler builtin ffs routines. 132 * 133 * ffs(value) returns 0 if value is 0 or the position of the first set bit if 134 * value is nonzero. The first (least significant) bit is at position 1. 135 */ 136 #define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x)) 137 138 static __always_inline int variable_fls(unsigned int x) 139 { 140 asm goto(ALTERNATIVE("j %l[legacy]", "nop", 0, 141 RISCV_ISA_EXT_ZBB, 1) 142 : : : : legacy); 143 144 if (!x) 145 return 0; 146 147 asm volatile (".option push\n" 148 ".option arch,+zbb\n" 149 CLZW "%0, %1\n" 150 ".option pop\n" 151 : "=r" (x) : "r" (x) :); 152 153 return 32 - x; 154 155 legacy: 156 return generic_fls(x); 157 } 158 159 /** 160 * fls - find last set bit in a word 161 * @x: the word to search 162 * 163 * This is defined in a similar way as ffs, but returns the position of the most 164 * significant set bit. 165 * 166 * fls(value) returns 0 if value is 0 or the position of the last set bit if 167 * value is nonzero. The last (most significant) bit is at position 32. 168 */ 169 #define fls(x) \ 170 ({ \ 171 typeof(x) x_ = (x); \ 172 __builtin_constant_p(x_) ? \ 173 (int)((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \ 174 : \ 175 variable_fls(x_); \ 176 }) 177 178 #endif /* !defined(CONFIG_RISCV_ISA_ZBB) || defined(NO_ALTERNATIVE) */ 179 180 #include <asm-generic/bitops/ffz.h> 181 #include <asm-generic/bitops/fls64.h> 182 #include <asm-generic/bitops/sched.h> 183 184 #include <asm/arch_hweight.h> 185 186 #include <asm-generic/bitops/const_hweight.h> 187 188 #if (BITS_PER_LONG == 64) 189 #define __AMO(op) "amo" #op ".d" 190 #elif (BITS_PER_LONG == 32) 191 #define __AMO(op) "amo" #op ".w" 192 #else 193 #error "Unexpected BITS_PER_LONG" 194 #endif 195 196 #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \ 197 ({ \ 198 unsigned long __res, __mask; \ 199 __mask = BIT_MASK(nr); \ 200 __asm__ __volatile__ ( \ 201 __AMO(op) #ord " %0, %2, %1" \ 202 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \ 203 : "r" (mod(__mask)) \ 204 : "memory"); \ 205 ((__res & __mask) != 0); \ 206 }) 207 208 #define __op_bit_ord(op, mod, nr, addr, ord) \ 209 __asm__ __volatile__ ( \ 210 __AMO(op) #ord " zero, %1, %0" \ 211 : "+A" (addr[BIT_WORD(nr)]) \ 212 : "r" (mod(BIT_MASK(nr))) \ 213 : "memory"); 214 215 #define __test_and_op_bit(op, mod, nr, addr) \ 216 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl) 217 #define __op_bit(op, mod, nr, addr) \ 218 __op_bit_ord(op, mod, nr, addr, ) 219 220 /* Bitmask modifiers */ 221 #define __NOP(x) (x) 222 #define __NOT(x) (~(x)) 223 224 /** 225 * test_and_set_bit - Set a bit and return its old value 226 * @nr: Bit to set 227 * @addr: Address to count from 228 * 229 * This operation may be reordered on other architectures than x86. 230 */ 231 static inline int test_and_set_bit(int nr, volatile unsigned long *addr) 232 { 233 return __test_and_op_bit(or, __NOP, nr, addr); 234 } 235 236 /** 237 * test_and_clear_bit - Clear a bit and return its old value 238 * @nr: Bit to clear 239 * @addr: Address to count from 240 * 241 * This operation can be reordered on other architectures other than x86. 242 */ 243 static inline int test_and_clear_bit(int nr, volatile unsigned long *addr) 244 { 245 return __test_and_op_bit(and, __NOT, nr, addr); 246 } 247 248 /** 249 * test_and_change_bit - Change a bit and return its old value 250 * @nr: Bit to change 251 * @addr: Address to count from 252 * 253 * This operation is atomic and cannot be reordered. 254 * It also implies a memory barrier. 255 */ 256 static inline int test_and_change_bit(int nr, volatile unsigned long *addr) 257 { 258 return __test_and_op_bit(xor, __NOP, nr, addr); 259 } 260 261 /** 262 * set_bit - Atomically set a bit in memory 263 * @nr: the bit to set 264 * @addr: the address to start counting from 265 * 266 * Note: there are no guarantees that this function will not be reordered 267 * on non x86 architectures, so if you are writing portable code, 268 * make sure not to rely on its reordering guarantees. 269 * 270 * Note that @nr may be almost arbitrarily large; this function is not 271 * restricted to acting on a single-word quantity. 272 */ 273 static inline void set_bit(int nr, volatile unsigned long *addr) 274 { 275 __op_bit(or, __NOP, nr, addr); 276 } 277 278 /** 279 * clear_bit - Clears a bit in memory 280 * @nr: Bit to clear 281 * @addr: Address to start counting from 282 * 283 * Note: there are no guarantees that this function will not be reordered 284 * on non x86 architectures, so if you are writing portable code, 285 * make sure not to rely on its reordering guarantees. 286 */ 287 static inline void clear_bit(int nr, volatile unsigned long *addr) 288 { 289 __op_bit(and, __NOT, nr, addr); 290 } 291 292 /** 293 * change_bit - Toggle a bit in memory 294 * @nr: Bit to change 295 * @addr: Address to start counting from 296 * 297 * change_bit() may be reordered on other architectures than x86. 298 * Note that @nr may be almost arbitrarily large; this function is not 299 * restricted to acting on a single-word quantity. 300 */ 301 static inline void change_bit(int nr, volatile unsigned long *addr) 302 { 303 __op_bit(xor, __NOP, nr, addr); 304 } 305 306 /** 307 * test_and_set_bit_lock - Set a bit and return its old value, for lock 308 * @nr: Bit to set 309 * @addr: Address to count from 310 * 311 * This operation is atomic and provides acquire barrier semantics. 312 * It can be used to implement bit locks. 313 */ 314 static inline int test_and_set_bit_lock( 315 unsigned long nr, volatile unsigned long *addr) 316 { 317 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq); 318 } 319 320 /** 321 * clear_bit_unlock - Clear a bit in memory, for unlock 322 * @nr: the bit to set 323 * @addr: the address to start counting from 324 * 325 * This operation is atomic and provides release barrier semantics. 326 */ 327 static inline void clear_bit_unlock( 328 unsigned long nr, volatile unsigned long *addr) 329 { 330 __op_bit_ord(and, __NOT, nr, addr, .rl); 331 } 332 333 /** 334 * __clear_bit_unlock - Clear a bit in memory, for unlock 335 * @nr: the bit to set 336 * @addr: the address to start counting from 337 * 338 * This operation is like clear_bit_unlock, however it is not atomic. 339 * It does provide release barrier semantics so it can be used to unlock 340 * a bit lock, however it would only be used if no other CPU can modify 341 * any bits in the memory until the lock is released (a good example is 342 * if the bit lock itself protects access to the other bits in the word). 343 * 344 * On RISC-V systems there seems to be no benefit to taking advantage of the 345 * non-atomic property here: it's a lot more instructions and we still have to 346 * provide release semantics anyway. 347 */ 348 static inline void __clear_bit_unlock( 349 unsigned long nr, volatile unsigned long *addr) 350 { 351 clear_bit_unlock(nr, addr); 352 } 353 354 static inline bool xor_unlock_is_negative_byte(unsigned long mask, 355 volatile unsigned long *addr) 356 { 357 unsigned long res; 358 __asm__ __volatile__ ( 359 __AMO(xor) ".rl %0, %2, %1" 360 : "=r" (res), "+A" (*addr) 361 : "r" (__NOP(mask)) 362 : "memory"); 363 return (res & BIT(7)) != 0; 364 } 365 366 #undef __test_and_op_bit 367 #undef __op_bit 368 #undef __NOP 369 #undef __NOT 370 #undef __AMO 371 372 #include <asm-generic/bitops/non-atomic.h> 373 #include <asm-generic/bitops/le.h> 374 #include <asm-generic/bitops/ext2-atomic.h> 375 376 #endif /* _ASM_RISCV_BITOPS_H */ 377