1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (C) 2012 Regents of the University of California
4 */
5
6 #ifndef _ASM_RISCV_BITOPS_H
7 #define _ASM_RISCV_BITOPS_H
8
9 #ifndef _LINUX_BITOPS_H
10 #error "Only <linux/bitops.h> can be included directly"
11 #endif /* _LINUX_BITOPS_H */
12
13 #include <linux/compiler.h>
14 #include <linux/irqflags.h>
15 #include <asm/barrier.h>
16 #include <asm/bitsperlong.h>
17
18 #if !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE)
19 #include <asm-generic/bitops/__ffs.h>
20 #include <asm-generic/bitops/__fls.h>
21 #include <asm-generic/bitops/ffs.h>
22 #include <asm-generic/bitops/fls.h>
23
24 #else
25 #define __HAVE_ARCH___FFS
26 #define __HAVE_ARCH___FLS
27 #define __HAVE_ARCH_FFS
28 #define __HAVE_ARCH_FLS
29
30 #include <asm-generic/bitops/__ffs.h>
31 #include <asm-generic/bitops/__fls.h>
32 #include <asm-generic/bitops/ffs.h>
33 #include <asm-generic/bitops/fls.h>
34
35 #include <asm/alternative-macros.h>
36 #include <asm/hwcap.h>
37
38 #if (BITS_PER_LONG == 64)
39 #define CTZW "ctzw "
40 #define CLZW "clzw "
41 #elif (BITS_PER_LONG == 32)
42 #define CTZW "ctz "
43 #define CLZW "clz "
44 #else
45 #error "Unexpected BITS_PER_LONG"
46 #endif
47
variable__ffs(unsigned long word)48 static __always_inline __attribute_const__ unsigned long variable__ffs(unsigned long word)
49 {
50 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
51 return generic___ffs(word);
52
53 asm volatile (".option push\n"
54 ".option arch,+zbb\n"
55 "ctz %0, %1\n"
56 ".option pop\n"
57 : "=r" (word) : "r" (word) :);
58
59 return word;
60 }
61
62 /**
63 * __ffs - find first set bit in a long word
64 * @word: The word to search
65 *
66 * Undefined if no set bit exists, so code should check against 0 first.
67 */
68 #define __ffs(word) \
69 (__builtin_constant_p(word) ? \
70 (unsigned long)__builtin_ctzl(word) : \
71 variable__ffs(word))
72
variable__fls(unsigned long word)73 static __always_inline __attribute_const__ unsigned long variable__fls(unsigned long word)
74 {
75 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
76 return generic___fls(word);
77
78 asm volatile (".option push\n"
79 ".option arch,+zbb\n"
80 "clz %0, %1\n"
81 ".option pop\n"
82 : "=r" (word) : "r" (word) :);
83
84 return BITS_PER_LONG - 1 - word;
85 }
86
87 /**
88 * __fls - find last set bit in a long word
89 * @word: the word to search
90 *
91 * Undefined if no set bit exists, so code should check against 0 first.
92 */
93 #define __fls(word) \
94 (__builtin_constant_p(word) ? \
95 (unsigned long)(BITS_PER_LONG - 1 - __builtin_clzl(word)) : \
96 variable__fls(word))
97
variable_ffs(int x)98 static __always_inline __attribute_const__ int variable_ffs(int x)
99 {
100 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
101 return generic_ffs(x);
102
103 if (!x)
104 return 0;
105
106 asm volatile (".option push\n"
107 ".option arch,+zbb\n"
108 CTZW "%0, %1\n"
109 ".option pop\n"
110 : "=r" (x) : "r" (x) :);
111
112 return x + 1;
113 }
114
115 /**
116 * ffs - find first set bit in a word
117 * @x: the word to search
118 *
119 * This is defined the same way as the libc and compiler builtin ffs routines.
120 *
121 * ffs(value) returns 0 if value is 0 or the position of the first set bit if
122 * value is nonzero. The first (least significant) bit is at position 1.
123 */
124 #define ffs(x) (__builtin_constant_p(x) ? __builtin_ffs(x) : variable_ffs(x))
125
variable_fls(unsigned int x)126 static __always_inline int variable_fls(unsigned int x)
127 {
128 if (!riscv_has_extension_likely(RISCV_ISA_EXT_ZBB))
129 return generic_fls(x);
130
131 if (!x)
132 return 0;
133
134 asm volatile (".option push\n"
135 ".option arch,+zbb\n"
136 CLZW "%0, %1\n"
137 ".option pop\n"
138 : "=r" (x) : "r" (x) :);
139
140 return 32 - x;
141 }
142
143 /**
144 * fls - find last set bit in a word
145 * @x: the word to search
146 *
147 * This is defined in a similar way as ffs, but returns the position of the most
148 * significant set bit.
149 *
150 * fls(value) returns 0 if value is 0 or the position of the last set bit if
151 * value is nonzero. The last (most significant) bit is at position 32.
152 */
153 #define fls(x) \
154 ({ \
155 typeof(x) x_ = (x); \
156 __builtin_constant_p(x_) ? \
157 ((x_ != 0) ? (32 - __builtin_clz(x_)) : 0) \
158 : \
159 variable_fls(x_); \
160 })
161
162 #endif /* !(defined(CONFIG_RISCV_ISA_ZBB) && defined(CONFIG_TOOLCHAIN_HAS_ZBB)) || defined(NO_ALTERNATIVE) */
163
164 #include <asm-generic/bitops/ffz.h>
165 #include <asm-generic/bitops/fls64.h>
166 #include <asm-generic/bitops/sched.h>
167
168 #include <asm/arch_hweight.h>
169
170 #include <asm-generic/bitops/const_hweight.h>
171
172 #if (BITS_PER_LONG == 64)
173 #define __AMO(op) "amo" #op ".d"
174 #elif (BITS_PER_LONG == 32)
175 #define __AMO(op) "amo" #op ".w"
176 #else
177 #error "Unexpected BITS_PER_LONG"
178 #endif
179
180 #define __test_and_op_bit_ord(op, mod, nr, addr, ord) \
181 ({ \
182 unsigned long __res, __mask; \
183 __mask = BIT_MASK(nr); \
184 __asm__ __volatile__ ( \
185 __AMO(op) #ord " %0, %2, %1" \
186 : "=r" (__res), "+A" (addr[BIT_WORD(nr)]) \
187 : "r" (mod(__mask)) \
188 : "memory"); \
189 ((__res & __mask) != 0); \
190 })
191
192 #define __op_bit_ord(op, mod, nr, addr, ord) \
193 __asm__ __volatile__ ( \
194 __AMO(op) #ord " zero, %1, %0" \
195 : "+A" (addr[BIT_WORD(nr)]) \
196 : "r" (mod(BIT_MASK(nr))) \
197 : "memory");
198
199 #define __test_and_op_bit(op, mod, nr, addr) \
200 __test_and_op_bit_ord(op, mod, nr, addr, .aqrl)
201 #define __op_bit(op, mod, nr, addr) \
202 __op_bit_ord(op, mod, nr, addr, )
203
204 /* Bitmask modifiers */
205 #define __NOP(x) (x)
206 #define __NOT(x) (~(x))
207
208 /**
209 * arch_test_and_set_bit - Set a bit and return its old value
210 * @nr: Bit to set
211 * @addr: Address to count from
212 *
213 * This is an atomic fully-ordered operation (implied full memory barrier).
214 */
arch_test_and_set_bit(int nr,volatile unsigned long * addr)215 static __always_inline int arch_test_and_set_bit(int nr, volatile unsigned long *addr)
216 {
217 return __test_and_op_bit(or, __NOP, nr, addr);
218 }
219
220 /**
221 * arch_test_and_clear_bit - Clear a bit and return its old value
222 * @nr: Bit to clear
223 * @addr: Address to count from
224 *
225 * This is an atomic fully-ordered operation (implied full memory barrier).
226 */
arch_test_and_clear_bit(int nr,volatile unsigned long * addr)227 static __always_inline int arch_test_and_clear_bit(int nr, volatile unsigned long *addr)
228 {
229 return __test_and_op_bit(and, __NOT, nr, addr);
230 }
231
232 /**
233 * arch_test_and_change_bit - Change a bit and return its old value
234 * @nr: Bit to change
235 * @addr: Address to count from
236 *
237 * This operation is atomic and cannot be reordered.
238 * It also implies a memory barrier.
239 */
arch_test_and_change_bit(int nr,volatile unsigned long * addr)240 static __always_inline int arch_test_and_change_bit(int nr, volatile unsigned long *addr)
241 {
242 return __test_and_op_bit(xor, __NOP, nr, addr);
243 }
244
245 /**
246 * arch_set_bit - Atomically set a bit in memory
247 * @nr: the bit to set
248 * @addr: the address to start counting from
249 *
250 * Note: there are no guarantees that this function will not be reordered
251 * on non x86 architectures, so if you are writing portable code,
252 * make sure not to rely on its reordering guarantees.
253 *
254 * Note that @nr may be almost arbitrarily large; this function is not
255 * restricted to acting on a single-word quantity.
256 */
arch_set_bit(int nr,volatile unsigned long * addr)257 static __always_inline void arch_set_bit(int nr, volatile unsigned long *addr)
258 {
259 __op_bit(or, __NOP, nr, addr);
260 }
261
262 /**
263 * arch_clear_bit - Clears a bit in memory
264 * @nr: Bit to clear
265 * @addr: Address to start counting from
266 *
267 * Note: there are no guarantees that this function will not be reordered
268 * on non x86 architectures, so if you are writing portable code,
269 * make sure not to rely on its reordering guarantees.
270 */
arch_clear_bit(int nr,volatile unsigned long * addr)271 static __always_inline void arch_clear_bit(int nr, volatile unsigned long *addr)
272 {
273 __op_bit(and, __NOT, nr, addr);
274 }
275
276 /**
277 * arch_change_bit - Toggle a bit in memory
278 * @nr: Bit to change
279 * @addr: Address to start counting from
280 *
281 * change_bit() may be reordered on other architectures than x86.
282 * Note that @nr may be almost arbitrarily large; this function is not
283 * restricted to acting on a single-word quantity.
284 */
arch_change_bit(int nr,volatile unsigned long * addr)285 static __always_inline void arch_change_bit(int nr, volatile unsigned long *addr)
286 {
287 __op_bit(xor, __NOP, nr, addr);
288 }
289
290 /**
291 * arch_test_and_set_bit_lock - Set a bit and return its old value, for lock
292 * @nr: Bit to set
293 * @addr: Address to count from
294 *
295 * This operation is atomic and provides acquire barrier semantics.
296 * It can be used to implement bit locks.
297 */
arch_test_and_set_bit_lock(unsigned long nr,volatile unsigned long * addr)298 static __always_inline int arch_test_and_set_bit_lock(
299 unsigned long nr, volatile unsigned long *addr)
300 {
301 return __test_and_op_bit_ord(or, __NOP, nr, addr, .aq);
302 }
303
304 /**
305 * arch_clear_bit_unlock - Clear a bit in memory, for unlock
306 * @nr: the bit to set
307 * @addr: the address to start counting from
308 *
309 * This operation is atomic and provides release barrier semantics.
310 */
arch_clear_bit_unlock(unsigned long nr,volatile unsigned long * addr)311 static __always_inline void arch_clear_bit_unlock(
312 unsigned long nr, volatile unsigned long *addr)
313 {
314 __op_bit_ord(and, __NOT, nr, addr, .rl);
315 }
316
317 /**
318 * arch___clear_bit_unlock - Clear a bit in memory, for unlock
319 * @nr: the bit to set
320 * @addr: the address to start counting from
321 *
322 * This operation is like clear_bit_unlock, however it is not atomic.
323 * It does provide release barrier semantics so it can be used to unlock
324 * a bit lock, however it would only be used if no other CPU can modify
325 * any bits in the memory until the lock is released (a good example is
326 * if the bit lock itself protects access to the other bits in the word).
327 *
328 * On RISC-V systems there seems to be no benefit to taking advantage of the
329 * non-atomic property here: it's a lot more instructions and we still have to
330 * provide release semantics anyway.
331 */
arch___clear_bit_unlock(unsigned long nr,volatile unsigned long * addr)332 static __always_inline void arch___clear_bit_unlock(
333 unsigned long nr, volatile unsigned long *addr)
334 {
335 arch_clear_bit_unlock(nr, addr);
336 }
337
arch_xor_unlock_is_negative_byte(unsigned long mask,volatile unsigned long * addr)338 static __always_inline bool arch_xor_unlock_is_negative_byte(unsigned long mask,
339 volatile unsigned long *addr)
340 {
341 unsigned long res;
342 __asm__ __volatile__ (
343 __AMO(xor) ".rl %0, %2, %1"
344 : "=r" (res), "+A" (*addr)
345 : "r" (__NOP(mask))
346 : "memory");
347 return (res & BIT(7)) != 0;
348 }
349
350 #undef __test_and_op_bit
351 #undef __op_bit
352 #undef __NOP
353 #undef __NOT
354 #undef __AMO
355
356 #include <asm-generic/bitops/instrumented-atomic.h>
357 #include <asm-generic/bitops/instrumented-lock.h>
358
359 #include <asm-generic/bitops/non-atomic.h>
360 #include <asm-generic/bitops/le.h>
361 #include <asm-generic/bitops/ext2-atomic.h>
362
363 #endif /* _ASM_RISCV_BITOPS_H */
364