1 /*-
5  * Copyright (c) 2013-2018 Mellanox Technologies, Ltd.
42 /*------------------------------------------------------------------------*
43  *	32-bit atomic operations
44  *------------------------------------------------------------------------*/
46 #define	atomic_add(i, v)		atomic_add_return((i), (v))  argument
47 #define	atomic_sub(i, v)		atomic_sub_return((i), (v))  argument
48 #define	atomic_inc_return(v)		atomic_add_return(1, (v))  argument
49 #define	atomic_add_negative(i, v)	(atomic_add_return((i), (v)) < 0)  argument
50 #define	atomic_add_and_test(i, v)	(atomic_add_return((i), (v)) == 0)  argument
51 #define	atomic_sub_and_test(i, v)	(atomic_sub_return((i), (v)) == 0)  argument
52 #define	atomic_dec_and_test(v)		(atomic_sub_return(1, (v)) == 0)  argument
53 #define	atomic_inc_and_test(v)		(atomic_add_return(1, (v)) == 0)  argument
54 #define	atomic_dec_return(v)		atomic_sub_return(1, (v))  argument
55 #define	atomic_inc_not_zero(v)		atomic_add_unless((v), 1, 0)  argument
58 atomic_add_return(int i, atomic_t *v)  in atomic_add_return()  argument
60 	return i + atomic_fetchadd_int(&v->counter, i);  in atomic_add_return()
64 atomic_sub_return(int i, atomic_t *v)  in atomic_sub_return()  argument
66 	return atomic_fetchadd_int(&v->counter, -i) - i;  in atomic_sub_return()
70 atomic_set(atomic_t *v, int i)  in atomic_set()  argument
72 	WRITE_ONCE(v->counter, i);  in atomic_set()
76 atomic_set_release(atomic_t *v, int i)  in atomic_set_release()  argument
78 	atomic_store_rel_int(&v->counter, i);  in atomic_set_release()
82 atomic_set_mask(unsigned int mask, atomic_t *v)  in atomic_set_mask()  argument
84 	atomic_set_int(&v->counter, mask);  in atomic_set_mask()
88 atomic_read(const atomic_t *v)  in atomic_read()  argument
90 	return READ_ONCE(v->counter);  in atomic_read()
94 atomic_inc(atomic_t *v)  in atomic_inc()  argument
96 	return atomic_fetchadd_int(&v->counter, 1) + 1;  in atomic_inc()
100 atomic_dec(atomic_t *v)  in atomic_dec()  argument
102 	return atomic_fetchadd_int(&v->counter, -1) - 1;  in atomic_dec()
106 atomic_add_unless(atomic_t *v, int a, int u)  in atomic_add_unless()  argument
108 	int c = atomic_read(v);  in atomic_add_unless()
113 		if (likely(atomic_fcmpset_int(&v->counter, &c, c + a)))  in atomic_add_unless()
120 atomic_fetch_add_unless(atomic_t *v, int a, int u)  in atomic_fetch_add_unless()  argument
122 	int c = atomic_read(v);  in atomic_fetch_add_unless()
127 		if (likely(atomic_fcmpset_int(&v->counter, &c, c + a)))  in atomic_fetch_add_unless()
134 atomic_clear_mask(unsigned int mask, atomic_t *v)  in atomic_clear_mask()  argument
136 	atomic_clear_int(&v->counter, mask);  in atomic_clear_mask()
140 atomic_xchg(atomic_t *v, int i)  in atomic_xchg()  argument
142 	return (atomic_swap_int(&v->counter, i));  in atomic_xchg()
146 atomic_cmpxchg(atomic_t *v, int old, int new)  in atomic_cmpxchg()  argument
151 		if (atomic_fcmpset_int(&v->counter, &ret, new))  in atomic_cmpxchg()
291 atomic_dec_if_positive(atomic_t *v)  in atomic_dec_if_positive()  argument
296 	old = atomic_read(v);  in atomic_dec_if_positive()
298 		retval = old - 1;  in atomic_dec_if_positive()
301 		if (likely(atomic_fcmpset_int(&v->counter, &old, retval)))  in atomic_dec_if_positive()
308 static inline void atomic_##op(int i, atomic_t *v)		\
312 	c = v->counter;						\
313 	while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)	\
318 static inline int atomic_fetch_##op(int i, atomic_t *v)		\
322 	c = v->counter;						\
323 	while ((old = atomic_cmpxchg(v, c, c c_op i)) != c)	\
330 atomic_fetch_inc(atomic_t *v)  in atomic_fetch_inc()  argument
333 	return ((atomic_inc_return(v) - 1));  in atomic_fetch_inc()