xref: /freebsd/sys/arm64/include/atomic.h (revision 66df505066f51e6d8411b966765d828817f88971)
1 /*-
2  * Copyright (c) 2013 Andrew Turner <andrew@freebsd.org>
3  * All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  * $FreeBSD$
27  */
28 
29 #ifndef	_MACHINE_ATOMIC_H_
30 #define	_MACHINE_ATOMIC_H_
31 
32 #define	isb()		__asm __volatile("isb" : : : "memory")
33 
34 /*
35  * Options for DMB and DSB:
36  *	oshld	Outer Shareable, load
37  *	oshst	Outer Shareable, store
38  *	osh	Outer Shareable, all
39  *	nshld	Non-shareable, load
40  *	nshst	Non-shareable, store
41  *	nsh	Non-shareable, all
42  *	ishld	Inner Shareable, load
43  *	ishst	Inner Shareable, store
44  *	ish	Inner Shareable, all
45  *	ld	Full system, load
46  *	st	Full system, store
47  *	sy	Full system, all
48  */
49 #define	dsb(opt)	__asm __volatile("dsb " __STRING(opt) : : : "memory")
50 #define	dmb(opt)	__asm __volatile("dmb " __STRING(opt) : : : "memory")
51 
52 #define	mb()	dmb(sy)	/* Full system memory barrier all */
53 #define	wmb()	dmb(st)	/* Full system memory barrier store */
54 #define	rmb()	dmb(ld)	/* Full system memory barrier load */
55 
56 #if defined(KCSAN) && !defined(KCSAN_RUNTIME)
57 #include <sys/atomic_san.h>
58 #else
59 
60 #include <sys/atomic_common.h>
61 
62 #ifdef _KERNEL
63 extern bool lse_supported;
64 
65 #ifdef LSE_ATOMICS
66 #define	_ATOMIC_LSE_SUPPORTED	1
67 #else
68 #define	_ATOMIC_LSE_SUPPORTED	lse_supported
69 #endif
70 #else
71 #define	_ATOMIC_LSE_SUPPORTED	0
72 #endif
73 
74 #define	_ATOMIC_OP_PROTO(t, op, bar, flav)				\
75 static __inline void							\
76 atomic_##op##_##bar##t##flav(volatile uint##t##_t *p, uint##t##_t val)
77 
78 #define	_ATOMIC_OP_IMPL(t, w, s, op, llsc_asm_op, lse_asm_op, pre, bar, a, l) \
79 _ATOMIC_OP_PROTO(t, op, bar, _llsc)					\
80 {									\
81 	uint##t##_t tmp;						\
82 	int res;							\
83 									\
84 	pre;								\
85 	__asm __volatile(						\
86 	    "1: ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
87 	    "   "#llsc_asm_op"	%"#w"0, %"#w"0, %"#w"3\n"		\
88 	    "   st"#l"xr"#s"	%w1, %"#w"0, [%2]\n"			\
89 	    "   cbnz		%w1, 1b\n"				\
90 	    : "=&r"(tmp), "=&r"(res)					\
91 	    : "r" (p), "r" (val)					\
92 	    : "memory"							\
93 	);								\
94 }									\
95 									\
96 _ATOMIC_OP_PROTO(t, op, bar, _lse)					\
97 {									\
98 	uint##t##_t tmp;						\
99 									\
100 	pre;								\
101 	__asm __volatile(						\
102 	    ".arch_extension lse\n"					\
103 	    "ld"#lse_asm_op#a#l#s"	%"#w"2, %"#w"0, [%1]\n"		\
104 	    ".arch_extension nolse\n"					\
105 	    : "=r" (tmp)						\
106 	    : "r" (p), "r" (val)					\
107 	    : "memory"							\
108 	);								\
109 }									\
110 									\
111 _ATOMIC_OP_PROTO(t, op, bar, )						\
112 {									\
113 	if (_ATOMIC_LSE_SUPPORTED)					\
114 		atomic_##op##_##bar##t##_lse(p, val);			\
115 	else								\
116 		atomic_##op##_##bar##t##_llsc(p, val);			\
117 }
118 
119 #define	__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, bar, a, l)	\
120 	_ATOMIC_OP_IMPL(8,  w, b, op, llsc_asm_op, lse_asm_op, pre,	\
121 	    bar, a, l)							\
122 	_ATOMIC_OP_IMPL(16, w, h, op, llsc_asm_op, lse_asm_op, pre,	\
123 	    bar, a, l)							\
124 	_ATOMIC_OP_IMPL(32, w,  , op, llsc_asm_op, lse_asm_op, pre,	\
125 	    bar, a, l)							\
126 	_ATOMIC_OP_IMPL(64,  ,  , op, llsc_asm_op, lse_asm_op, pre,	\
127 	    bar, a, l)
128 
129 #define	_ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre)			\
130 	__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre,     ,  ,  )	\
131 	__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, acq_, a,  )	\
132 	__ATOMIC_OP(op, llsc_asm_op, lse_asm_op, pre, rel_,  , l)
133 
134 _ATOMIC_OP(add,      add, add, )
135 _ATOMIC_OP(clear,    bic, clr, )
136 _ATOMIC_OP(set,      orr, set, )
137 _ATOMIC_OP(subtract, add, add, val = -val)
138 
139 #define	_ATOMIC_CMPSET_PROTO(t, bar, flav)				\
140 static __inline int							\
141 atomic_cmpset_##bar##t##flav(volatile uint##t##_t *p,			\
142     uint##t##_t cmpval, uint##t##_t newval)
143 
144 #define	_ATOMIC_FCMPSET_PROTO(t, bar, flav)				\
145 static __inline int							\
146 atomic_fcmpset_##bar##t##flav(volatile uint##t##_t *p,			\
147     uint##t##_t *cmpval, uint##t##_t newval)
148 
149 #define	_ATOMIC_CMPSET_IMPL(t, w, s, bar, a, l)				\
150 _ATOMIC_CMPSET_PROTO(t, bar, _llsc)					\
151 {									\
152 	uint##t##_t tmp;						\
153 	int res;							\
154 									\
155 	__asm __volatile(						\
156 	    "1: mov		%w1, #1\n"				\
157 	    "   ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
158 	    "   cmp		%"#w"0, %"#w"3\n"			\
159 	    "   b.ne		2f\n"					\
160 	    "   st"#l"xr"#s"	%w1, %"#w"4, [%2]\n"			\
161 	    "   cbnz		%w1, 1b\n"				\
162 	    "2:"							\
163 	    : "=&r"(tmp), "=&r"(res)					\
164 	    : "r" (p), "r" (cmpval), "r" (newval)			\
165 	    : "cc", "memory"						\
166 	);								\
167 									\
168 	return (!res);							\
169 }									\
170 									\
171 _ATOMIC_CMPSET_PROTO(t, bar, _lse)					\
172 {									\
173 	uint##t##_t oldval;						\
174 	int res;							\
175 									\
176 	oldval = cmpval;						\
177 	__asm __volatile(						\
178 	    ".arch_extension lse\n"					\
179 	    "cas"#a#l#s"	%"#w"1, %"#w"4, [%3]\n"			\
180 	    "cmp		%"#w"1, %"#w"2\n"			\
181 	    "cset		%w0, eq\n"				\
182 	    ".arch_extension nolse\n"					\
183 	    : "=r" (res), "+&r" (cmpval)				\
184 	    : "r" (oldval), "r" (p), "r" (newval)			\
185 	    : "cc", "memory"						\
186 	);								\
187 									\
188 	return (res);							\
189 }									\
190 									\
191 _ATOMIC_CMPSET_PROTO(t, bar, )						\
192 {									\
193 	if (_ATOMIC_LSE_SUPPORTED)					\
194 		return (atomic_cmpset_##bar##t##_lse(p, cmpval,		\
195 		    newval));						\
196 	else								\
197 		return (atomic_cmpset_##bar##t##_llsc(p, cmpval,	\
198 		    newval));						\
199 }									\
200 									\
201 _ATOMIC_FCMPSET_PROTO(t, bar, _llsc)					\
202 {									\
203 	uint##t##_t _cmpval, tmp;					\
204 	int res;							\
205 									\
206 	_cmpval = *cmpval;						\
207 	__asm __volatile(						\
208 	    "   mov		%w1, #1\n"				\
209 	    "   ld"#a"xr"#s"	%"#w"0, [%2]\n"				\
210 	    "   cmp		%"#w"0, %"#w"3\n"			\
211 	    "   b.ne		1f\n"					\
212 	    "   st"#l"xr"#s"	%w1, %"#w"4, [%2]\n"			\
213 	    "1:"							\
214 	    : "=&r"(tmp), "=&r"(res)					\
215 	    : "r" (p), "r" (_cmpval), "r" (newval)			\
216 	    : "cc", "memory"						\
217 	);								\
218 	*cmpval = tmp;							\
219 									\
220 	return (!res);							\
221 }									\
222 									\
223 _ATOMIC_FCMPSET_PROTO(t, bar, _lse)					\
224 {									\
225 	uint##t##_t _cmpval, tmp;					\
226 	int res;							\
227 									\
228 	_cmpval = tmp = *cmpval;					\
229 	__asm __volatile(						\
230 	    ".arch_extension lse\n"					\
231 	    "cas"#a#l#s"	%"#w"1, %"#w"4, [%3]\n"			\
232 	    "cmp		%"#w"1, %"#w"2\n"			\
233 	    "cset		%w0, eq\n"				\
234 	    ".arch_extension nolse\n"					\
235 	    : "=r" (res), "+&r" (tmp)					\
236 	    : "r" (_cmpval), "r" (p), "r" (newval)			\
237 	    : "cc", "memory"						\
238 	);								\
239 	*cmpval = tmp;							\
240 									\
241 	return (res);							\
242 }									\
243 									\
244 _ATOMIC_FCMPSET_PROTO(t, bar, )						\
245 {									\
246 	if (_ATOMIC_LSE_SUPPORTED)					\
247 		return (atomic_fcmpset_##bar##t##_lse(p, cmpval,	\
248 		    newval));						\
249 	else								\
250 		return (atomic_fcmpset_##bar##t##_llsc(p, cmpval,	\
251 		    newval));						\
252 }
253 
254 #define	_ATOMIC_CMPSET(bar, a, l)					\
255 	_ATOMIC_CMPSET_IMPL(8,  w, b, bar, a, l)			\
256 	_ATOMIC_CMPSET_IMPL(16, w, h, bar, a, l)			\
257 	_ATOMIC_CMPSET_IMPL(32, w,  , bar, a, l)			\
258 	_ATOMIC_CMPSET_IMPL(64,  ,  , bar, a, l)
259 
260 #define	atomic_cmpset_8		atomic_cmpset_8
261 #define	atomic_fcmpset_8	atomic_fcmpset_8
262 #define	atomic_cmpset_16	atomic_cmpset_16
263 #define	atomic_fcmpset_16	atomic_fcmpset_16
264 
265 _ATOMIC_CMPSET(    ,  , )
266 _ATOMIC_CMPSET(acq_, a, )
267 _ATOMIC_CMPSET(rel_,  ,l)
268 
269 #define	_ATOMIC_FETCHADD_PROTO(t, flav)					\
270 static __inline uint##t##_t						\
271 atomic_fetchadd_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
272 
273 #define	_ATOMIC_FETCHADD_IMPL(t, w)					\
274 _ATOMIC_FETCHADD_PROTO(t, _llsc)					\
275 {									\
276 	uint##t##_t ret, tmp;						\
277 	int res;							\
278 									\
279 	__asm __volatile(						\
280 	    "1: ldxr	%"#w"2, [%3]\n"					\
281 	    "   add	%"#w"0, %"#w"2, %"#w"4\n"			\
282 	    "   stxr	%w1, %"#w"0, [%3]\n"				\
283             "   cbnz	%w1, 1b\n"					\
284 	    : "=&r" (tmp), "=&r" (res), "=&r" (ret)			\
285 	    : "r" (p), "r" (val)					\
286 	    : "memory"							\
287 	);								\
288 									\
289 	return (ret);							\
290 }									\
291 									\
292 _ATOMIC_FETCHADD_PROTO(t, _lse)						\
293 {									\
294 	uint##t##_t ret;						\
295 									\
296 	__asm __volatile(						\
297 	    ".arch_extension lse\n"					\
298 	    "ldadd	%"#w"2, %"#w"0, [%1]\n"				\
299 	    ".arch_extension nolse\n"					\
300 	    : "=r" (ret)						\
301 	    : "r" (p), "r" (val)					\
302 	    : "memory"							\
303 	);								\
304 									\
305 	return (ret);							\
306 }									\
307 									\
308 _ATOMIC_FETCHADD_PROTO(t, )						\
309 {									\
310 	if (_ATOMIC_LSE_SUPPORTED)					\
311 		return (atomic_fetchadd_##t##_lse(p, val));		\
312 	else								\
313 		return (atomic_fetchadd_##t##_llsc(p, val));		\
314 }
315 
316 _ATOMIC_FETCHADD_IMPL(32, w)
317 _ATOMIC_FETCHADD_IMPL(64,  )
318 
319 #define	_ATOMIC_SWAP_PROTO(t, flav)					\
320 static __inline uint##t##_t						\
321 atomic_swap_##t##flav(volatile uint##t##_t *p, uint##t##_t val)
322 
323 #define	_ATOMIC_READANDCLEAR_PROTO(t, flav)				\
324 static __inline uint##t##_t						\
325 atomic_readandclear_##t##flav(volatile uint##t##_t *p)
326 
327 #define	_ATOMIC_SWAP_IMPL(t, w, zreg)					\
328 _ATOMIC_SWAP_PROTO(t, _llsc)						\
329 {									\
330 	uint##t##_t ret;						\
331 	int res;							\
332 									\
333 	__asm __volatile(						\
334 	    "1: ldxr	%"#w"1, [%2]\n"					\
335 	    "   stxr	%w0, %"#w"3, [%2]\n"				\
336             "   cbnz	%w0, 1b\n"					\
337 	    : "=&r" (res), "=&r" (ret)					\
338 	    : "r" (p), "r" (val)					\
339 	    : "memory"							\
340 	);								\
341 									\
342 	return (ret);							\
343 }									\
344 									\
345 _ATOMIC_SWAP_PROTO(t, _lse)						\
346 {									\
347 	uint##t##_t ret;						\
348 									\
349 	__asm __volatile(						\
350 	    ".arch_extension lse\n"					\
351 	    "swp	%"#w"2, %"#w"0, [%1]\n"				\
352 	    ".arch_extension nolse\n"					\
353 	    : "=r" (ret)						\
354 	    : "r" (p), "r" (val)					\
355 	    : "memory"							\
356 	);								\
357 									\
358 	return (ret);							\
359 }									\
360 									\
361 _ATOMIC_SWAP_PROTO(t, )							\
362 {									\
363 	if (_ATOMIC_LSE_SUPPORTED)					\
364 		return (atomic_swap_##t##_lse(p, val));			\
365 	else								\
366 		return (atomic_swap_##t##_llsc(p, val));		\
367 }									\
368 									\
369 _ATOMIC_READANDCLEAR_PROTO(t, _llsc)					\
370 {									\
371 	uint##t##_t ret;						\
372 	int res;							\
373 									\
374 	__asm __volatile(						\
375 	    "1: ldxr	%"#w"1, [%2]\n"					\
376 	    "   stxr	%w0, "#zreg", [%2]\n"				\
377 	    "   cbnz	%w0, 1b\n"					\
378 	    : "=&r" (res), "=&r" (ret)					\
379 	    : "r" (p)							\
380 	    : "memory"							\
381 	);								\
382 									\
383 	return (ret);							\
384 }									\
385 									\
386 _ATOMIC_READANDCLEAR_PROTO(t, _lse)					\
387 {									\
388 	return (atomic_swap_##t##_lse(p, 0));				\
389 }									\
390 									\
391 _ATOMIC_READANDCLEAR_PROTO(t, )						\
392 {									\
393 	if (_ATOMIC_LSE_SUPPORTED)					\
394 		return (atomic_readandclear_##t##_lse(p));		\
395 	else								\
396 		return (atomic_readandclear_##t##_llsc(p));		\
397 }
398 
399 _ATOMIC_SWAP_IMPL(32, w, wzr)
400 _ATOMIC_SWAP_IMPL(64,  , xzr)
401 
402 #define	_ATOMIC_TEST_OP_PROTO(t, op, flav)				\
403 static __inline int							\
404 atomic_testand##op##_##t##flav(volatile uint##t##_t *p, u_int val)
405 
406 #define	_ATOMIC_TEST_OP_IMPL(t, w, op, llsc_asm_op, lse_asm_op)		\
407 _ATOMIC_TEST_OP_PROTO(t, op, _llsc)					\
408 {									\
409 	uint##t##_t mask, old, tmp;					\
410 	int res;							\
411 									\
412 	mask = ((uint##t##_t)1) << (val & (t - 1));			\
413 	__asm __volatile(						\
414 	    "1: ldxr		%"#w"2, [%3]\n"				\
415 	    "  "#llsc_asm_op"	%"#w"0, %"#w"2, %"#w"4\n"		\
416 	    "   stxr		%w1, %"#w"0, [%3]\n"			\
417 	    "   cbnz		%w1, 1b\n"				\
418 	    : "=&r" (tmp), "=&r" (res), "=&r" (old)			\
419 	    : "r" (p), "r" (mask)					\
420 	    : "memory"							\
421 	);								\
422 									\
423 	return ((old & mask) != 0);					\
424 }									\
425 									\
426 _ATOMIC_TEST_OP_PROTO(t, op, _lse)					\
427 {									\
428 	uint##t##_t mask, old;						\
429 									\
430 	mask = ((uint##t##_t)1) << (val & (t - 1));			\
431 	__asm __volatile(						\
432 	    ".arch_extension lse\n"					\
433 	    "ld"#lse_asm_op"	%"#w"2, %"#w"0, [%1]\n"			\
434 	    ".arch_extension nolse\n"					\
435 	    : "=r" (old)						\
436 	    : "r" (p), "r" (mask)					\
437 	    : "memory"							\
438 	);								\
439 									\
440 	return ((old & mask) != 0);					\
441 }									\
442 									\
443 _ATOMIC_TEST_OP_PROTO(t, op, )						\
444 {									\
445 	if (_ATOMIC_LSE_SUPPORTED)					\
446 		return (atomic_testand##op##_##t##_lse(p, val));	\
447 	else								\
448 		return (atomic_testand##op##_##t##_llsc(p, val));	\
449 }
450 
451 #define	_ATOMIC_TEST_OP(op, llsc_asm_op, lse_asm_op)			\
452 	_ATOMIC_TEST_OP_IMPL(32, w, op, llsc_asm_op, lse_asm_op)	\
453 	_ATOMIC_TEST_OP_IMPL(64,  , op, llsc_asm_op, lse_asm_op)
454 
455 _ATOMIC_TEST_OP(clear, bic, clr)
456 _ATOMIC_TEST_OP(set,   orr, set)
457 
458 #define	_ATOMIC_LOAD_ACQ_IMPL(t, w, s)					\
459 static __inline uint##t##_t						\
460 atomic_load_acq_##t(volatile uint##t##_t *p)				\
461 {									\
462 	uint##t##_t ret;						\
463 									\
464 	__asm __volatile(						\
465 	    "ldar"#s"	%"#w"0, [%1]\n"					\
466 	    : "=&r" (ret)						\
467 	    : "r" (p)							\
468 	    : "memory");						\
469 									\
470 	return (ret);							\
471 }
472 
473 #define	atomic_load_acq_8	atomic_load_acq_8
474 #define	atomic_load_acq_16	atomic_load_acq_16
475 _ATOMIC_LOAD_ACQ_IMPL(8,  w, b)
476 _ATOMIC_LOAD_ACQ_IMPL(16, w, h)
477 _ATOMIC_LOAD_ACQ_IMPL(32, w,  )
478 _ATOMIC_LOAD_ACQ_IMPL(64,  ,  )
479 
480 #define	_ATOMIC_STORE_REL_IMPL(t, w, s)					\
481 static __inline void							\
482 atomic_store_rel_##t(volatile uint##t##_t *p, uint##t##_t val)		\
483 {									\
484 	__asm __volatile(						\
485 	    "stlr"#s"	%"#w"0, [%1]\n"					\
486 	    :								\
487 	    : "r" (val), "r" (p)					\
488 	    : "memory");						\
489 }
490 
491 _ATOMIC_STORE_REL_IMPL(8,  w, b)
492 _ATOMIC_STORE_REL_IMPL(16, w, h)
493 _ATOMIC_STORE_REL_IMPL(32, w,  )
494 _ATOMIC_STORE_REL_IMPL(64,  ,  )
495 
496 #define	atomic_add_int			atomic_add_32
497 #define	atomic_fcmpset_int		atomic_fcmpset_32
498 #define	atomic_clear_int		atomic_clear_32
499 #define	atomic_cmpset_int		atomic_cmpset_32
500 #define	atomic_fetchadd_int		atomic_fetchadd_32
501 #define	atomic_readandclear_int		atomic_readandclear_32
502 #define	atomic_set_int			atomic_set_32
503 #define	atomic_swap_int			atomic_swap_32
504 #define	atomic_subtract_int		atomic_subtract_32
505 #define	atomic_testandclear_int		atomic_testandclear_32
506 #define	atomic_testandset_int		atomic_testandset_32
507 
508 #define	atomic_add_acq_int		atomic_add_acq_32
509 #define	atomic_fcmpset_acq_int		atomic_fcmpset_acq_32
510 #define	atomic_clear_acq_int		atomic_clear_acq_32
511 #define	atomic_cmpset_acq_int		atomic_cmpset_acq_32
512 #define	atomic_load_acq_int		atomic_load_acq_32
513 #define	atomic_set_acq_int		atomic_set_acq_32
514 #define	atomic_subtract_acq_int		atomic_subtract_acq_32
515 
516 #define	atomic_add_rel_int		atomic_add_rel_32
517 #define	atomic_fcmpset_rel_int		atomic_fcmpset_rel_32
518 #define	atomic_clear_rel_int		atomic_clear_rel_32
519 #define	atomic_cmpset_rel_int		atomic_cmpset_rel_32
520 #define	atomic_set_rel_int		atomic_set_rel_32
521 #define	atomic_subtract_rel_int		atomic_subtract_rel_32
522 #define	atomic_store_rel_int		atomic_store_rel_32
523 
524 #define	atomic_add_long			atomic_add_64
525 #define	atomic_fcmpset_long		atomic_fcmpset_64
526 #define	atomic_clear_long		atomic_clear_64
527 #define	atomic_cmpset_long		atomic_cmpset_64
528 #define	atomic_fetchadd_long		atomic_fetchadd_64
529 #define	atomic_readandclear_long	atomic_readandclear_64
530 #define	atomic_set_long			atomic_set_64
531 #define	atomic_swap_long		atomic_swap_64
532 #define	atomic_subtract_long		atomic_subtract_64
533 #define	atomic_testandclear_long	atomic_testandclear_64
534 #define	atomic_testandset_long		atomic_testandset_64
535 
536 #define	atomic_add_ptr			atomic_add_64
537 #define	atomic_fcmpset_ptr		atomic_fcmpset_64
538 #define	atomic_clear_ptr		atomic_clear_64
539 #define	atomic_cmpset_ptr		atomic_cmpset_64
540 #define	atomic_fetchadd_ptr		atomic_fetchadd_64
541 #define	atomic_readandclear_ptr		atomic_readandclear_64
542 #define	atomic_set_ptr			atomic_set_64
543 #define	atomic_swap_ptr			atomic_swap_64
544 #define	atomic_subtract_ptr		atomic_subtract_64
545 
546 #define	atomic_add_acq_long		atomic_add_acq_64
547 #define	atomic_fcmpset_acq_long		atomic_fcmpset_acq_64
548 #define	atomic_clear_acq_long		atomic_clear_acq_64
549 #define	atomic_cmpset_acq_long		atomic_cmpset_acq_64
550 #define	atomic_load_acq_long		atomic_load_acq_64
551 #define	atomic_set_acq_long		atomic_set_acq_64
552 #define	atomic_subtract_acq_long	atomic_subtract_acq_64
553 
554 #define	atomic_add_acq_ptr		atomic_add_acq_64
555 #define	atomic_fcmpset_acq_ptr		atomic_fcmpset_acq_64
556 #define	atomic_clear_acq_ptr		atomic_clear_acq_64
557 #define	atomic_cmpset_acq_ptr		atomic_cmpset_acq_64
558 #define	atomic_load_acq_ptr		atomic_load_acq_64
559 #define	atomic_set_acq_ptr		atomic_set_acq_64
560 #define	atomic_subtract_acq_ptr		atomic_subtract_acq_64
561 
562 #define	atomic_add_rel_long		atomic_add_rel_64
563 #define	atomic_fcmpset_rel_long		atomic_fcmpset_rel_64
564 #define	atomic_clear_rel_long		atomic_clear_rel_64
565 #define	atomic_cmpset_rel_long		atomic_cmpset_rel_64
566 #define	atomic_set_rel_long		atomic_set_rel_64
567 #define	atomic_subtract_rel_long	atomic_subtract_rel_64
568 #define	atomic_store_rel_long		atomic_store_rel_64
569 
570 #define	atomic_add_rel_ptr		atomic_add_rel_64
571 #define	atomic_fcmpset_rel_ptr		atomic_fcmpset_rel_64
572 #define	atomic_clear_rel_ptr		atomic_clear_rel_64
573 #define	atomic_cmpset_rel_ptr		atomic_cmpset_rel_64
574 #define	atomic_set_rel_ptr		atomic_set_rel_64
575 #define	atomic_subtract_rel_ptr		atomic_subtract_rel_64
576 #define	atomic_store_rel_ptr		atomic_store_rel_64
577 
578 static __inline void
579 atomic_thread_fence_acq(void)
580 {
581 
582 	dmb(ld);
583 }
584 
585 static __inline void
586 atomic_thread_fence_rel(void)
587 {
588 
589 	dmb(sy);
590 }
591 
592 static __inline void
593 atomic_thread_fence_acq_rel(void)
594 {
595 
596 	dmb(sy);
597 }
598 
599 static __inline void
600 atomic_thread_fence_seq_cst(void)
601 {
602 
603 	dmb(sy);
604 }
605 
606 #include <sys/_atomic_subword.h>
607 
608 #endif /* KCSAN && !KCSAN_RUNTIME */
609 #endif /* _MACHINE_ATOMIC_H_ */
610