xref: /freebsd/sys/powerpc/include/atomic.h (revision 30d239bc4c510432e65a84fa1c14ed67a3ab1c92)
1 /*-
2  * Copyright (c) 2001 Benno Rice
3  * Copyright (c) 2001 David E. O'Brien
4  * Copyright (c) 1998 Doug Rabson
5  * All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  * $FreeBSD$
29  */
30 
31 #ifndef _MACHINE_ATOMIC_H_
32 #define	_MACHINE_ATOMIC_H_
33 
34 #include <machine/cpufunc.h>
35 
36 #ifndef _SYS_CDEFS_H_
37 #error this file needs sys/cdefs.h as a prerequisite
38 #endif
39 
40 /*
41  * Various simple arithmetic on memory which is atomic in the presence
42  * of interrupts and SMP safe.
43  */
44 
45 void	atomic_set_8(volatile uint8_t *, uint8_t);
46 void	atomic_clear_8(volatile uint8_t *, uint8_t);
47 void	atomic_add_8(volatile uint8_t *, uint8_t);
48 void	atomic_subtract_8(volatile uint8_t *, uint8_t);
49 
50 void	atomic_set_16(volatile uint16_t *, uint16_t);
51 void	atomic_clear_16(volatile uint16_t *, uint16_t);
52 void	atomic_add_16(volatile uint16_t *, uint16_t);
53 void	atomic_subtract_16(volatile uint16_t *, uint16_t);
54 
55 static __inline void
56 atomic_set_32(volatile uint32_t *p, uint32_t v)
57 {
58 	uint32_t temp;
59 
60 #ifdef __GNUCLIKE_ASM
61 	__asm __volatile (
62 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
63 		"or %0, %3, %0\n\t"		/* calculate new value */
64 		"stwcx. %0, 0, %2\n\t"      	/* attempt to store */
65 		"bne- 1b\n\t"			/* spin if failed */
66 		: "=&r" (temp), "=m" (*p)
67 		: "r" (p), "r" (v), "m" (*p)
68 		: "cc", "memory");
69 #endif
70 }
71 
72 static __inline void
73 atomic_clear_32(volatile uint32_t *p, uint32_t v)
74 {
75 	uint32_t temp;
76 
77 #ifdef __GNUCLIKE_ASM
78 	__asm __volatile (
79 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
80 		"andc %0, %0, %3\n\t"		/* calculate new value */
81 		"stwcx. %0, 0, %2\n\t"      	/* attempt to store */
82 		"bne- 1b\n\t"			/* spin if failed */
83 		: "=&r" (temp), "=m" (*p)
84 		: "r" (p), "r" (v), "m" (*p)
85 		: "cc", "memory");
86 #endif
87 }
88 
89 static __inline void
90 atomic_add_32(volatile uint32_t *p, uint32_t v)
91 {
92 	uint32_t temp;
93 
94 #ifdef __GNUCLIKE_ASM
95 	__asm __volatile (
96 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
97 		"add %0, %3, %0\n\t"		/* calculate new value */
98 		"stwcx. %0, 0, %2\n\t"      	/* attempt to store */
99 		"bne- 1b\n\t"			/* spin if failed */
100 		: "=&r" (temp), "=m" (*p)
101 		: "r" (p), "r" (v), "m" (*p)
102 		: "cc", "memory");
103 #endif
104 }
105 
106 static __inline void
107 atomic_subtract_32(volatile uint32_t *p, uint32_t v)
108 {
109 	uint32_t temp;
110 
111 #ifdef __GNUCLIKE_ASM
112 	__asm __volatile (
113 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
114 		"subf %0, %3, %0\n\t"		/* calculate new value */
115 		"stwcx. %0, 0, %2\n\t"      	/* attempt to store */
116 		"bne- 1b\n\t"			/* spin if failed */
117 		: "=&r" (temp), "=m" (*p)
118 		: "r" (p), "r" (v), "m" (*p)
119 		: "cc", "memory");
120 #endif
121 }
122 
123 static __inline uint32_t
124 atomic_readandclear_32(volatile uint32_t *addr)
125 {
126 	uint32_t result,temp;
127 
128 #ifdef __GNUCLIKE_ASM
129 	__asm __volatile (
130 		"\tsync\n"			/* drain writes */
131 		"1:\tlwarx %0, 0, %3\n\t"	/* load old value */
132 		"li %1, 0\n\t"			/* load new value */
133 		"stwcx. %1, 0, %3\n\t"      	/* attempt to store */
134 		"bne- 1b\n\t"			/* spin if failed */
135 		: "=&r"(result), "=&r"(temp), "=m" (*addr)
136 		: "r" (addr), "m" (*addr)
137 		: "cc", "memory");
138 #endif
139 
140 	return (result);
141 }
142 
143 #if 0
144 
145 /*
146  * So far I haven't found a way to implement atomic 64-bit ops on the
147  * 32-bit PowerPC without involving major headaches.  If anyone has
148  * any ideas, please let me know. =)
149  * 	- benno@FreeBSD.org
150  */
151 
152 static __inline void
153 atomic_set_64(volatile u_int64_t *p, u_int64_t v)
154 {
155 	u_int64_t temp;
156 
157 	__asm __volatile (
158 		: "=&r" (temp), "=r" (*p)
159 		: "r" (*p), "r" (v)
160 		: "memory");
161 }
162 
163 static __inline void
164 atomic_clear_64(volatile u_int64_t *p, u_int64_t v)
165 {
166 	u_int64_t temp;
167 
168 	__asm __volatile (
169 		: "=&r" (temp), "=r" (*p)
170 		: "r" (*p), "r" (v)
171 		: "memory");
172 }
173 
174 static __inline void
175 atomic_add_64(volatile u_int64_t *p, u_int64_t v)
176 {
177 	u_int64_t temp;
178 
179 	__asm __volatile (
180 		: "=&r" (temp), "=r" (*p)
181 		: "r" (*p), "r" (v)
182 		: "memory");
183 }
184 
185 static __inline void
186 atomic_subtract_64(volatile u_int64_t *p, u_int64_t v)
187 {
188 	u_int64_t temp;
189 
190 	__asm __volatile (
191 		: "=&r" (temp), "=r" (*p)
192 		: "r" (*p), "r" (v)
193 		: "memory");
194 }
195 
196 static __inline u_int64_t
197 atomic_readandclear_64(volatile u_int64_t *addr)
198 {
199 	u_int64_t result,temp;
200 
201 	__asm __volatile (
202 		: "=&r"(result), "=&r"(temp), "=r" (*addr)
203 		: "r"(*addr)
204 		: "memory");
205 
206 	return result;
207 }
208 
209 #endif /* 0 */
210 
211 #define	atomic_set_char			atomic_set_8
212 #define	atomic_clear_char		atomic_clear_8
213 #define	atomic_add_char			atomic_add_8
214 #define	atomic_subtract_char		atomic_subtract_8
215 
216 #define	atomic_set_short		atomic_set_16
217 #define	atomic_clear_short		atomic_clear_16
218 #define	atomic_add_short		atomic_add_16
219 #define	atomic_subtract_short		atomic_subtract_16
220 
221 #define	atomic_set_int			atomic_set_32
222 #define	atomic_clear_int		atomic_clear_32
223 #define	atomic_add_int			atomic_add_32
224 #define	atomic_subtract_int		atomic_subtract_32
225 #define	atomic_readandclear_int		atomic_readandclear_32
226 
227 #define	atomic_set_long			atomic_set_32
228 #define	atomic_clear_long		atomic_clear_32
229 #define	atomic_add_long(p, v)		atomic_add_32((uint32_t *)p, (uint32_t)v)
230 #define	atomic_subtract_long(p, v)     	atomic_subtract_32((uint32_t *)p, (uint32_t)v)
231 #define	atomic_readandclear_long	atomic_readandclear_32
232 
233 #define	atomic_set_ptr			atomic_set_32
234 #define	atomic_clear_ptr		atomic_clear_32
235 #define	atomic_add_ptr			atomic_add_32
236 #define	atomic_subtract_ptr     	atomic_subtract_32
237 
238 #if 0
239 
240 /* See above. */
241 
242 #define	atomic_set_long_long		atomic_set_64
243 #define	atomic_clear_long_long		atomic_clear_64
244 #define	atomic_add_long_long		atomic_add_64
245 #define	atomic_subtract_long_long	atomic_subtract_64
246 #define	atomic_readandclear_long_long	atomic_readandclear_64
247 
248 #endif /* 0 */
249 
250 #define	ATOMIC_ACQ_REL(NAME, WIDTH, TYPE)				\
251 static __inline void							\
252 atomic_##NAME##_acq_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \
253 {									\
254 	atomic_##NAME##_##WIDTH(p, v);					\
255 	powerpc_mb();							\
256 }									\
257 									\
258 static __inline void							\
259 atomic_##NAME##_rel_##WIDTH(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \
260 {									\
261 	powerpc_mb();							\
262 	atomic_##NAME##_##WIDTH(p, v);					\
263 }									\
264 									\
265 static __inline void							\
266 atomic_##NAME##_acq_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \
267 {									\
268 	atomic_##NAME##_##WIDTH(p, v);					\
269 	powerpc_mb();							\
270 }									\
271 									\
272 static __inline void							\
273 atomic_##NAME##_rel_##TYPE(volatile u_int##WIDTH##_t *p, u_int##WIDTH##_t v) \
274 {									\
275 	powerpc_mb();							\
276 	atomic_##NAME##_##WIDTH(p, v);					\
277 }
278 
279 ATOMIC_ACQ_REL(set, 8, char)
280 ATOMIC_ACQ_REL(clear, 8, char)
281 ATOMIC_ACQ_REL(add, 8, char)
282 ATOMIC_ACQ_REL(subtract, 8, char)
283 ATOMIC_ACQ_REL(set, 16, short)
284 ATOMIC_ACQ_REL(clear, 16, short)
285 ATOMIC_ACQ_REL(add, 16, short)
286 ATOMIC_ACQ_REL(subtract, 16, short)
287 ATOMIC_ACQ_REL(set, 32, int)
288 ATOMIC_ACQ_REL(clear, 32, int)
289 ATOMIC_ACQ_REL(add, 32, int)
290 ATOMIC_ACQ_REL(subtract, 32, int)
291 
292 #define	atomic_set_acq_long		atomic_set_acq_32
293 #define	atomic_set_rel_long		atomic_set_rel_32
294 #define	atomic_clear_acq_long		atomic_clear_acq_32
295 #define	atomic_clear_rel_long		atomic_clear_rel_32
296 #define	atomic_add_acq_long		atomic_add_acq_32
297 #define	atomic_add_rel_long		atomic_add_rel_32
298 #define	atomic_subtract_acq_long	atomic_subtract_acq_32
299 #define	atomic_subtract_rel_long	atomic_subtract_rel_32
300 
301 #define	atomic_set_acq_ptr		atomic_set_acq_32
302 #define	atomic_set_rel_ptr		atomic_set_rel_32
303 #define	atomic_clear_acq_ptr		atomic_clear_acq_32
304 #define	atomic_clear_rel_ptr		atomic_clear_rel_32
305 #define	atomic_add_acq_ptr		atomic_add_acq_32
306 #define	atomic_add_rel_ptr		atomic_add_rel_32
307 #define	atomic_subtract_acq_ptr		atomic_subtract_acq_32
308 #define	atomic_subtract_rel_ptr		atomic_subtract_rel_32
309 
310 #undef ATOMIC_ACQ_REL
311 
312 /*
313  * We assume that a = b will do atomic loads and stores.
314  */
315 #define	ATOMIC_STORE_LOAD(TYPE, WIDTH)				\
316 static __inline u_##TYPE					\
317 atomic_load_acq_##WIDTH(volatile u_##TYPE *p)			\
318 {								\
319 	u_##TYPE v;						\
320 								\
321 	v = *p;							\
322 	powerpc_mb();						\
323 	return (v);						\
324 }								\
325 								\
326 static __inline void						\
327 atomic_store_rel_##WIDTH(volatile u_##TYPE *p, u_##TYPE v)	\
328 {								\
329 	powerpc_mb();						\
330 	*p = v;							\
331 }								\
332 								\
333 static __inline u_##TYPE					\
334 atomic_load_acq_##TYPE(volatile u_##TYPE *p)			\
335 {								\
336 	u_##TYPE v;						\
337 								\
338 	v = *p;							\
339 	powerpc_mb();						\
340 	return (v);						\
341 }								\
342 								\
343 static __inline void						\
344 atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v)	\
345 {								\
346 	powerpc_mb();						\
347 	*p = v;							\
348 }
349 
350 ATOMIC_STORE_LOAD(char,		8)
351 ATOMIC_STORE_LOAD(short,	16)
352 ATOMIC_STORE_LOAD(int,		32)
353 
354 #define	atomic_load_acq_long	atomic_load_acq_32
355 #define	atomic_store_rel_long	atomic_store_rel_32
356 
357 #define	atomic_load_acq_ptr	atomic_load_acq_32
358 #define	atomic_store_rel_ptr	atomic_store_rel_32
359 
360 #undef ATOMIC_STORE_LOAD
361 
362 /*
363  * Atomically compare the value stored at *p with cmpval and if the
364  * two values are equal, update the value of *p with newval. Returns
365  * zero if the compare failed, nonzero otherwise.
366  */
367 static __inline uint32_t
368 atomic_cmpset_32(volatile uint32_t* p, uint32_t cmpval, uint32_t newval)
369 {
370 	uint32_t	ret;
371 
372 #ifdef __GNUCLIKE_ASM
373 	__asm __volatile (
374 		"1:\tlwarx %0, 0, %2\n\t"	/* load old value */
375 		"cmplw %3, %0\n\t"		/* compare */
376 		"bne 2f\n\t"			/* exit if not equal */
377 		"stwcx. %4, 0, %2\n\t"      	/* attempt to store */
378 		"bne- 1b\n\t"			/* spin if failed */
379 		"li %0, 1\n\t"			/* success - retval = 1 */
380 		"b 3f\n\t"			/* we've succeeded */
381 		"2:\n\t"
382 		"stwcx. %0, 0, %2\n\t"       	/* clear reservation (74xx) */
383 		"li %0, 0\n\t"			/* failure - retval = 0 */
384 		"3:\n\t"
385 		: "=&r" (ret), "=m" (*p)
386 		: "r" (p), "r" (cmpval), "r" (newval), "m" (*p)
387 		: "cc", "memory");
388 #endif
389 
390 	return (ret);
391 }
392 
393 #if 0
394 
395 /*
396  * Atomically compare the value stored at *p with cmpval and if the
397  * two values are equal, update the value of *p with newval. Returns
398  * zero if the compare failed, nonzero otherwise.
399  */
400 static __inline u_int64_t
401 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval)
402 {
403 	u_int64_t ret;
404 
405 	__asm __volatile (
406 		: "=&r" (ret), "=r" (*p)
407 		: "r" (cmpval), "r" (newval), "r" (*p)
408 		: "memory");
409 
410 	return ret;
411 }
412 
413 #endif /* 0 */
414 
415 #define	atomic_cmpset_int	atomic_cmpset_32
416 #define	atomic_cmpset_long	atomic_cmpset_32
417 
418 #define	atomic_cmpset_ptr(dst, old, new)	\
419     atomic_cmpset_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
420 
421 #if 0
422 #define	atomic_cmpset_long_long	atomic_cmpset_64
423 #endif /* 0 */
424 
425 static __inline uint32_t
426 atomic_cmpset_acq_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
427 {
428 	int retval;
429 
430 	retval = atomic_cmpset_32(p, cmpval, newval);
431 	powerpc_mb();
432 	return (retval);
433 }
434 
435 static __inline uint32_t
436 atomic_cmpset_rel_32(volatile uint32_t *p, uint32_t cmpval, uint32_t newval)
437 {
438 	powerpc_mb();
439 	return (atomic_cmpset_32(p, cmpval, newval));
440 }
441 
442 #define	atomic_cmpset_acq_int	atomic_cmpset_acq_32
443 #define	atomic_cmpset_rel_int	atomic_cmpset_rel_32
444 #define	atomic_cmpset_acq_long	atomic_cmpset_acq_32
445 #define	atomic_cmpset_rel_long	atomic_cmpset_rel_32
446 
447 #define	atomic_cmpset_acq_ptr(dst, old, new)	\
448     atomic_cmpset_acq_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
449 #define	atomic_cmpset_rel_ptr(dst, old, new)	\
450     atomic_cmpset_rel_32((volatile u_int *)(dst), (u_int)(old), (u_int)(new))
451 
452 static __inline uint32_t
453 atomic_fetchadd_32(volatile uint32_t *p, uint32_t v)
454 {
455 	uint32_t value;
456 
457 	do {
458 		value = *p;
459 	} while (!atomic_cmpset_32(p, value, value + v));
460 	return (value);
461 }
462 
463 #define	atomic_fetchadd_int	atomic_fetchadd_32
464 
465 #endif /* ! _MACHINE_ATOMIC_H_ */
466