1 /*
2 * Copyright 2009-2015 Samy Al Bahra.
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24 * SUCH DAMAGE.
25 */
26
27 #ifndef CK_PR_PPC64_H
28 #define CK_PR_PPC64_H
29
30 #ifndef CK_PR_H
31 #error Do not include this file directly, use ck_pr.h
32 #endif
33
34 #include <ck_cc.h>
35 #include <ck_md.h>
36
37 /*
38 * The following represent supported atomic operations.
39 * These operations may be emulated.
40 */
41 #include "ck_f_pr.h"
42
43 /*
44 * Minimum interface requirement met.
45 */
46 #define CK_F_PR
47
48 /*
49 * This bounces the hardware thread from low to medium
50 * priority. I am unsure of the benefits of this approach
51 * but it is used by the Linux kernel.
52 */
53 CK_CC_INLINE static void
ck_pr_stall(void)54 ck_pr_stall(void)
55 {
56
57 __asm__ __volatile__("or 1, 1, 1;"
58 "or 2, 2, 2;" ::: "memory");
59 return;
60 }
61
62 #define CK_PR_FENCE(T, I) \
63 CK_CC_INLINE static void \
64 ck_pr_fence_strict_##T(void) \
65 { \
66 __asm__ __volatile__(I ::: "memory"); \
67 }
68
69 /*
70 * These are derived from:
71 * http://www.ibm.com/developerworks/systems/articles/powerpc.html
72 */
73 CK_PR_FENCE(atomic, "lwsync")
74 CK_PR_FENCE(atomic_store, "lwsync")
75 CK_PR_FENCE(atomic_load, "sync")
76 CK_PR_FENCE(store_atomic, "lwsync")
77 CK_PR_FENCE(load_atomic, "lwsync")
78 CK_PR_FENCE(store, "lwsync")
79 CK_PR_FENCE(store_load, "sync")
80 CK_PR_FENCE(load, "lwsync")
81 CK_PR_FENCE(load_store, "lwsync")
82 CK_PR_FENCE(memory, "sync")
83 CK_PR_FENCE(acquire, "lwsync")
84 CK_PR_FENCE(release, "lwsync")
85 CK_PR_FENCE(acqrel, "lwsync")
86 CK_PR_FENCE(lock, "lwsync")
87 CK_PR_FENCE(unlock, "lwsync")
88
89 #undef CK_PR_FENCE
90
91 #define CK_PR_LOAD(S, M, T, C, I) \
92 CK_CC_INLINE static T \
93 ck_pr_md_load_##S(const M *target) \
94 { \
95 T r; \
96 __asm__ __volatile__(I "%U1%X1 %0, %1" \
97 : "=r" (r) \
98 : "m" (*(const C *)target) \
99 : "memory"); \
100 return (r); \
101 }
102
103 CK_PR_LOAD(ptr, void, void *, uint64_t, "ld")
104
105 #define CK_PR_LOAD_S(S, T, I) CK_PR_LOAD(S, T, T, T, I)
106
107 CK_PR_LOAD_S(64, uint64_t, "ld")
108 CK_PR_LOAD_S(32, uint32_t, "lwz")
109 CK_PR_LOAD_S(16, uint16_t, "lhz")
110 CK_PR_LOAD_S(8, uint8_t, "lbz")
111 CK_PR_LOAD_S(uint, unsigned int, "lwz")
112 CK_PR_LOAD_S(int, int, "lwz")
113 CK_PR_LOAD_S(short, short, "lhz")
114 CK_PR_LOAD_S(char, char, "lbz")
115 #ifndef CK_PR_DISABLE_DOUBLE
116 CK_PR_LOAD_S(double, double, "ld")
117 #endif
118
119 #undef CK_PR_LOAD_S
120 #undef CK_PR_LOAD
121
122 #define CK_PR_STORE(S, M, T, C, I) \
123 CK_CC_INLINE static void \
124 ck_pr_md_store_##S(M *target, T v) \
125 { \
126 __asm__ __volatile__(I "%U0%X0 %1, %0" \
127 : "=m" (*(C *)target) \
128 : "r" (v) \
129 : "memory"); \
130 return; \
131 }
132
133 CK_PR_STORE(ptr, void, const void *, uint64_t, "std")
134
135 #define CK_PR_STORE_S(S, T, I) CK_PR_STORE(S, T, T, T, I)
136
137 CK_PR_STORE_S(64, uint64_t, "std")
138 CK_PR_STORE_S(32, uint32_t, "stw")
139 CK_PR_STORE_S(16, uint16_t, "sth")
140 CK_PR_STORE_S(8, uint8_t, "stb")
141 CK_PR_STORE_S(uint, unsigned int, "stw")
142 CK_PR_STORE_S(int, int, "stw")
143 CK_PR_STORE_S(short, short, "sth")
144 CK_PR_STORE_S(char, char, "stb")
145 #ifndef CK_PR_DISABLE_DOUBLE
146 CK_PR_STORE_S(double, double, "std")
147 #endif
148
149 #undef CK_PR_STORE_S
150 #undef CK_PR_STORE
151
152 CK_CC_INLINE static bool
ck_pr_cas_64_value(uint64_t * target,uint64_t compare,uint64_t set,uint64_t * value)153 ck_pr_cas_64_value(uint64_t *target, uint64_t compare, uint64_t set, uint64_t *value)
154 {
155 uint64_t previous;
156
157 __asm__ __volatile__("1:"
158 "ldarx %0, 0, %1;"
159 "cmpd 0, %0, %3;"
160 "bne- 2f;"
161 "stdcx. %2, 0, %1;"
162 "bne- 1b;"
163 "2:"
164 : "=&r" (previous)
165 : "r" (target),
166 "r" (set),
167 "r" (compare)
168 : "memory", "cc");
169
170 *value = previous;
171 return (previous == compare);
172 }
173
174 CK_CC_INLINE static bool
ck_pr_cas_ptr_value(void * target,void * compare,void * set,void * value)175 ck_pr_cas_ptr_value(void *target, void *compare, void *set, void *value)
176 {
177 void *previous;
178
179 __asm__ __volatile__("1:"
180 "ldarx %0, 0, %1;"
181 "cmpd 0, %0, %3;"
182 "bne- 2f;"
183 "stdcx. %2, 0, %1;"
184 "bne- 1b;"
185 "2:"
186 : "=&r" (previous)
187 : "r" (target),
188 "r" (set),
189 "r" (compare)
190 : "memory", "cc");
191
192 ck_pr_md_store_ptr(value, previous);
193 return (previous == compare);
194 }
195
196 CK_CC_INLINE static bool
ck_pr_cas_64(uint64_t * target,uint64_t compare,uint64_t set)197 ck_pr_cas_64(uint64_t *target, uint64_t compare, uint64_t set)
198 {
199 uint64_t previous;
200
201 __asm__ __volatile__("1:"
202 "ldarx %0, 0, %1;"
203 "cmpd 0, %0, %3;"
204 "bne- 2f;"
205 "stdcx. %2, 0, %1;"
206 "bne- 1b;"
207 "2:"
208 : "=&r" (previous)
209 : "r" (target),
210 "r" (set),
211 "r" (compare)
212 : "memory", "cc");
213
214 return (previous == compare);
215 }
216
217 CK_CC_INLINE static bool
ck_pr_cas_ptr(void * target,void * compare,void * set)218 ck_pr_cas_ptr(void *target, void *compare, void *set)
219 {
220 void *previous;
221
222 __asm__ __volatile__("1:"
223 "ldarx %0, 0, %1;"
224 "cmpd 0, %0, %3;"
225 "bne- 2f;"
226 "stdcx. %2, 0, %1;"
227 "bne- 1b;"
228 "2:"
229 : "=&r" (previous)
230 : "r" (target),
231 "r" (set),
232 "r" (compare)
233 : "memory", "cc");
234
235 return (previous == compare);
236 }
237
238 #define CK_PR_CAS(N, T) \
239 CK_CC_INLINE static bool \
240 ck_pr_cas_##N##_value(T *target, T compare, T set, T *value) \
241 { \
242 T previous; \
243 __asm__ __volatile__("1:" \
244 "lwarx %0, 0, %1;" \
245 "cmpw 0, %0, %3;" \
246 "bne- 2f;" \
247 "stwcx. %2, 0, %1;" \
248 "bne- 1b;" \
249 "2:" \
250 : "=&r" (previous) \
251 : "r" (target), \
252 "r" (set), \
253 "r" (compare) \
254 : "memory", "cc"); \
255 *value = previous; \
256 return (previous == compare); \
257 } \
258 CK_CC_INLINE static bool \
259 ck_pr_cas_##N(T *target, T compare, T set) \
260 { \
261 T previous; \
262 __asm__ __volatile__("1:" \
263 "lwarx %0, 0, %1;" \
264 "cmpw 0, %0, %3;" \
265 "bne- 2f;" \
266 "stwcx. %2, 0, %1;" \
267 "bne- 1b;" \
268 "2:" \
269 : "=&r" (previous) \
270 : "r" (target), \
271 "r" (set), \
272 "r" (compare) \
273 : "memory", "cc"); \
274 return (previous == compare); \
275 }
276
277 CK_PR_CAS(32, uint32_t)
CK_PR_CAS(uint,unsigned int)278 CK_PR_CAS(uint, unsigned int)
279 CK_PR_CAS(int, int)
280
281 #undef CK_PR_CAS
282
283 #define CK_PR_FAS(N, M, T, W) \
284 CK_CC_INLINE static T \
285 ck_pr_fas_##N(M *target, T v) \
286 { \
287 T previous; \
288 __asm__ __volatile__("1:" \
289 "l" W "arx %0, 0, %1;" \
290 "st" W "cx. %2, 0, %1;" \
291 "bne- 1b;" \
292 : "=&r" (previous) \
293 : "r" (target), \
294 "r" (v) \
295 : "memory", "cc"); \
296 return (previous); \
297 }
298
299 CK_PR_FAS(64, uint64_t, uint64_t, "d")
300 CK_PR_FAS(32, uint32_t, uint32_t, "w")
301 #ifndef CK_PR_DISABLE_DOUBLE
302 CK_PR_FAS(double, double, double, "d")
303 #endif
304 CK_PR_FAS(ptr, void, void *, "d")
305 CK_PR_FAS(int, int, int, "w")
306 CK_PR_FAS(uint, unsigned int, unsigned int, "w")
307
308 #undef CK_PR_FAS
309
310 #define CK_PR_UNARY(O, N, M, T, I, W) \
311 CK_CC_INLINE static void \
312 ck_pr_##O##_##N(M *target) \
313 { \
314 T previous; \
315 __asm__ __volatile__("1:" \
316 "l" W "arx %0, 0, %1;" \
317 I ";" \
318 "st" W "cx. %0, 0, %1;" \
319 "bne- 1b;" \
320 : "=&r" (previous) \
321 : "r" (target) \
322 : "memory", "cc"); \
323 return; \
324 }
325
326 CK_PR_UNARY(inc, ptr, void, void *, "addic %0, %0, 1", "d")
327 CK_PR_UNARY(dec, ptr, void, void *, "addic %0, %0, -1", "d")
328 CK_PR_UNARY(not, ptr, void, void *, "not %0, %0", "d")
329 CK_PR_UNARY(neg, ptr, void, void *, "neg %0, %0", "d")
330
331 #define CK_PR_UNARY_S(S, T, W) \
332 CK_PR_UNARY(inc, S, T, T, "addic %0, %0, 1", W) \
333 CK_PR_UNARY(dec, S, T, T, "addic %0, %0, -1", W) \
334 CK_PR_UNARY(not, S, T, T, "not %0, %0", W) \
335 CK_PR_UNARY(neg, S, T, T, "neg %0, %0", W)
336
337 CK_PR_UNARY_S(64, uint64_t, "d")
338 CK_PR_UNARY_S(32, uint32_t, "w")
339 CK_PR_UNARY_S(uint, unsigned int, "w")
340 CK_PR_UNARY_S(int, int, "w")
341
342 #undef CK_PR_UNARY_S
343 #undef CK_PR_UNARY
344
345 #define CK_PR_BINARY(O, N, M, T, I, W) \
346 CK_CC_INLINE static void \
347 ck_pr_##O##_##N(M *target, T delta) \
348 { \
349 T previous; \
350 __asm__ __volatile__("1:" \
351 "l" W "arx %0, 0, %1;" \
352 I " %0, %2, %0;" \
353 "st" W "cx. %0, 0, %1;" \
354 "bne- 1b;" \
355 : "=&r" (previous) \
356 : "r" (target), \
357 "r" (delta) \
358 : "memory", "cc"); \
359 return; \
360 }
361
362 CK_PR_BINARY(and, ptr, void, uintptr_t, "and", "d")
363 CK_PR_BINARY(add, ptr, void, uintptr_t, "add", "d")
364 CK_PR_BINARY(or, ptr, void, uintptr_t, "or", "d")
365 CK_PR_BINARY(sub, ptr, void, uintptr_t, "sub", "d")
366 CK_PR_BINARY(xor, ptr, void, uintptr_t, "xor", "d")
367
368 #define CK_PR_BINARY_S(S, T, W) \
369 CK_PR_BINARY(and, S, T, T, "and", W) \
370 CK_PR_BINARY(add, S, T, T, "add", W) \
371 CK_PR_BINARY(or, S, T, T, "or", W) \
372 CK_PR_BINARY(sub, S, T, T, "subf", W) \
373 CK_PR_BINARY(xor, S, T, T, "xor", W)
374
375 CK_PR_BINARY_S(64, uint64_t, "d")
376 CK_PR_BINARY_S(32, uint32_t, "w")
377 CK_PR_BINARY_S(uint, unsigned int, "w")
378 CK_PR_BINARY_S(int, int, "w")
379
380 #undef CK_PR_BINARY_S
381 #undef CK_PR_BINARY
382
383 CK_CC_INLINE static void *
384 ck_pr_faa_ptr(void *target, uintptr_t delta)
385 {
386 uintptr_t previous, r;
387
388 __asm__ __volatile__("1:"
389 "ldarx %0, 0, %2;"
390 "add %1, %3, %0;"
391 "stdcx. %1, 0, %2;"
392 "bne- 1b;"
393 : "=&r" (previous),
394 "=&r" (r)
395 : "r" (target),
396 "r" (delta)
397 : "memory", "cc");
398
399 return (void *)(previous);
400 }
401
402 #define CK_PR_FAA(S, T, W) \
403 CK_CC_INLINE static T \
404 ck_pr_faa_##S(T *target, T delta) \
405 { \
406 T previous, r; \
407 __asm__ __volatile__("1:" \
408 "l" W "arx %0, 0, %2;" \
409 "add %1, %3, %0;" \
410 "st" W "cx. %1, 0, %2;" \
411 "bne- 1b;" \
412 : "=&r" (previous), \
413 "=&r" (r) \
414 : "r" (target), \
415 "r" (delta) \
416 : "memory", "cc"); \
417 return (previous); \
418 }
419
420 CK_PR_FAA(64, uint64_t, "d")
421 CK_PR_FAA(32, uint32_t, "w")
422 CK_PR_FAA(uint, unsigned int, "w")
423 CK_PR_FAA(int, int, "w")
424
425 #undef CK_PR_FAA
426
427 #endif /* CK_PR_PPC64_H */
428