1 /*
2 * Copyright 2009-2015 Samy Al Bahra.
3 * Copyright 2011 David Joseph.
4 * All rights reserved.
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #ifndef CK_PR_H
29 #define CK_PR_H
30
31 #include <ck_cc.h>
32 #include <ck_limits.h>
33 #include <ck_md.h>
34 #include <ck_stdint.h>
35 #include <ck_stdbool.h>
36
37 /*
38 * Default to using builtins for clang analyzer, coverity, and sparse:
39 * inline assembly is often too opaque for useful analysis. Override
40 * the defaults by defining CK_USE_CC_BUILTINS=0 or 1.
41 */
42 #if !defined(CK_USE_CC_BUILTINS)
43 #if defined(__clang_analyzer__) || defined(__COVERITY__) || defined(__CHECKER__)
44 #define CK_USE_CC_BUILTINS 1
45 #else
46 #define CK_USE_CC_BUILTINS 0
47 #endif
48 #endif
49
50 #if !CK_USE_CC_BUILTINS
51 #if defined(__x86_64__)
52 #include "gcc/x86_64/ck_pr.h"
53 #elif defined(__x86__)
54 #include "gcc/x86/ck_pr.h"
55 #elif defined(__sparcv9__)
56 #include "gcc/sparcv9/ck_pr.h"
57 #elif defined(__ppc64__)
58 #include "gcc/ppc64/ck_pr.h"
59 #elif defined(__s390x__)
60 #include "gcc/s390x/ck_pr.h"
61 #elif defined(__ppc__)
62 #include "gcc/ppc/ck_pr.h"
63 #elif defined(__arm__)
64 #include "gcc/arm/ck_pr.h"
65 #elif defined(__aarch64__)
66 #include "gcc/aarch64/ck_pr.h"
67 #elif !defined(__GNUC__)
68 #error Your platform is unsupported
69 #endif
70 #endif /* !CK_USE_CC_BUILTINS */
71
72 #if defined(__GNUC__)
73 #include "gcc/ck_pr.h"
74 #endif
75
76 #define CK_PR_FENCE_EMIT(T) \
77 CK_CC_INLINE static void \
78 ck_pr_fence_##T(void) \
79 { \
80 ck_pr_fence_strict_##T(); \
81 return; \
82 }
83 #define CK_PR_FENCE_NOOP(T) \
84 CK_CC_INLINE static void \
85 ck_pr_fence_##T(void) \
86 { \
87 ck_pr_barrier(); \
88 return; \
89 }
90
91 /*
92 * None of the currently supported platforms allow for data-dependent
93 * load ordering.
94 */
95 CK_PR_FENCE_NOOP(load_depends)
96 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends
97
98 /*
99 * In memory models where atomic operations do not have serializing
100 * effects, atomic read-modify-write operations are modeled as stores.
101 */
102 #if defined(CK_MD_RMO)
103 /*
104 * Only stores to the same location have a global
105 * ordering.
106 */
CK_PR_FENCE_EMIT(atomic)107 CK_PR_FENCE_EMIT(atomic)
108 CK_PR_FENCE_EMIT(atomic_load)
109 CK_PR_FENCE_EMIT(atomic_store)
110 CK_PR_FENCE_EMIT(store_atomic)
111 CK_PR_FENCE_EMIT(load_atomic)
112 CK_PR_FENCE_EMIT(load_store)
113 CK_PR_FENCE_EMIT(store_load)
114 CK_PR_FENCE_EMIT(load)
115 CK_PR_FENCE_EMIT(store)
116 CK_PR_FENCE_EMIT(memory)
117 CK_PR_FENCE_EMIT(acquire)
118 CK_PR_FENCE_EMIT(release)
119 CK_PR_FENCE_EMIT(acqrel)
120 CK_PR_FENCE_EMIT(lock)
121 CK_PR_FENCE_EMIT(unlock)
122 #elif defined(CK_MD_PSO)
123 /*
124 * Anything can be re-ordered with respect to stores.
125 * Otherwise, loads are executed in-order.
126 */
127 CK_PR_FENCE_EMIT(atomic)
128 CK_PR_FENCE_NOOP(atomic_load)
129 CK_PR_FENCE_EMIT(atomic_store)
130 CK_PR_FENCE_EMIT(store_atomic)
131 CK_PR_FENCE_NOOP(load_atomic)
132 CK_PR_FENCE_EMIT(load_store)
133 CK_PR_FENCE_EMIT(store_load)
134 CK_PR_FENCE_NOOP(load)
135 CK_PR_FENCE_EMIT(store)
136 CK_PR_FENCE_EMIT(memory)
137 CK_PR_FENCE_EMIT(acquire)
138 CK_PR_FENCE_EMIT(release)
139 CK_PR_FENCE_EMIT(acqrel)
140 CK_PR_FENCE_EMIT(lock)
141 CK_PR_FENCE_EMIT(unlock)
142 #elif defined(CK_MD_TSO)
143 /*
144 * Only loads are re-ordered and only with respect to
145 * prior stores. Atomic operations are serializing.
146 */
147 CK_PR_FENCE_NOOP(atomic)
148 CK_PR_FENCE_NOOP(atomic_load)
149 CK_PR_FENCE_NOOP(atomic_store)
150 CK_PR_FENCE_NOOP(store_atomic)
151 CK_PR_FENCE_NOOP(load_atomic)
152 CK_PR_FENCE_NOOP(load_store)
153 CK_PR_FENCE_EMIT(store_load)
154 CK_PR_FENCE_NOOP(load)
155 CK_PR_FENCE_NOOP(store)
156 CK_PR_FENCE_EMIT(memory)
157 CK_PR_FENCE_NOOP(acquire)
158 CK_PR_FENCE_NOOP(release)
159 CK_PR_FENCE_NOOP(acqrel)
160 CK_PR_FENCE_NOOP(lock)
161 CK_PR_FENCE_NOOP(unlock)
162 #else
163 #error "No memory model has been defined."
164 #endif /* CK_MD_TSO */
165
166 #undef CK_PR_FENCE_EMIT
167 #undef CK_PR_FENCE_NOOP
168
169 #ifndef CK_F_PR_RFO
170 #define CK_F_PR_RFO
171 CK_CC_INLINE static void
172 ck_pr_rfo(const void *m)
173 {
174
175 (void)m;
176 return;
177 }
178 #endif /* CK_F_PR_RFO */
179
180 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \
181 ck_pr_md_store_##TYPE( \
182 ((void)sizeof(*(DST) = (VAL)), (DST)), \
183 (VAL))
184
185 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr)
186 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char)
187 #ifndef CK_PR_DISABLE_DOUBLE
188 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double)
189 #endif
190 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint)
191 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int)
192 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32)
193 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16)
194 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8)
195
196 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL))
197
198 #ifdef CK_F_PR_LOAD_64
199 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64)
200 #endif /* CK_F_PR_LOAD_64 */
201
202 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC))
203 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC))
204
205 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC))
206 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char)
207 #ifndef CK_PR_DISABLE_DOUBLE
208 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double)
209 #endif
210 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint)
211 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int)
212 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32)
213 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16)
214 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8)
215
216 #ifdef CK_F_PR_LOAD_64
217 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64)
218 #endif /* CK_F_PR_LOAD_64 */
219
220 #define CK_PR_BIN(K, S, M, T, P, C) \
221 CK_CC_INLINE static void \
222 ck_pr_##K##_##S(M *target, T value) \
223 { \
224 T previous; \
225 C punt; \
226 punt = ck_pr_md_load_##S(target); \
227 previous = (T)punt; \
228 while (ck_pr_cas_##S##_value(target, \
229 (C)previous, \
230 (C)(previous P value), \
231 &previous) == false) \
232 ck_pr_stall(); \
233 \
234 return; \
235 }
236
237 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T)
238
239 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
240
241 #ifndef CK_F_PR_ADD_CHAR
242 #define CK_F_PR_ADD_CHAR
243 CK_PR_BIN_S(add, char, char, +)
244 #endif /* CK_F_PR_ADD_CHAR */
245
246 #ifndef CK_F_PR_SUB_CHAR
247 #define CK_F_PR_SUB_CHAR
248 CK_PR_BIN_S(sub, char, char, -)
249 #endif /* CK_F_PR_SUB_CHAR */
250
251 #ifndef CK_F_PR_AND_CHAR
252 #define CK_F_PR_AND_CHAR
253 CK_PR_BIN_S(and, char, char, &)
254 #endif /* CK_F_PR_AND_CHAR */
255
256 #ifndef CK_F_PR_XOR_CHAR
257 #define CK_F_PR_XOR_CHAR
258 CK_PR_BIN_S(xor, char, char, ^)
259 #endif /* CK_F_PR_XOR_CHAR */
260
261 #ifndef CK_F_PR_OR_CHAR
262 #define CK_F_PR_OR_CHAR
263 CK_PR_BIN_S(or, char, char, |)
264 #endif /* CK_F_PR_OR_CHAR */
265
266 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
267
268 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
269
270 #ifndef CK_F_PR_ADD_INT
271 #define CK_F_PR_ADD_INT
272 CK_PR_BIN_S(add, int, int, +)
273 #endif /* CK_F_PR_ADD_INT */
274
275 #ifndef CK_F_PR_SUB_INT
276 #define CK_F_PR_SUB_INT
277 CK_PR_BIN_S(sub, int, int, -)
278 #endif /* CK_F_PR_SUB_INT */
279
280 #ifndef CK_F_PR_AND_INT
281 #define CK_F_PR_AND_INT
282 CK_PR_BIN_S(and, int, int, &)
283 #endif /* CK_F_PR_AND_INT */
284
285 #ifndef CK_F_PR_XOR_INT
286 #define CK_F_PR_XOR_INT
287 CK_PR_BIN_S(xor, int, int, ^)
288 #endif /* CK_F_PR_XOR_INT */
289
290 #ifndef CK_F_PR_OR_INT
291 #define CK_F_PR_OR_INT
292 CK_PR_BIN_S(or, int, int, |)
293 #endif /* CK_F_PR_OR_INT */
294
295 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
296
297 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
298 !defined(CK_PR_DISABLE_DOUBLE)
299
300 #ifndef CK_F_PR_ADD_DOUBLE
301 #define CK_F_PR_ADD_DOUBLE
302 CK_PR_BIN_S(add, double, double, +)
303 #endif /* CK_F_PR_ADD_DOUBLE */
304
305 #ifndef CK_F_PR_SUB_DOUBLE
306 #define CK_F_PR_SUB_DOUBLE
307 CK_PR_BIN_S(sub, double, double, -)
308 #endif /* CK_F_PR_SUB_DOUBLE */
309
310 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
311
312 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
313
314 #ifndef CK_F_PR_ADD_UINT
315 #define CK_F_PR_ADD_UINT
316 CK_PR_BIN_S(add, uint, unsigned int, +)
317 #endif /* CK_F_PR_ADD_UINT */
318
319 #ifndef CK_F_PR_SUB_UINT
320 #define CK_F_PR_SUB_UINT
321 CK_PR_BIN_S(sub, uint, unsigned int, -)
322 #endif /* CK_F_PR_SUB_UINT */
323
324 #ifndef CK_F_PR_AND_UINT
325 #define CK_F_PR_AND_UINT
326 CK_PR_BIN_S(and, uint, unsigned int, &)
327 #endif /* CK_F_PR_AND_UINT */
328
329 #ifndef CK_F_PR_XOR_UINT
330 #define CK_F_PR_XOR_UINT
331 CK_PR_BIN_S(xor, uint, unsigned int, ^)
332 #endif /* CK_F_PR_XOR_UINT */
333
334 #ifndef CK_F_PR_OR_UINT
335 #define CK_F_PR_OR_UINT
336 CK_PR_BIN_S(or, uint, unsigned int, |)
337 #endif /* CK_F_PR_OR_UINT */
338
339 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
340
341 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
342
343 #ifndef CK_F_PR_ADD_PTR
344 #define CK_F_PR_ADD_PTR
345 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *)
346 #endif /* CK_F_PR_ADD_PTR */
347
348 #ifndef CK_F_PR_SUB_PTR
349 #define CK_F_PR_SUB_PTR
350 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *)
351 #endif /* CK_F_PR_SUB_PTR */
352
353 #ifndef CK_F_PR_AND_PTR
354 #define CK_F_PR_AND_PTR
355 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *)
356 #endif /* CK_F_PR_AND_PTR */
357
358 #ifndef CK_F_PR_XOR_PTR
359 #define CK_F_PR_XOR_PTR
360 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *)
361 #endif /* CK_F_PR_XOR_PTR */
362
363 #ifndef CK_F_PR_OR_PTR
364 #define CK_F_PR_OR_PTR
365 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *)
366 #endif /* CK_F_PR_OR_PTR */
367
368 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
369
370 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
371
372 #ifndef CK_F_PR_ADD_64
373 #define CK_F_PR_ADD_64
374 CK_PR_BIN_S(add, 64, uint64_t, +)
375 #endif /* CK_F_PR_ADD_64 */
376
377 #ifndef CK_F_PR_SUB_64
378 #define CK_F_PR_SUB_64
379 CK_PR_BIN_S(sub, 64, uint64_t, -)
380 #endif /* CK_F_PR_SUB_64 */
381
382 #ifndef CK_F_PR_AND_64
383 #define CK_F_PR_AND_64
384 CK_PR_BIN_S(and, 64, uint64_t, &)
385 #endif /* CK_F_PR_AND_64 */
386
387 #ifndef CK_F_PR_XOR_64
388 #define CK_F_PR_XOR_64
389 CK_PR_BIN_S(xor, 64, uint64_t, ^)
390 #endif /* CK_F_PR_XOR_64 */
391
392 #ifndef CK_F_PR_OR_64
393 #define CK_F_PR_OR_64
394 CK_PR_BIN_S(or, 64, uint64_t, |)
395 #endif /* CK_F_PR_OR_64 */
396
397 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
398
399 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
400
401 #ifndef CK_F_PR_ADD_32
402 #define CK_F_PR_ADD_32
403 CK_PR_BIN_S(add, 32, uint32_t, +)
404 #endif /* CK_F_PR_ADD_32 */
405
406 #ifndef CK_F_PR_SUB_32
407 #define CK_F_PR_SUB_32
408 CK_PR_BIN_S(sub, 32, uint32_t, -)
409 #endif /* CK_F_PR_SUB_32 */
410
411 #ifndef CK_F_PR_AND_32
412 #define CK_F_PR_AND_32
413 CK_PR_BIN_S(and, 32, uint32_t, &)
414 #endif /* CK_F_PR_AND_32 */
415
416 #ifndef CK_F_PR_XOR_32
417 #define CK_F_PR_XOR_32
418 CK_PR_BIN_S(xor, 32, uint32_t, ^)
419 #endif /* CK_F_PR_XOR_32 */
420
421 #ifndef CK_F_PR_OR_32
422 #define CK_F_PR_OR_32
423 CK_PR_BIN_S(or, 32, uint32_t, |)
424 #endif /* CK_F_PR_OR_32 */
425
426 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
427
428 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
429
430 #ifndef CK_F_PR_ADD_16
431 #define CK_F_PR_ADD_16
432 CK_PR_BIN_S(add, 16, uint16_t, +)
433 #endif /* CK_F_PR_ADD_16 */
434
435 #ifndef CK_F_PR_SUB_16
436 #define CK_F_PR_SUB_16
437 CK_PR_BIN_S(sub, 16, uint16_t, -)
438 #endif /* CK_F_PR_SUB_16 */
439
440 #ifndef CK_F_PR_AND_16
441 #define CK_F_PR_AND_16
442 CK_PR_BIN_S(and, 16, uint16_t, &)
443 #endif /* CK_F_PR_AND_16 */
444
445 #ifndef CK_F_PR_XOR_16
446 #define CK_F_PR_XOR_16
447 CK_PR_BIN_S(xor, 16, uint16_t, ^)
448 #endif /* CK_F_PR_XOR_16 */
449
450 #ifndef CK_F_PR_OR_16
451 #define CK_F_PR_OR_16
452 CK_PR_BIN_S(or, 16, uint16_t, |)
453 #endif /* CK_F_PR_OR_16 */
454
455 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
456
457 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
458
459 #ifndef CK_F_PR_ADD_8
460 #define CK_F_PR_ADD_8
461 CK_PR_BIN_S(add, 8, uint8_t, +)
462 #endif /* CK_F_PR_ADD_8 */
463
464 #ifndef CK_F_PR_SUB_8
465 #define CK_F_PR_SUB_8
466 CK_PR_BIN_S(sub, 8, uint8_t, -)
467 #endif /* CK_F_PR_SUB_8 */
468
469 #ifndef CK_F_PR_AND_8
470 #define CK_F_PR_AND_8
471 CK_PR_BIN_S(and, 8, uint8_t, &)
472 #endif /* CK_F_PR_AND_8 */
473
474 #ifndef CK_F_PR_XOR_8
475 #define CK_F_PR_XOR_8
476 CK_PR_BIN_S(xor, 8, uint8_t, ^)
477 #endif /* CK_F_PR_XOR_8 */
478
479 #ifndef CK_F_PR_OR_8
480 #define CK_F_PR_OR_8
481 CK_PR_BIN_S(or, 8, uint8_t, |)
482 #endif /* CK_F_PR_OR_8 */
483
484 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
485
486 #undef CK_PR_BIN_S
487 #undef CK_PR_BIN
488
489 #define CK_PR_BTX(K, S, M, T, P, C, R) \
490 CK_CC_INLINE static bool \
491 ck_pr_##K##_##S(M *target, unsigned int offset) \
492 { \
493 T previous; \
494 C punt; \
495 punt = ck_pr_md_load_##S(target); \
496 previous = (T)punt; \
497 while (ck_pr_cas_##S##_value(target, (C)previous, \
498 (C)(previous P (R ((T)1 << offset))), &previous) == false) \
499 ck_pr_stall(); \
500 return ((previous >> offset) & 1); \
501 }
502
503 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R)
504
505 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
506
507 #ifndef CK_F_PR_BTC_INT
508 #define CK_F_PR_BTC_INT
509 CK_PR_BTX_S(btc, int, int, ^,)
510 #endif /* CK_F_PR_BTC_INT */
511
512 #ifndef CK_F_PR_BTR_INT
513 #define CK_F_PR_BTR_INT
514 CK_PR_BTX_S(btr, int, int, &, ~)
515 #endif /* CK_F_PR_BTR_INT */
516
517 #ifndef CK_F_PR_BTS_INT
518 #define CK_F_PR_BTS_INT
519 CK_PR_BTX_S(bts, int, int, |,)
520 #endif /* CK_F_PR_BTS_INT */
521
522 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
523
524 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
525
526 #ifndef CK_F_PR_BTC_UINT
527 #define CK_F_PR_BTC_UINT
528 CK_PR_BTX_S(btc, uint, unsigned int, ^,)
529 #endif /* CK_F_PR_BTC_UINT */
530
531 #ifndef CK_F_PR_BTR_UINT
532 #define CK_F_PR_BTR_UINT
533 CK_PR_BTX_S(btr, uint, unsigned int, &, ~)
534 #endif /* CK_F_PR_BTR_UINT */
535
536 #ifndef CK_F_PR_BTS_UINT
537 #define CK_F_PR_BTS_UINT
538 CK_PR_BTX_S(bts, uint, unsigned int, |,)
539 #endif /* CK_F_PR_BTS_UINT */
540
541 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
542
543 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
544
545 #ifndef CK_F_PR_BTC_PTR
546 #define CK_F_PR_BTC_PTR
547 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,)
548 #endif /* CK_F_PR_BTC_PTR */
549
550 #ifndef CK_F_PR_BTR_PTR
551 #define CK_F_PR_BTR_PTR
552 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~)
553 #endif /* CK_F_PR_BTR_PTR */
554
555 #ifndef CK_F_PR_BTS_PTR
556 #define CK_F_PR_BTS_PTR
557 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,)
558 #endif /* CK_F_PR_BTS_PTR */
559
560 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
561
562 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
563
564 #ifndef CK_F_PR_BTC_64
565 #define CK_F_PR_BTC_64
566 CK_PR_BTX_S(btc, 64, uint64_t, ^,)
567 #endif /* CK_F_PR_BTC_64 */
568
569 #ifndef CK_F_PR_BTR_64
570 #define CK_F_PR_BTR_64
571 CK_PR_BTX_S(btr, 64, uint64_t, &, ~)
572 #endif /* CK_F_PR_BTR_64 */
573
574 #ifndef CK_F_PR_BTS_64
575 #define CK_F_PR_BTS_64
576 CK_PR_BTX_S(bts, 64, uint64_t, |,)
577 #endif /* CK_F_PR_BTS_64 */
578
579 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
580
581 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
582
583 #ifndef CK_F_PR_BTC_32
584 #define CK_F_PR_BTC_32
585 CK_PR_BTX_S(btc, 32, uint32_t, ^,)
586 #endif /* CK_F_PR_BTC_32 */
587
588 #ifndef CK_F_PR_BTR_32
589 #define CK_F_PR_BTR_32
590 CK_PR_BTX_S(btr, 32, uint32_t, &, ~)
591 #endif /* CK_F_PR_BTR_32 */
592
593 #ifndef CK_F_PR_BTS_32
594 #define CK_F_PR_BTS_32
595 CK_PR_BTX_S(bts, 32, uint32_t, |,)
596 #endif /* CK_F_PR_BTS_32 */
597
598 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
599
600 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
601
602 #ifndef CK_F_PR_BTC_16
603 #define CK_F_PR_BTC_16
604 CK_PR_BTX_S(btc, 16, uint16_t, ^,)
605 #endif /* CK_F_PR_BTC_16 */
606
607 #ifndef CK_F_PR_BTR_16
608 #define CK_F_PR_BTR_16
609 CK_PR_BTX_S(btr, 16, uint16_t, &, ~)
610 #endif /* CK_F_PR_BTR_16 */
611
612 #ifndef CK_F_PR_BTS_16
613 #define CK_F_PR_BTS_16
614 CK_PR_BTX_S(bts, 16, uint16_t, |,)
615 #endif /* CK_F_PR_BTS_16 */
616
617 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
618
619 #undef CK_PR_BTX_S
620 #undef CK_PR_BTX
621
622 #define CK_PR_UNARY(K, X, S, M, T) \
623 CK_CC_INLINE static void \
624 ck_pr_##K##_##S(M *target) \
625 { \
626 ck_pr_##X##_##S(target, (T)1); \
627 return; \
628 }
629
630 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \
631 CK_CC_INLINE static bool \
632 ck_pr_##K##_##S##_is_zero(M *target) \
633 { \
634 T previous; \
635 C punt; \
636 punt = (C)ck_pr_md_load_##S(target); \
637 previous = (T)punt; \
638 while (ck_pr_cas_##S##_value(target, \
639 (C)previous, \
640 (C)(previous P 1), \
641 &previous) == false) \
642 ck_pr_stall(); \
643 return previous == (T)Z; \
644 }
645
646 #define CK_PR_UNARY_Z_STUB(K, S, M) \
647 CK_CC_INLINE static void \
648 ck_pr_##K##_##S##_zero(M *target, bool *zero) \
649 { \
650 *zero = ck_pr_##K##_##S##_is_zero(target); \
651 return; \
652 }
653
654 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M)
655 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) \
656 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \
657 CK_PR_UNARY_Z_STUB(K, S, M)
658
659 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
660
661 #ifndef CK_F_PR_INC_CHAR
662 #define CK_F_PR_INC_CHAR
663 CK_PR_UNARY_S(inc, add, char, char)
664 #endif /* CK_F_PR_INC_CHAR */
665
666 #ifndef CK_F_PR_INC_CHAR_ZERO
667 #define CK_F_PR_INC_CHAR_ZERO
668 CK_PR_UNARY_Z_S(inc, char, char, +, -1)
669 #else
670 CK_PR_UNARY_Z_STUB(inc, char, char)
671 #endif /* CK_F_PR_INC_CHAR_ZERO */
672
673 #ifndef CK_F_PR_DEC_CHAR
674 #define CK_F_PR_DEC_CHAR
675 CK_PR_UNARY_S(dec, sub, char, char)
676 #endif /* CK_F_PR_DEC_CHAR */
677
678 #ifndef CK_F_PR_DEC_CHAR_ZERO
679 #define CK_F_PR_DEC_CHAR_ZERO
680 CK_PR_UNARY_Z_S(dec, char, char, -, 1)
681 #else
682 CK_PR_UNARY_Z_STUB(dec, char, char)
683 #endif /* CK_F_PR_DEC_CHAR_ZERO */
684
685 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
686
687 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
688
689 #ifndef CK_F_PR_INC_INT
690 #define CK_F_PR_INC_INT
691 CK_PR_UNARY_S(inc, add, int, int)
692 #endif /* CK_F_PR_INC_INT */
693
694 #ifndef CK_F_PR_INC_INT_ZERO
695 #define CK_F_PR_INC_INT_ZERO
696 CK_PR_UNARY_Z_S(inc, int, int, +, -1)
697 #else
698 CK_PR_UNARY_Z_STUB(inc, int, int)
699 #endif /* CK_F_PR_INC_INT_ZERO */
700
701 #ifndef CK_F_PR_DEC_INT
702 #define CK_F_PR_DEC_INT
703 CK_PR_UNARY_S(dec, sub, int, int)
704 #endif /* CK_F_PR_DEC_INT */
705
706 #ifndef CK_F_PR_DEC_INT_ZERO
707 #define CK_F_PR_DEC_INT_ZERO
708 CK_PR_UNARY_Z_S(dec, int, int, -, 1)
709 #else
710 CK_PR_UNARY_Z_STUB(dec, int, int)
711 #endif /* CK_F_PR_DEC_INT_ZERO */
712
713 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
714
715 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
716 !defined(CK_PR_DISABLE_DOUBLE)
717
718 #ifndef CK_F_PR_INC_DOUBLE
719 #define CK_F_PR_INC_DOUBLE
720 CK_PR_UNARY_S(inc, add, double, double)
721 #endif /* CK_F_PR_INC_DOUBLE */
722
723 #ifndef CK_F_PR_DEC_DOUBLE
724 #define CK_F_PR_DEC_DOUBLE
725 CK_PR_UNARY_S(dec, sub, double, double)
726 #endif /* CK_F_PR_DEC_DOUBLE */
727
728 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
729
730 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
731
732 #ifndef CK_F_PR_INC_UINT
733 #define CK_F_PR_INC_UINT
734 CK_PR_UNARY_S(inc, add, uint, unsigned int)
735 #endif /* CK_F_PR_INC_UINT */
736
737 #ifndef CK_F_PR_INC_UINT_ZERO
738 #define CK_F_PR_INC_UINT_ZERO
739 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX)
740 #else
741 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int)
742 #endif /* CK_F_PR_INC_UINT_ZERO */
743
744 #ifndef CK_F_PR_DEC_UINT
745 #define CK_F_PR_DEC_UINT
746 CK_PR_UNARY_S(dec, sub, uint, unsigned int)
747 #endif /* CK_F_PR_DEC_UINT */
748
749 #ifndef CK_F_PR_DEC_UINT_ZERO
750 #define CK_F_PR_DEC_UINT_ZERO
751 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1)
752 #else
753 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int)
754 #endif /* CK_F_PR_DEC_UINT_ZERO */
755
756 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
757
758 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
759
760 #ifndef CK_F_PR_INC_PTR
761 #define CK_F_PR_INC_PTR
762 CK_PR_UNARY(inc, add, ptr, void, uintptr_t)
763 #endif /* CK_F_PR_INC_PTR */
764
765 #ifndef CK_F_PR_INC_PTR_ZERO
766 #define CK_F_PR_INC_PTR_ZERO
767 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX)
768 #else
769 CK_PR_UNARY_Z_STUB(inc, ptr, void)
770 #endif /* CK_F_PR_INC_PTR_ZERO */
771
772 #ifndef CK_F_PR_DEC_PTR
773 #define CK_F_PR_DEC_PTR
774 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t)
775 #endif /* CK_F_PR_DEC_PTR */
776
777 #ifndef CK_F_PR_DEC_PTR_ZERO
778 #define CK_F_PR_DEC_PTR_ZERO
779 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1)
780 #else
781 CK_PR_UNARY_Z_STUB(dec, ptr, void)
782 #endif /* CK_F_PR_DEC_PTR_ZERO */
783
784 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
785
786 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
787
788 #ifndef CK_F_PR_INC_64
789 #define CK_F_PR_INC_64
790 CK_PR_UNARY_S(inc, add, 64, uint64_t)
791 #endif /* CK_F_PR_INC_64 */
792
793 #ifndef CK_F_PR_INC_64_ZERO
794 #define CK_F_PR_INC_64_ZERO
795 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX)
796 #else
797 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t)
798 #endif /* CK_F_PR_INC_64_ZERO */
799
800 #ifndef CK_F_PR_DEC_64
801 #define CK_F_PR_DEC_64
802 CK_PR_UNARY_S(dec, sub, 64, uint64_t)
803 #endif /* CK_F_PR_DEC_64 */
804
805 #ifndef CK_F_PR_DEC_64_ZERO
806 #define CK_F_PR_DEC_64_ZERO
807 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1)
808 #else
809 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t)
810 #endif /* CK_F_PR_DEC_64_ZERO */
811
812 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
813
814 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
815
816 #ifndef CK_F_PR_INC_32
817 #define CK_F_PR_INC_32
818 CK_PR_UNARY_S(inc, add, 32, uint32_t)
819 #endif /* CK_F_PR_INC_32 */
820
821 #ifndef CK_F_PR_INC_32_ZERO
822 #define CK_F_PR_INC_32_ZERO
823 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX)
824 #else
825 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t)
826 #endif /* CK_F_PR_INC_32_ZERO */
827
828 #ifndef CK_F_PR_DEC_32
829 #define CK_F_PR_DEC_32
830 CK_PR_UNARY_S(dec, sub, 32, uint32_t)
831 #endif /* CK_F_PR_DEC_32 */
832
833 #ifndef CK_F_PR_DEC_32_ZERO
834 #define CK_F_PR_DEC_32_ZERO
835 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1)
836 #else
837 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t)
838 #endif /* CK_F_PR_DEC_32_ZERO */
839
840 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
841
842 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
843
844 #ifndef CK_F_PR_INC_16
845 #define CK_F_PR_INC_16
846 CK_PR_UNARY_S(inc, add, 16, uint16_t)
847 #endif /* CK_F_PR_INC_16 */
848
849 #ifndef CK_F_PR_INC_16_ZERO
850 #define CK_F_PR_INC_16_ZERO
851 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX)
852 #else
853 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t)
854 #endif /* CK_F_PR_INC_16_ZERO */
855
856 #ifndef CK_F_PR_DEC_16
857 #define CK_F_PR_DEC_16
858 CK_PR_UNARY_S(dec, sub, 16, uint16_t)
859 #endif /* CK_F_PR_DEC_16 */
860
861 #ifndef CK_F_PR_DEC_16_ZERO
862 #define CK_F_PR_DEC_16_ZERO
863 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1)
864 #else
865 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t)
866 #endif /* CK_F_PR_DEC_16_ZERO */
867
868 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
869
870 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
871
872 #ifndef CK_F_PR_INC_8
873 #define CK_F_PR_INC_8
874 CK_PR_UNARY_S(inc, add, 8, uint8_t)
875 #endif /* CK_F_PR_INC_8 */
876
877 #ifndef CK_F_PR_INC_8_ZERO
878 #define CK_F_PR_INC_8_ZERO
879 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX)
880 #else
881 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t)
882 #endif /* CK_F_PR_INC_8_ZERO */
883
884 #ifndef CK_F_PR_DEC_8
885 #define CK_F_PR_DEC_8
886 CK_PR_UNARY_S(dec, sub, 8, uint8_t)
887 #endif /* CK_F_PR_DEC_8 */
888
889 #ifndef CK_F_PR_DEC_8_ZERO
890 #define CK_F_PR_DEC_8_ZERO
891 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1)
892 #else
893 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t)
894 #endif /* CK_F_PR_DEC_8_ZERO */
895
896 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
897
898 #undef CK_PR_UNARY_Z_S
899 #undef CK_PR_UNARY_S
900 #undef CK_PR_UNARY_Z
901 #undef CK_PR_UNARY
902
903 #define CK_PR_N(K, S, M, T, P, C) \
904 CK_CC_INLINE static void \
905 ck_pr_##K##_##S(M *target) \
906 { \
907 T previous; \
908 C punt; \
909 punt = (C)ck_pr_md_load_##S(target); \
910 previous = (T)punt; \
911 while (ck_pr_cas_##S##_value(target, \
912 (C)previous, \
913 (C)(P previous), \
914 &previous) == false) \
915 ck_pr_stall(); \
916 \
917 return; \
918 }
919
920 #define CK_PR_N_Z(S, M, T, C) \
921 CK_CC_INLINE static void \
922 ck_pr_neg_##S##_zero(M *target, bool *zero) \
923 { \
924 T previous; \
925 C punt; \
926 punt = (C)ck_pr_md_load_##S(target); \
927 previous = (T)punt; \
928 while (ck_pr_cas_##S##_value(target, \
929 (C)previous, \
930 (C)(-previous), \
931 &previous) == false) \
932 ck_pr_stall(); \
933 \
934 *zero = previous == 0; \
935 return; \
936 }
937
938 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M)
939 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M)
940
941 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
942
943 #ifndef CK_F_PR_NOT_CHAR
944 #define CK_F_PR_NOT_CHAR
945 CK_PR_N_S(not, char, char, ~)
946 #endif /* CK_F_PR_NOT_CHAR */
947
948 #ifndef CK_F_PR_NEG_CHAR
949 #define CK_F_PR_NEG_CHAR
950 CK_PR_N_S(neg, char, char, -)
951 #endif /* CK_F_PR_NEG_CHAR */
952
953 #ifndef CK_F_PR_NEG_CHAR_ZERO
954 #define CK_F_PR_NEG_CHAR_ZERO
955 CK_PR_N_Z_S(char, char)
956 #endif /* CK_F_PR_NEG_CHAR_ZERO */
957
958 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
959
960 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
961
962 #ifndef CK_F_PR_NOT_INT
963 #define CK_F_PR_NOT_INT
964 CK_PR_N_S(not, int, int, ~)
965 #endif /* CK_F_PR_NOT_INT */
966
967 #ifndef CK_F_PR_NEG_INT
968 #define CK_F_PR_NEG_INT
969 CK_PR_N_S(neg, int, int, -)
970 #endif /* CK_F_PR_NEG_INT */
971
972 #ifndef CK_F_PR_NEG_INT_ZERO
973 #define CK_F_PR_NEG_INT_ZERO
974 CK_PR_N_Z_S(int, int)
975 #endif /* CK_F_PR_NEG_INT_ZERO */
976
977 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
978
979 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
980 !defined(CK_PR_DISABLE_DOUBLE)
981
982 #ifndef CK_F_PR_NEG_DOUBLE
983 #define CK_F_PR_NEG_DOUBLE
984 CK_PR_N_S(neg, double, double, -)
985 #endif /* CK_F_PR_NEG_DOUBLE */
986
987 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
988
989 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
990
991 #ifndef CK_F_PR_NOT_UINT
992 #define CK_F_PR_NOT_UINT
993 CK_PR_N_S(not, uint, unsigned int, ~)
994 #endif /* CK_F_PR_NOT_UINT */
995
996 #ifndef CK_F_PR_NEG_UINT
997 #define CK_F_PR_NEG_UINT
998 CK_PR_N_S(neg, uint, unsigned int, -)
999 #endif /* CK_F_PR_NEG_UINT */
1000
1001 #ifndef CK_F_PR_NEG_UINT_ZERO
1002 #define CK_F_PR_NEG_UINT_ZERO
1003 CK_PR_N_Z_S(uint, unsigned int)
1004 #endif /* CK_F_PR_NEG_UINT_ZERO */
1005
1006 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1007
1008 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1009
1010 #ifndef CK_F_PR_NOT_PTR
1011 #define CK_F_PR_NOT_PTR
1012 CK_PR_N(not, ptr, void, uintptr_t, ~, void *)
1013 #endif /* CK_F_PR_NOT_PTR */
1014
1015 #ifndef CK_F_PR_NEG_PTR
1016 #define CK_F_PR_NEG_PTR
1017 CK_PR_N(neg, ptr, void, uintptr_t, -, void *)
1018 #endif /* CK_F_PR_NEG_PTR */
1019
1020 #ifndef CK_F_PR_NEG_PTR_ZERO
1021 #define CK_F_PR_NEG_PTR_ZERO
1022 CK_PR_N_Z(ptr, void, uintptr_t, void *)
1023 #endif /* CK_F_PR_NEG_PTR_ZERO */
1024
1025 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1026
1027 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1028
1029 #ifndef CK_F_PR_NOT_64
1030 #define CK_F_PR_NOT_64
1031 CK_PR_N_S(not, 64, uint64_t, ~)
1032 #endif /* CK_F_PR_NOT_64 */
1033
1034 #ifndef CK_F_PR_NEG_64
1035 #define CK_F_PR_NEG_64
1036 CK_PR_N_S(neg, 64, uint64_t, -)
1037 #endif /* CK_F_PR_NEG_64 */
1038
1039 #ifndef CK_F_PR_NEG_64_ZERO
1040 #define CK_F_PR_NEG_64_ZERO
1041 CK_PR_N_Z_S(64, uint64_t)
1042 #endif /* CK_F_PR_NEG_64_ZERO */
1043
1044 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1045
1046 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1047
1048 #ifndef CK_F_PR_NOT_32
1049 #define CK_F_PR_NOT_32
1050 CK_PR_N_S(not, 32, uint32_t, ~)
1051 #endif /* CK_F_PR_NOT_32 */
1052
1053 #ifndef CK_F_PR_NEG_32
1054 #define CK_F_PR_NEG_32
1055 CK_PR_N_S(neg, 32, uint32_t, -)
1056 #endif /* CK_F_PR_NEG_32 */
1057
1058 #ifndef CK_F_PR_NEG_32_ZERO
1059 #define CK_F_PR_NEG_32_ZERO
1060 CK_PR_N_Z_S(32, uint32_t)
1061 #endif /* CK_F_PR_NEG_32_ZERO */
1062
1063 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1064
1065 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1066
1067 #ifndef CK_F_PR_NOT_16
1068 #define CK_F_PR_NOT_16
1069 CK_PR_N_S(not, 16, uint16_t, ~)
1070 #endif /* CK_F_PR_NOT_16 */
1071
1072 #ifndef CK_F_PR_NEG_16
1073 #define CK_F_PR_NEG_16
1074 CK_PR_N_S(neg, 16, uint16_t, -)
1075 #endif /* CK_F_PR_NEG_16 */
1076
1077 #ifndef CK_F_PR_NEG_16_ZERO
1078 #define CK_F_PR_NEG_16_ZERO
1079 CK_PR_N_Z_S(16, uint16_t)
1080 #endif /* CK_F_PR_NEG_16_ZERO */
1081
1082 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1083
1084 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1085
1086 #ifndef CK_F_PR_NOT_8
1087 #define CK_F_PR_NOT_8
1088 CK_PR_N_S(not, 8, uint8_t, ~)
1089 #endif /* CK_F_PR_NOT_8 */
1090
1091 #ifndef CK_F_PR_NEG_8
1092 #define CK_F_PR_NEG_8
1093 CK_PR_N_S(neg, 8, uint8_t, -)
1094 #endif /* CK_F_PR_NEG_8 */
1095
1096 #ifndef CK_F_PR_NEG_8_ZERO
1097 #define CK_F_PR_NEG_8_ZERO
1098 CK_PR_N_Z_S(8, uint8_t)
1099 #endif /* CK_F_PR_NEG_8_ZERO */
1100
1101 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1102
1103 #undef CK_PR_N_Z_S
1104 #undef CK_PR_N_S
1105 #undef CK_PR_N_Z
1106 #undef CK_PR_N
1107
1108 #define CK_PR_FAA(S, M, T, C) \
1109 CK_CC_INLINE static C \
1110 ck_pr_faa_##S(M *target, T delta) \
1111 { \
1112 T previous; \
1113 C punt; \
1114 punt = (C)ck_pr_md_load_##S(target); \
1115 previous = (T)punt; \
1116 while (ck_pr_cas_##S##_value(target, \
1117 (C)previous, \
1118 (C)(previous + delta), \
1119 &previous) == false) \
1120 ck_pr_stall(); \
1121 \
1122 return ((C)previous); \
1123 }
1124
1125 #define CK_PR_FAS(S, M, C) \
1126 CK_CC_INLINE static C \
1127 ck_pr_fas_##S(M *target, C update) \
1128 { \
1129 C previous; \
1130 previous = ck_pr_md_load_##S(target); \
1131 while (ck_pr_cas_##S##_value(target, \
1132 previous, \
1133 update, \
1134 &previous) == false) \
1135 ck_pr_stall(); \
1136 \
1137 return (previous); \
1138 }
1139
1140 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M)
1141 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M)
1142
1143 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE)
1144
1145 #ifndef CK_F_PR_FAA_CHAR
1146 #define CK_F_PR_FAA_CHAR
1147 CK_PR_FAA_S(char, char)
1148 #endif /* CK_F_PR_FAA_CHAR */
1149
1150 #ifndef CK_F_PR_FAS_CHAR
1151 #define CK_F_PR_FAS_CHAR
1152 CK_PR_FAS_S(char, char)
1153 #endif /* CK_F_PR_FAS_CHAR */
1154
1155 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */
1156
1157 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE)
1158
1159 #ifndef CK_F_PR_FAA_INT
1160 #define CK_F_PR_FAA_INT
1161 CK_PR_FAA_S(int, int)
1162 #endif /* CK_F_PR_FAA_INT */
1163
1164 #ifndef CK_F_PR_FAS_INT
1165 #define CK_F_PR_FAS_INT
1166 CK_PR_FAS_S(int, int)
1167 #endif /* CK_F_PR_FAS_INT */
1168
1169 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */
1170
1171 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \
1172 !defined(CK_PR_DISABLE_DOUBLE)
1173
1174 #ifndef CK_F_PR_FAA_DOUBLE
1175 #define CK_F_PR_FAA_DOUBLE
1176 CK_PR_FAA_S(double, double)
1177 #endif /* CK_F_PR_FAA_DOUBLE */
1178
1179 #ifndef CK_F_PR_FAS_DOUBLE
1180 #define CK_F_PR_FAS_DOUBLE
1181 CK_PR_FAS_S(double, double)
1182 #endif /* CK_F_PR_FAS_DOUBLE */
1183
1184 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */
1185
1186 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE)
1187
1188 #ifndef CK_F_PR_FAA_UINT
1189 #define CK_F_PR_FAA_UINT
1190 CK_PR_FAA_S(uint, unsigned int)
1191 #endif /* CK_F_PR_FAA_UINT */
1192
1193 #ifndef CK_F_PR_FAS_UINT
1194 #define CK_F_PR_FAS_UINT
1195 CK_PR_FAS_S(uint, unsigned int)
1196 #endif /* CK_F_PR_FAS_UINT */
1197
1198 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */
1199
1200 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE)
1201
1202 #ifndef CK_F_PR_FAA_PTR
1203 #define CK_F_PR_FAA_PTR
1204 CK_PR_FAA(ptr, void, uintptr_t, void *)
1205 #endif /* CK_F_PR_FAA_PTR */
1206
1207 #ifndef CK_F_PR_FAS_PTR
1208 #define CK_F_PR_FAS_PTR
1209 CK_PR_FAS(ptr, void, void *)
1210 #endif /* CK_F_PR_FAS_PTR */
1211
1212 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */
1213
1214 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE)
1215
1216 #ifndef CK_F_PR_FAA_64
1217 #define CK_F_PR_FAA_64
1218 CK_PR_FAA_S(64, uint64_t)
1219 #endif /* CK_F_PR_FAA_64 */
1220
1221 #ifndef CK_F_PR_FAS_64
1222 #define CK_F_PR_FAS_64
1223 CK_PR_FAS_S(64, uint64_t)
1224 #endif /* CK_F_PR_FAS_64 */
1225
1226 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */
1227
1228 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE)
1229
1230 #ifndef CK_F_PR_FAA_32
1231 #define CK_F_PR_FAA_32
1232 CK_PR_FAA_S(32, uint32_t)
1233 #endif /* CK_F_PR_FAA_32 */
1234
1235 #ifndef CK_F_PR_FAS_32
1236 #define CK_F_PR_FAS_32
1237 CK_PR_FAS_S(32, uint32_t)
1238 #endif /* CK_F_PR_FAS_32 */
1239
1240 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */
1241
1242 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE)
1243
1244 #ifndef CK_F_PR_FAA_16
1245 #define CK_F_PR_FAA_16
1246 CK_PR_FAA_S(16, uint16_t)
1247 #endif /* CK_F_PR_FAA_16 */
1248
1249 #ifndef CK_F_PR_FAS_16
1250 #define CK_F_PR_FAS_16
1251 CK_PR_FAS_S(16, uint16_t)
1252 #endif /* CK_F_PR_FAS_16 */
1253
1254 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */
1255
1256 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE)
1257
1258 #ifndef CK_F_PR_FAA_8
1259 #define CK_F_PR_FAA_8
1260 CK_PR_FAA_S(8, uint8_t)
1261 #endif /* CK_F_PR_FAA_8 */
1262
1263 #ifndef CK_F_PR_FAS_8
1264 #define CK_F_PR_FAS_8
1265 CK_PR_FAS_S(8, uint8_t)
1266 #endif /* CK_F_PR_FAS_8 */
1267
1268 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */
1269
1270 #undef CK_PR_FAA_S
1271 #undef CK_PR_FAS_S
1272 #undef CK_PR_FAA
1273 #undef CK_PR_FAS
1274
1275 #endif /* CK_PR_H */
1276