1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 #ifndef CK_USE_CC_BUILTINS 38 #if defined(__x86_64__) 39 #include "gcc/x86_64/ck_pr.h" 40 #elif defined(__x86__) 41 #include "gcc/x86/ck_pr.h" 42 #elif defined(__sparcv9__) 43 #include "gcc/sparcv9/ck_pr.h" 44 #elif defined(__ppc64__) 45 #include "gcc/ppc64/ck_pr.h" 46 #elif defined(__s390x__) 47 #include "gcc/s390x/ck_pr.h" 48 #elif defined(__ppc__) 49 #include "gcc/ppc/ck_pr.h" 50 #elif defined(__arm__) 51 #if __ARM_ARCH >= 6 52 #include "gcc/arm/ck_pr.h" 53 #else 54 #include "gcc/arm/ck_pr_armv4.h" 55 #endif 56 #elif defined(__aarch64__) 57 #include "gcc/aarch64/ck_pr.h" 58 #elif !defined(__GNUC__) 59 #error Your platform is unsupported 60 #endif 61 #endif /* !CK_USE_CC_BUILTINS */ 62 63 #if defined(__GNUC__) 64 #include "gcc/ck_pr.h" 65 #endif 66 67 #define CK_PR_FENCE_EMIT(T) \ 68 CK_CC_INLINE static void \ 69 ck_pr_fence_##T(void) \ 70 { \ 71 ck_pr_fence_strict_##T(); \ 72 return; \ 73 } 74 #define CK_PR_FENCE_NOOP(T) \ 75 CK_CC_INLINE static void \ 76 ck_pr_fence_##T(void) \ 77 { \ 78 ck_pr_barrier(); \ 79 return; \ 80 } 81 82 /* 83 * None of the currently supported platforms allow for data-dependent 84 * load ordering. 85 */ 86 CK_PR_FENCE_NOOP(load_depends) 87 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 88 89 /* 90 * In memory models where atomic operations do not have serializing 91 * effects, atomic read-modify-write operations are modeled as stores. 92 */ 93 #if defined(CK_MD_RMO) 94 /* 95 * Only stores to the same location have a global 96 * ordering. 97 */ 98 CK_PR_FENCE_EMIT(atomic) 99 CK_PR_FENCE_EMIT(atomic_load) 100 CK_PR_FENCE_EMIT(atomic_store) 101 CK_PR_FENCE_EMIT(store_atomic) 102 CK_PR_FENCE_EMIT(load_atomic) 103 CK_PR_FENCE_EMIT(load_store) 104 CK_PR_FENCE_EMIT(store_load) 105 CK_PR_FENCE_EMIT(load) 106 CK_PR_FENCE_EMIT(store) 107 CK_PR_FENCE_EMIT(memory) 108 CK_PR_FENCE_EMIT(acquire) 109 CK_PR_FENCE_EMIT(release) 110 CK_PR_FENCE_EMIT(acqrel) 111 CK_PR_FENCE_EMIT(lock) 112 CK_PR_FENCE_EMIT(unlock) 113 #elif defined(CK_MD_PSO) 114 /* 115 * Anything can be re-ordered with respect to stores. 116 * Otherwise, loads are executed in-order. 117 */ 118 CK_PR_FENCE_EMIT(atomic) 119 CK_PR_FENCE_NOOP(atomic_load) 120 CK_PR_FENCE_EMIT(atomic_store) 121 CK_PR_FENCE_EMIT(store_atomic) 122 CK_PR_FENCE_NOOP(load_atomic) 123 CK_PR_FENCE_EMIT(load_store) 124 CK_PR_FENCE_EMIT(store_load) 125 CK_PR_FENCE_NOOP(load) 126 CK_PR_FENCE_EMIT(store) 127 CK_PR_FENCE_EMIT(memory) 128 CK_PR_FENCE_EMIT(acquire) 129 CK_PR_FENCE_EMIT(release) 130 CK_PR_FENCE_EMIT(acqrel) 131 CK_PR_FENCE_EMIT(lock) 132 CK_PR_FENCE_EMIT(unlock) 133 #elif defined(CK_MD_TSO) 134 /* 135 * Only loads are re-ordered and only with respect to 136 * prior stores. Atomic operations are serializing. 137 */ 138 CK_PR_FENCE_NOOP(atomic) 139 CK_PR_FENCE_NOOP(atomic_load) 140 CK_PR_FENCE_NOOP(atomic_store) 141 CK_PR_FENCE_NOOP(store_atomic) 142 CK_PR_FENCE_NOOP(load_atomic) 143 CK_PR_FENCE_NOOP(load_store) 144 CK_PR_FENCE_EMIT(store_load) 145 CK_PR_FENCE_NOOP(load) 146 CK_PR_FENCE_NOOP(store) 147 CK_PR_FENCE_EMIT(memory) 148 CK_PR_FENCE_NOOP(acquire) 149 CK_PR_FENCE_NOOP(release) 150 CK_PR_FENCE_NOOP(acqrel) 151 CK_PR_FENCE_NOOP(lock) 152 CK_PR_FENCE_NOOP(unlock) 153 #else 154 #error "No memory model has been defined." 155 #endif /* CK_MD_TSO */ 156 157 #undef CK_PR_FENCE_EMIT 158 #undef CK_PR_FENCE_NOOP 159 160 #ifndef CK_F_PR_RFO 161 #define CK_F_PR_RFO 162 CK_CC_INLINE static void 163 ck_pr_rfo(const void *m) 164 { 165 166 (void)m; 167 return; 168 } 169 #endif /* CK_F_PR_RFO */ 170 171 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 172 ck_pr_md_store_##TYPE( \ 173 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 174 (VAL)) 175 176 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 177 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 178 #ifndef CK_PR_DISABLE_DOUBLE 179 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 180 #endif 181 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 182 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 183 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 184 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 185 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 186 187 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 188 189 #ifdef CK_F_PR_LOAD_64 190 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 191 #endif /* CK_F_PR_LOAD_64 */ 192 193 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 194 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 195 196 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 197 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 198 #ifndef CK_PR_DISABLE_DOUBLE 199 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 200 #endif 201 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 202 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 203 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 204 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 205 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 206 207 #ifdef CK_F_PR_LOAD_64 208 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 209 #endif /* CK_F_PR_LOAD_64 */ 210 211 #define CK_PR_BIN(K, S, M, T, P, C) \ 212 CK_CC_INLINE static void \ 213 ck_pr_##K##_##S(M *target, T value) \ 214 { \ 215 T previous; \ 216 C punt; \ 217 punt = ck_pr_md_load_##S(target); \ 218 previous = (T)punt; \ 219 while (ck_pr_cas_##S##_value(target, \ 220 (C)previous, \ 221 (C)(previous P value), \ 222 &previous) == false) \ 223 ck_pr_stall(); \ 224 \ 225 return; \ 226 } 227 228 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 229 230 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 231 232 #ifndef CK_F_PR_ADD_CHAR 233 #define CK_F_PR_ADD_CHAR 234 CK_PR_BIN_S(add, char, char, +) 235 #endif /* CK_F_PR_ADD_CHAR */ 236 237 #ifndef CK_F_PR_SUB_CHAR 238 #define CK_F_PR_SUB_CHAR 239 CK_PR_BIN_S(sub, char, char, -) 240 #endif /* CK_F_PR_SUB_CHAR */ 241 242 #ifndef CK_F_PR_AND_CHAR 243 #define CK_F_PR_AND_CHAR 244 CK_PR_BIN_S(and, char, char, &) 245 #endif /* CK_F_PR_AND_CHAR */ 246 247 #ifndef CK_F_PR_XOR_CHAR 248 #define CK_F_PR_XOR_CHAR 249 CK_PR_BIN_S(xor, char, char, ^) 250 #endif /* CK_F_PR_XOR_CHAR */ 251 252 #ifndef CK_F_PR_OR_CHAR 253 #define CK_F_PR_OR_CHAR 254 CK_PR_BIN_S(or, char, char, |) 255 #endif /* CK_F_PR_OR_CHAR */ 256 257 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 258 259 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 260 261 #ifndef CK_F_PR_ADD_INT 262 #define CK_F_PR_ADD_INT 263 CK_PR_BIN_S(add, int, int, +) 264 #endif /* CK_F_PR_ADD_INT */ 265 266 #ifndef CK_F_PR_SUB_INT 267 #define CK_F_PR_SUB_INT 268 CK_PR_BIN_S(sub, int, int, -) 269 #endif /* CK_F_PR_SUB_INT */ 270 271 #ifndef CK_F_PR_AND_INT 272 #define CK_F_PR_AND_INT 273 CK_PR_BIN_S(and, int, int, &) 274 #endif /* CK_F_PR_AND_INT */ 275 276 #ifndef CK_F_PR_XOR_INT 277 #define CK_F_PR_XOR_INT 278 CK_PR_BIN_S(xor, int, int, ^) 279 #endif /* CK_F_PR_XOR_INT */ 280 281 #ifndef CK_F_PR_OR_INT 282 #define CK_F_PR_OR_INT 283 CK_PR_BIN_S(or, int, int, |) 284 #endif /* CK_F_PR_OR_INT */ 285 286 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 287 288 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 289 !defined(CK_PR_DISABLE_DOUBLE) 290 291 #ifndef CK_F_PR_ADD_DOUBLE 292 #define CK_F_PR_ADD_DOUBLE 293 CK_PR_BIN_S(add, double, double, +) 294 #endif /* CK_F_PR_ADD_DOUBLE */ 295 296 #ifndef CK_F_PR_SUB_DOUBLE 297 #define CK_F_PR_SUB_DOUBLE 298 CK_PR_BIN_S(sub, double, double, -) 299 #endif /* CK_F_PR_SUB_DOUBLE */ 300 301 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 302 303 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 304 305 #ifndef CK_F_PR_ADD_UINT 306 #define CK_F_PR_ADD_UINT 307 CK_PR_BIN_S(add, uint, unsigned int, +) 308 #endif /* CK_F_PR_ADD_UINT */ 309 310 #ifndef CK_F_PR_SUB_UINT 311 #define CK_F_PR_SUB_UINT 312 CK_PR_BIN_S(sub, uint, unsigned int, -) 313 #endif /* CK_F_PR_SUB_UINT */ 314 315 #ifndef CK_F_PR_AND_UINT 316 #define CK_F_PR_AND_UINT 317 CK_PR_BIN_S(and, uint, unsigned int, &) 318 #endif /* CK_F_PR_AND_UINT */ 319 320 #ifndef CK_F_PR_XOR_UINT 321 #define CK_F_PR_XOR_UINT 322 CK_PR_BIN_S(xor, uint, unsigned int, ^) 323 #endif /* CK_F_PR_XOR_UINT */ 324 325 #ifndef CK_F_PR_OR_UINT 326 #define CK_F_PR_OR_UINT 327 CK_PR_BIN_S(or, uint, unsigned int, |) 328 #endif /* CK_F_PR_OR_UINT */ 329 330 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 331 332 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 333 334 #ifndef CK_F_PR_ADD_PTR 335 #define CK_F_PR_ADD_PTR 336 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 337 #endif /* CK_F_PR_ADD_PTR */ 338 339 #ifndef CK_F_PR_SUB_PTR 340 #define CK_F_PR_SUB_PTR 341 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 342 #endif /* CK_F_PR_SUB_PTR */ 343 344 #ifndef CK_F_PR_AND_PTR 345 #define CK_F_PR_AND_PTR 346 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 347 #endif /* CK_F_PR_AND_PTR */ 348 349 #ifndef CK_F_PR_XOR_PTR 350 #define CK_F_PR_XOR_PTR 351 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 352 #endif /* CK_F_PR_XOR_PTR */ 353 354 #ifndef CK_F_PR_OR_PTR 355 #define CK_F_PR_OR_PTR 356 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 357 #endif /* CK_F_PR_OR_PTR */ 358 359 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 360 361 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 362 363 #ifndef CK_F_PR_ADD_64 364 #define CK_F_PR_ADD_64 365 CK_PR_BIN_S(add, 64, uint64_t, +) 366 #endif /* CK_F_PR_ADD_64 */ 367 368 #ifndef CK_F_PR_SUB_64 369 #define CK_F_PR_SUB_64 370 CK_PR_BIN_S(sub, 64, uint64_t, -) 371 #endif /* CK_F_PR_SUB_64 */ 372 373 #ifndef CK_F_PR_AND_64 374 #define CK_F_PR_AND_64 375 CK_PR_BIN_S(and, 64, uint64_t, &) 376 #endif /* CK_F_PR_AND_64 */ 377 378 #ifndef CK_F_PR_XOR_64 379 #define CK_F_PR_XOR_64 380 CK_PR_BIN_S(xor, 64, uint64_t, ^) 381 #endif /* CK_F_PR_XOR_64 */ 382 383 #ifndef CK_F_PR_OR_64 384 #define CK_F_PR_OR_64 385 CK_PR_BIN_S(or, 64, uint64_t, |) 386 #endif /* CK_F_PR_OR_64 */ 387 388 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 389 390 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 391 392 #ifndef CK_F_PR_ADD_32 393 #define CK_F_PR_ADD_32 394 CK_PR_BIN_S(add, 32, uint32_t, +) 395 #endif /* CK_F_PR_ADD_32 */ 396 397 #ifndef CK_F_PR_SUB_32 398 #define CK_F_PR_SUB_32 399 CK_PR_BIN_S(sub, 32, uint32_t, -) 400 #endif /* CK_F_PR_SUB_32 */ 401 402 #ifndef CK_F_PR_AND_32 403 #define CK_F_PR_AND_32 404 CK_PR_BIN_S(and, 32, uint32_t, &) 405 #endif /* CK_F_PR_AND_32 */ 406 407 #ifndef CK_F_PR_XOR_32 408 #define CK_F_PR_XOR_32 409 CK_PR_BIN_S(xor, 32, uint32_t, ^) 410 #endif /* CK_F_PR_XOR_32 */ 411 412 #ifndef CK_F_PR_OR_32 413 #define CK_F_PR_OR_32 414 CK_PR_BIN_S(or, 32, uint32_t, |) 415 #endif /* CK_F_PR_OR_32 */ 416 417 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 418 419 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 420 421 #ifndef CK_F_PR_ADD_16 422 #define CK_F_PR_ADD_16 423 CK_PR_BIN_S(add, 16, uint16_t, +) 424 #endif /* CK_F_PR_ADD_16 */ 425 426 #ifndef CK_F_PR_SUB_16 427 #define CK_F_PR_SUB_16 428 CK_PR_BIN_S(sub, 16, uint16_t, -) 429 #endif /* CK_F_PR_SUB_16 */ 430 431 #ifndef CK_F_PR_AND_16 432 #define CK_F_PR_AND_16 433 CK_PR_BIN_S(and, 16, uint16_t, &) 434 #endif /* CK_F_PR_AND_16 */ 435 436 #ifndef CK_F_PR_XOR_16 437 #define CK_F_PR_XOR_16 438 CK_PR_BIN_S(xor, 16, uint16_t, ^) 439 #endif /* CK_F_PR_XOR_16 */ 440 441 #ifndef CK_F_PR_OR_16 442 #define CK_F_PR_OR_16 443 CK_PR_BIN_S(or, 16, uint16_t, |) 444 #endif /* CK_F_PR_OR_16 */ 445 446 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 447 448 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 449 450 #ifndef CK_F_PR_ADD_8 451 #define CK_F_PR_ADD_8 452 CK_PR_BIN_S(add, 8, uint8_t, +) 453 #endif /* CK_F_PR_ADD_8 */ 454 455 #ifndef CK_F_PR_SUB_8 456 #define CK_F_PR_SUB_8 457 CK_PR_BIN_S(sub, 8, uint8_t, -) 458 #endif /* CK_F_PR_SUB_8 */ 459 460 #ifndef CK_F_PR_AND_8 461 #define CK_F_PR_AND_8 462 CK_PR_BIN_S(and, 8, uint8_t, &) 463 #endif /* CK_F_PR_AND_8 */ 464 465 #ifndef CK_F_PR_XOR_8 466 #define CK_F_PR_XOR_8 467 CK_PR_BIN_S(xor, 8, uint8_t, ^) 468 #endif /* CK_F_PR_XOR_8 */ 469 470 #ifndef CK_F_PR_OR_8 471 #define CK_F_PR_OR_8 472 CK_PR_BIN_S(or, 8, uint8_t, |) 473 #endif /* CK_F_PR_OR_8 */ 474 475 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 476 477 #undef CK_PR_BIN_S 478 #undef CK_PR_BIN 479 480 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 481 CK_CC_INLINE static bool \ 482 ck_pr_##K##_##S(M *target, unsigned int offset) \ 483 { \ 484 T previous; \ 485 C punt; \ 486 punt = ck_pr_md_load_##S(target); \ 487 previous = (T)punt; \ 488 while (ck_pr_cas_##S##_value(target, (C)previous, \ 489 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 490 ck_pr_stall(); \ 491 return ((previous >> offset) & 1); \ 492 } 493 494 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 495 496 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 497 498 #ifndef CK_F_PR_BTC_INT 499 #define CK_F_PR_BTC_INT 500 CK_PR_BTX_S(btc, int, int, ^,) 501 #endif /* CK_F_PR_BTC_INT */ 502 503 #ifndef CK_F_PR_BTR_INT 504 #define CK_F_PR_BTR_INT 505 CK_PR_BTX_S(btr, int, int, &, ~) 506 #endif /* CK_F_PR_BTR_INT */ 507 508 #ifndef CK_F_PR_BTS_INT 509 #define CK_F_PR_BTS_INT 510 CK_PR_BTX_S(bts, int, int, |,) 511 #endif /* CK_F_PR_BTS_INT */ 512 513 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 514 515 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 516 517 #ifndef CK_F_PR_BTC_UINT 518 #define CK_F_PR_BTC_UINT 519 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 520 #endif /* CK_F_PR_BTC_UINT */ 521 522 #ifndef CK_F_PR_BTR_UINT 523 #define CK_F_PR_BTR_UINT 524 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 525 #endif /* CK_F_PR_BTR_UINT */ 526 527 #ifndef CK_F_PR_BTS_UINT 528 #define CK_F_PR_BTS_UINT 529 CK_PR_BTX_S(bts, uint, unsigned int, |,) 530 #endif /* CK_F_PR_BTS_UINT */ 531 532 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 533 534 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 535 536 #ifndef CK_F_PR_BTC_PTR 537 #define CK_F_PR_BTC_PTR 538 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 539 #endif /* CK_F_PR_BTC_PTR */ 540 541 #ifndef CK_F_PR_BTR_PTR 542 #define CK_F_PR_BTR_PTR 543 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 544 #endif /* CK_F_PR_BTR_PTR */ 545 546 #ifndef CK_F_PR_BTS_PTR 547 #define CK_F_PR_BTS_PTR 548 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 549 #endif /* CK_F_PR_BTS_PTR */ 550 551 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 552 553 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 554 555 #ifndef CK_F_PR_BTC_64 556 #define CK_F_PR_BTC_64 557 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 558 #endif /* CK_F_PR_BTC_64 */ 559 560 #ifndef CK_F_PR_BTR_64 561 #define CK_F_PR_BTR_64 562 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 563 #endif /* CK_F_PR_BTR_64 */ 564 565 #ifndef CK_F_PR_BTS_64 566 #define CK_F_PR_BTS_64 567 CK_PR_BTX_S(bts, 64, uint64_t, |,) 568 #endif /* CK_F_PR_BTS_64 */ 569 570 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 571 572 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 573 574 #ifndef CK_F_PR_BTC_32 575 #define CK_F_PR_BTC_32 576 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 577 #endif /* CK_F_PR_BTC_32 */ 578 579 #ifndef CK_F_PR_BTR_32 580 #define CK_F_PR_BTR_32 581 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 582 #endif /* CK_F_PR_BTR_32 */ 583 584 #ifndef CK_F_PR_BTS_32 585 #define CK_F_PR_BTS_32 586 CK_PR_BTX_S(bts, 32, uint32_t, |,) 587 #endif /* CK_F_PR_BTS_32 */ 588 589 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 590 591 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 592 593 #ifndef CK_F_PR_BTC_16 594 #define CK_F_PR_BTC_16 595 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 596 #endif /* CK_F_PR_BTC_16 */ 597 598 #ifndef CK_F_PR_BTR_16 599 #define CK_F_PR_BTR_16 600 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 601 #endif /* CK_F_PR_BTR_16 */ 602 603 #ifndef CK_F_PR_BTS_16 604 #define CK_F_PR_BTS_16 605 CK_PR_BTX_S(bts, 16, uint16_t, |,) 606 #endif /* CK_F_PR_BTS_16 */ 607 608 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 609 610 #undef CK_PR_BTX_S 611 #undef CK_PR_BTX 612 613 #define CK_PR_UNARY(K, X, S, M, T) \ 614 CK_CC_INLINE static void \ 615 ck_pr_##K##_##S(M *target) \ 616 { \ 617 ck_pr_##X##_##S(target, (T)1); \ 618 return; \ 619 } 620 621 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 622 CK_CC_INLINE static bool \ 623 ck_pr_##K##_##S##_is_zero(M *target) \ 624 { \ 625 T previous; \ 626 C punt; \ 627 punt = (C)ck_pr_md_load_##S(target); \ 628 previous = (T)punt; \ 629 while (ck_pr_cas_##S##_value(target, \ 630 (C)previous, \ 631 (C)(previous P 1), \ 632 &previous) == false) \ 633 ck_pr_stall(); \ 634 return previous == (T)Z; \ 635 } 636 637 #define CK_PR_UNARY_Z_STUB(K, S, M) \ 638 CK_CC_INLINE static void \ 639 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 640 { \ 641 *zero = ck_pr_##K##_##S##_is_zero(target); \ 642 return; \ 643 } 644 645 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 646 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 647 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 648 CK_PR_UNARY_Z_STUB(K, S, M) 649 650 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 651 652 #ifndef CK_F_PR_INC_CHAR 653 #define CK_F_PR_INC_CHAR 654 CK_PR_UNARY_S(inc, add, char, char) 655 #endif /* CK_F_PR_INC_CHAR */ 656 657 #ifndef CK_F_PR_INC_CHAR_ZERO 658 #define CK_F_PR_INC_CHAR_ZERO 659 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 660 #else 661 CK_PR_UNARY_Z_STUB(inc, char, char) 662 #endif /* CK_F_PR_INC_CHAR_ZERO */ 663 664 #ifndef CK_F_PR_DEC_CHAR 665 #define CK_F_PR_DEC_CHAR 666 CK_PR_UNARY_S(dec, sub, char, char) 667 #endif /* CK_F_PR_DEC_CHAR */ 668 669 #ifndef CK_F_PR_DEC_CHAR_ZERO 670 #define CK_F_PR_DEC_CHAR_ZERO 671 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 672 #else 673 CK_PR_UNARY_Z_STUB(dec, char, char) 674 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 675 676 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 677 678 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 679 680 #ifndef CK_F_PR_INC_INT 681 #define CK_F_PR_INC_INT 682 CK_PR_UNARY_S(inc, add, int, int) 683 #endif /* CK_F_PR_INC_INT */ 684 685 #ifndef CK_F_PR_INC_INT_ZERO 686 #define CK_F_PR_INC_INT_ZERO 687 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 688 #else 689 CK_PR_UNARY_Z_STUB(inc, int, int) 690 #endif /* CK_F_PR_INC_INT_ZERO */ 691 692 #ifndef CK_F_PR_DEC_INT 693 #define CK_F_PR_DEC_INT 694 CK_PR_UNARY_S(dec, sub, int, int) 695 #endif /* CK_F_PR_DEC_INT */ 696 697 #ifndef CK_F_PR_DEC_INT_ZERO 698 #define CK_F_PR_DEC_INT_ZERO 699 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 700 #else 701 CK_PR_UNARY_Z_STUB(dec, int, int) 702 #endif /* CK_F_PR_DEC_INT_ZERO */ 703 704 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 705 706 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 707 !defined(CK_PR_DISABLE_DOUBLE) 708 709 #ifndef CK_F_PR_INC_DOUBLE 710 #define CK_F_PR_INC_DOUBLE 711 CK_PR_UNARY_S(inc, add, double, double) 712 #endif /* CK_F_PR_INC_DOUBLE */ 713 714 #ifndef CK_F_PR_DEC_DOUBLE 715 #define CK_F_PR_DEC_DOUBLE 716 CK_PR_UNARY_S(dec, sub, double, double) 717 #endif /* CK_F_PR_DEC_DOUBLE */ 718 719 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 720 721 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 722 723 #ifndef CK_F_PR_INC_UINT 724 #define CK_F_PR_INC_UINT 725 CK_PR_UNARY_S(inc, add, uint, unsigned int) 726 #endif /* CK_F_PR_INC_UINT */ 727 728 #ifndef CK_F_PR_INC_UINT_ZERO 729 #define CK_F_PR_INC_UINT_ZERO 730 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 731 #else 732 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 733 #endif /* CK_F_PR_INC_UINT_ZERO */ 734 735 #ifndef CK_F_PR_DEC_UINT 736 #define CK_F_PR_DEC_UINT 737 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 738 #endif /* CK_F_PR_DEC_UINT */ 739 740 #ifndef CK_F_PR_DEC_UINT_ZERO 741 #define CK_F_PR_DEC_UINT_ZERO 742 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 743 #else 744 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 745 #endif /* CK_F_PR_DEC_UINT_ZERO */ 746 747 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 748 749 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 750 751 #ifndef CK_F_PR_INC_PTR 752 #define CK_F_PR_INC_PTR 753 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 754 #endif /* CK_F_PR_INC_PTR */ 755 756 #ifndef CK_F_PR_INC_PTR_ZERO 757 #define CK_F_PR_INC_PTR_ZERO 758 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 759 #else 760 CK_PR_UNARY_Z_STUB(inc, ptr, void) 761 #endif /* CK_F_PR_INC_PTR_ZERO */ 762 763 #ifndef CK_F_PR_DEC_PTR 764 #define CK_F_PR_DEC_PTR 765 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 766 #endif /* CK_F_PR_DEC_PTR */ 767 768 #ifndef CK_F_PR_DEC_PTR_ZERO 769 #define CK_F_PR_DEC_PTR_ZERO 770 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 771 #else 772 CK_PR_UNARY_Z_STUB(dec, ptr, void) 773 #endif /* CK_F_PR_DEC_PTR_ZERO */ 774 775 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 776 777 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 778 779 #ifndef CK_F_PR_INC_64 780 #define CK_F_PR_INC_64 781 CK_PR_UNARY_S(inc, add, 64, uint64_t) 782 #endif /* CK_F_PR_INC_64 */ 783 784 #ifndef CK_F_PR_INC_64_ZERO 785 #define CK_F_PR_INC_64_ZERO 786 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 787 #else 788 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 789 #endif /* CK_F_PR_INC_64_ZERO */ 790 791 #ifndef CK_F_PR_DEC_64 792 #define CK_F_PR_DEC_64 793 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 794 #endif /* CK_F_PR_DEC_64 */ 795 796 #ifndef CK_F_PR_DEC_64_ZERO 797 #define CK_F_PR_DEC_64_ZERO 798 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 799 #else 800 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 801 #endif /* CK_F_PR_DEC_64_ZERO */ 802 803 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 804 805 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 806 807 #ifndef CK_F_PR_INC_32 808 #define CK_F_PR_INC_32 809 CK_PR_UNARY_S(inc, add, 32, uint32_t) 810 #endif /* CK_F_PR_INC_32 */ 811 812 #ifndef CK_F_PR_INC_32_ZERO 813 #define CK_F_PR_INC_32_ZERO 814 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 815 #else 816 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 817 #endif /* CK_F_PR_INC_32_ZERO */ 818 819 #ifndef CK_F_PR_DEC_32 820 #define CK_F_PR_DEC_32 821 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 822 #endif /* CK_F_PR_DEC_32 */ 823 824 #ifndef CK_F_PR_DEC_32_ZERO 825 #define CK_F_PR_DEC_32_ZERO 826 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 827 #else 828 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 829 #endif /* CK_F_PR_DEC_32_ZERO */ 830 831 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 832 833 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 834 835 #ifndef CK_F_PR_INC_16 836 #define CK_F_PR_INC_16 837 CK_PR_UNARY_S(inc, add, 16, uint16_t) 838 #endif /* CK_F_PR_INC_16 */ 839 840 #ifndef CK_F_PR_INC_16_ZERO 841 #define CK_F_PR_INC_16_ZERO 842 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 843 #else 844 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 845 #endif /* CK_F_PR_INC_16_ZERO */ 846 847 #ifndef CK_F_PR_DEC_16 848 #define CK_F_PR_DEC_16 849 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 850 #endif /* CK_F_PR_DEC_16 */ 851 852 #ifndef CK_F_PR_DEC_16_ZERO 853 #define CK_F_PR_DEC_16_ZERO 854 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 855 #else 856 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 857 #endif /* CK_F_PR_DEC_16_ZERO */ 858 859 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 860 861 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 862 863 #ifndef CK_F_PR_INC_8 864 #define CK_F_PR_INC_8 865 CK_PR_UNARY_S(inc, add, 8, uint8_t) 866 #endif /* CK_F_PR_INC_8 */ 867 868 #ifndef CK_F_PR_INC_8_ZERO 869 #define CK_F_PR_INC_8_ZERO 870 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 871 #else 872 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 873 #endif /* CK_F_PR_INC_8_ZERO */ 874 875 #ifndef CK_F_PR_DEC_8 876 #define CK_F_PR_DEC_8 877 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 878 #endif /* CK_F_PR_DEC_8 */ 879 880 #ifndef CK_F_PR_DEC_8_ZERO 881 #define CK_F_PR_DEC_8_ZERO 882 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 883 #else 884 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 885 #endif /* CK_F_PR_DEC_8_ZERO */ 886 887 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 888 889 #undef CK_PR_UNARY_Z_S 890 #undef CK_PR_UNARY_S 891 #undef CK_PR_UNARY_Z 892 #undef CK_PR_UNARY 893 894 #define CK_PR_N(K, S, M, T, P, C) \ 895 CK_CC_INLINE static void \ 896 ck_pr_##K##_##S(M *target) \ 897 { \ 898 T previous; \ 899 C punt; \ 900 punt = (C)ck_pr_md_load_##S(target); \ 901 previous = (T)punt; \ 902 while (ck_pr_cas_##S##_value(target, \ 903 (C)previous, \ 904 (C)(P previous), \ 905 &previous) == false) \ 906 ck_pr_stall(); \ 907 \ 908 return; \ 909 } 910 911 #define CK_PR_N_Z(S, M, T, C) \ 912 CK_CC_INLINE static void \ 913 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 914 { \ 915 T previous; \ 916 C punt; \ 917 punt = (C)ck_pr_md_load_##S(target); \ 918 previous = (T)punt; \ 919 while (ck_pr_cas_##S##_value(target, \ 920 (C)previous, \ 921 (C)(-previous), \ 922 &previous) == false) \ 923 ck_pr_stall(); \ 924 \ 925 *zero = previous == 0; \ 926 return; \ 927 } 928 929 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 930 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 931 932 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 933 934 #ifndef CK_F_PR_NOT_CHAR 935 #define CK_F_PR_NOT_CHAR 936 CK_PR_N_S(not, char, char, ~) 937 #endif /* CK_F_PR_NOT_CHAR */ 938 939 #ifndef CK_F_PR_NEG_CHAR 940 #define CK_F_PR_NEG_CHAR 941 CK_PR_N_S(neg, char, char, -) 942 #endif /* CK_F_PR_NEG_CHAR */ 943 944 #ifndef CK_F_PR_NEG_CHAR_ZERO 945 #define CK_F_PR_NEG_CHAR_ZERO 946 CK_PR_N_Z_S(char, char) 947 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 948 949 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 950 951 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 952 953 #ifndef CK_F_PR_NOT_INT 954 #define CK_F_PR_NOT_INT 955 CK_PR_N_S(not, int, int, ~) 956 #endif /* CK_F_PR_NOT_INT */ 957 958 #ifndef CK_F_PR_NEG_INT 959 #define CK_F_PR_NEG_INT 960 CK_PR_N_S(neg, int, int, -) 961 #endif /* CK_F_PR_NEG_INT */ 962 963 #ifndef CK_F_PR_NEG_INT_ZERO 964 #define CK_F_PR_NEG_INT_ZERO 965 CK_PR_N_Z_S(int, int) 966 #endif /* CK_F_PR_NEG_INT_ZERO */ 967 968 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 969 970 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 971 !defined(CK_PR_DISABLE_DOUBLE) 972 973 #ifndef CK_F_PR_NEG_DOUBLE 974 #define CK_F_PR_NEG_DOUBLE 975 CK_PR_N_S(neg, double, double, -) 976 #endif /* CK_F_PR_NEG_DOUBLE */ 977 978 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 979 980 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 981 982 #ifndef CK_F_PR_NOT_UINT 983 #define CK_F_PR_NOT_UINT 984 CK_PR_N_S(not, uint, unsigned int, ~) 985 #endif /* CK_F_PR_NOT_UINT */ 986 987 #ifndef CK_F_PR_NEG_UINT 988 #define CK_F_PR_NEG_UINT 989 CK_PR_N_S(neg, uint, unsigned int, -) 990 #endif /* CK_F_PR_NEG_UINT */ 991 992 #ifndef CK_F_PR_NEG_UINT_ZERO 993 #define CK_F_PR_NEG_UINT_ZERO 994 CK_PR_N_Z_S(uint, unsigned int) 995 #endif /* CK_F_PR_NEG_UINT_ZERO */ 996 997 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 998 999 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1000 1001 #ifndef CK_F_PR_NOT_PTR 1002 #define CK_F_PR_NOT_PTR 1003 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1004 #endif /* CK_F_PR_NOT_PTR */ 1005 1006 #ifndef CK_F_PR_NEG_PTR 1007 #define CK_F_PR_NEG_PTR 1008 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1009 #endif /* CK_F_PR_NEG_PTR */ 1010 1011 #ifndef CK_F_PR_NEG_PTR_ZERO 1012 #define CK_F_PR_NEG_PTR_ZERO 1013 CK_PR_N_Z(ptr, void, uintptr_t, void *) 1014 #endif /* CK_F_PR_NEG_PTR_ZERO */ 1015 1016 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1017 1018 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1019 1020 #ifndef CK_F_PR_NOT_64 1021 #define CK_F_PR_NOT_64 1022 CK_PR_N_S(not, 64, uint64_t, ~) 1023 #endif /* CK_F_PR_NOT_64 */ 1024 1025 #ifndef CK_F_PR_NEG_64 1026 #define CK_F_PR_NEG_64 1027 CK_PR_N_S(neg, 64, uint64_t, -) 1028 #endif /* CK_F_PR_NEG_64 */ 1029 1030 #ifndef CK_F_PR_NEG_64_ZERO 1031 #define CK_F_PR_NEG_64_ZERO 1032 CK_PR_N_Z_S(64, uint64_t) 1033 #endif /* CK_F_PR_NEG_64_ZERO */ 1034 1035 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1036 1037 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1038 1039 #ifndef CK_F_PR_NOT_32 1040 #define CK_F_PR_NOT_32 1041 CK_PR_N_S(not, 32, uint32_t, ~) 1042 #endif /* CK_F_PR_NOT_32 */ 1043 1044 #ifndef CK_F_PR_NEG_32 1045 #define CK_F_PR_NEG_32 1046 CK_PR_N_S(neg, 32, uint32_t, -) 1047 #endif /* CK_F_PR_NEG_32 */ 1048 1049 #ifndef CK_F_PR_NEG_32_ZERO 1050 #define CK_F_PR_NEG_32_ZERO 1051 CK_PR_N_Z_S(32, uint32_t) 1052 #endif /* CK_F_PR_NEG_32_ZERO */ 1053 1054 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1055 1056 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1057 1058 #ifndef CK_F_PR_NOT_16 1059 #define CK_F_PR_NOT_16 1060 CK_PR_N_S(not, 16, uint16_t, ~) 1061 #endif /* CK_F_PR_NOT_16 */ 1062 1063 #ifndef CK_F_PR_NEG_16 1064 #define CK_F_PR_NEG_16 1065 CK_PR_N_S(neg, 16, uint16_t, -) 1066 #endif /* CK_F_PR_NEG_16 */ 1067 1068 #ifndef CK_F_PR_NEG_16_ZERO 1069 #define CK_F_PR_NEG_16_ZERO 1070 CK_PR_N_Z_S(16, uint16_t) 1071 #endif /* CK_F_PR_NEG_16_ZERO */ 1072 1073 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1074 1075 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1076 1077 #ifndef CK_F_PR_NOT_8 1078 #define CK_F_PR_NOT_8 1079 CK_PR_N_S(not, 8, uint8_t, ~) 1080 #endif /* CK_F_PR_NOT_8 */ 1081 1082 #ifndef CK_F_PR_NEG_8 1083 #define CK_F_PR_NEG_8 1084 CK_PR_N_S(neg, 8, uint8_t, -) 1085 #endif /* CK_F_PR_NEG_8 */ 1086 1087 #ifndef CK_F_PR_NEG_8_ZERO 1088 #define CK_F_PR_NEG_8_ZERO 1089 CK_PR_N_Z_S(8, uint8_t) 1090 #endif /* CK_F_PR_NEG_8_ZERO */ 1091 1092 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1093 1094 #undef CK_PR_N_Z_S 1095 #undef CK_PR_N_S 1096 #undef CK_PR_N_Z 1097 #undef CK_PR_N 1098 1099 #define CK_PR_FAA(S, M, T, C) \ 1100 CK_CC_INLINE static C \ 1101 ck_pr_faa_##S(M *target, T delta) \ 1102 { \ 1103 T previous; \ 1104 C punt; \ 1105 punt = (C)ck_pr_md_load_##S(target); \ 1106 previous = (T)punt; \ 1107 while (ck_pr_cas_##S##_value(target, \ 1108 (C)previous, \ 1109 (C)(previous + delta), \ 1110 &previous) == false) \ 1111 ck_pr_stall(); \ 1112 \ 1113 return ((C)previous); \ 1114 } 1115 1116 #define CK_PR_FAS(S, M, C) \ 1117 CK_CC_INLINE static C \ 1118 ck_pr_fas_##S(M *target, C update) \ 1119 { \ 1120 C previous; \ 1121 previous = ck_pr_md_load_##S(target); \ 1122 while (ck_pr_cas_##S##_value(target, \ 1123 previous, \ 1124 update, \ 1125 &previous) == false) \ 1126 ck_pr_stall(); \ 1127 \ 1128 return (previous); \ 1129 } 1130 1131 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1132 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1133 1134 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1135 1136 #ifndef CK_F_PR_FAA_CHAR 1137 #define CK_F_PR_FAA_CHAR 1138 CK_PR_FAA_S(char, char) 1139 #endif /* CK_F_PR_FAA_CHAR */ 1140 1141 #ifndef CK_F_PR_FAS_CHAR 1142 #define CK_F_PR_FAS_CHAR 1143 CK_PR_FAS_S(char, char) 1144 #endif /* CK_F_PR_FAS_CHAR */ 1145 1146 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1147 1148 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1149 1150 #ifndef CK_F_PR_FAA_INT 1151 #define CK_F_PR_FAA_INT 1152 CK_PR_FAA_S(int, int) 1153 #endif /* CK_F_PR_FAA_INT */ 1154 1155 #ifndef CK_F_PR_FAS_INT 1156 #define CK_F_PR_FAS_INT 1157 CK_PR_FAS_S(int, int) 1158 #endif /* CK_F_PR_FAS_INT */ 1159 1160 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1161 1162 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1163 !defined(CK_PR_DISABLE_DOUBLE) 1164 1165 #ifndef CK_F_PR_FAA_DOUBLE 1166 #define CK_F_PR_FAA_DOUBLE 1167 CK_PR_FAA_S(double, double) 1168 #endif /* CK_F_PR_FAA_DOUBLE */ 1169 1170 #ifndef CK_F_PR_FAS_DOUBLE 1171 #define CK_F_PR_FAS_DOUBLE 1172 CK_PR_FAS_S(double, double) 1173 #endif /* CK_F_PR_FAS_DOUBLE */ 1174 1175 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1176 1177 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1178 1179 #ifndef CK_F_PR_FAA_UINT 1180 #define CK_F_PR_FAA_UINT 1181 CK_PR_FAA_S(uint, unsigned int) 1182 #endif /* CK_F_PR_FAA_UINT */ 1183 1184 #ifndef CK_F_PR_FAS_UINT 1185 #define CK_F_PR_FAS_UINT 1186 CK_PR_FAS_S(uint, unsigned int) 1187 #endif /* CK_F_PR_FAS_UINT */ 1188 1189 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1190 1191 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1192 1193 #ifndef CK_F_PR_FAA_PTR 1194 #define CK_F_PR_FAA_PTR 1195 CK_PR_FAA(ptr, void, uintptr_t, void *) 1196 #endif /* CK_F_PR_FAA_PTR */ 1197 1198 #ifndef CK_F_PR_FAS_PTR 1199 #define CK_F_PR_FAS_PTR 1200 CK_PR_FAS(ptr, void, void *) 1201 #endif /* CK_F_PR_FAS_PTR */ 1202 1203 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1204 1205 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1206 1207 #ifndef CK_F_PR_FAA_64 1208 #define CK_F_PR_FAA_64 1209 CK_PR_FAA_S(64, uint64_t) 1210 #endif /* CK_F_PR_FAA_64 */ 1211 1212 #ifndef CK_F_PR_FAS_64 1213 #define CK_F_PR_FAS_64 1214 CK_PR_FAS_S(64, uint64_t) 1215 #endif /* CK_F_PR_FAS_64 */ 1216 1217 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1218 1219 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1220 1221 #ifndef CK_F_PR_FAA_32 1222 #define CK_F_PR_FAA_32 1223 CK_PR_FAA_S(32, uint32_t) 1224 #endif /* CK_F_PR_FAA_32 */ 1225 1226 #ifndef CK_F_PR_FAS_32 1227 #define CK_F_PR_FAS_32 1228 CK_PR_FAS_S(32, uint32_t) 1229 #endif /* CK_F_PR_FAS_32 */ 1230 1231 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1232 1233 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1234 1235 #ifndef CK_F_PR_FAA_16 1236 #define CK_F_PR_FAA_16 1237 CK_PR_FAA_S(16, uint16_t) 1238 #endif /* CK_F_PR_FAA_16 */ 1239 1240 #ifndef CK_F_PR_FAS_16 1241 #define CK_F_PR_FAS_16 1242 CK_PR_FAS_S(16, uint16_t) 1243 #endif /* CK_F_PR_FAS_16 */ 1244 1245 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1246 1247 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1248 1249 #ifndef CK_F_PR_FAA_8 1250 #define CK_F_PR_FAA_8 1251 CK_PR_FAA_S(8, uint8_t) 1252 #endif /* CK_F_PR_FAA_8 */ 1253 1254 #ifndef CK_F_PR_FAS_8 1255 #define CK_F_PR_FAS_8 1256 CK_PR_FAS_S(8, uint8_t) 1257 #endif /* CK_F_PR_FAS_8 */ 1258 1259 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1260 1261 #undef CK_PR_FAA_S 1262 #undef CK_PR_FAS_S 1263 #undef CK_PR_FAA 1264 #undef CK_PR_FAS 1265 1266 #endif /* CK_PR_H */ 1267