1 /* 2 * Copyright 2009-2015 Samy Al Bahra. 3 * Copyright 2011 David Joseph. 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef CK_PR_H 29 #define CK_PR_H 30 31 #include <ck_cc.h> 32 #include <ck_limits.h> 33 #include <ck_md.h> 34 #include <ck_stdint.h> 35 #include <ck_stdbool.h> 36 37 /* 38 * Default to using builtins for clang analyzer, coverity, and sparse: 39 * inline assembly is often too opaque for useful analysis. Override 40 * the defaults by defining CK_USE_CC_BUILTINS=0 or 1. 41 */ 42 #if !defined(CK_USE_CC_BUILTINS) 43 #if defined(__clang_analyzer__) || defined(__COVERITY__) || defined(__CHECKER__) 44 #define CK_USE_CC_BUILTINS 1 45 #else 46 #define CK_USE_CC_BUILTINS 0 47 #endif 48 #endif 49 50 #if !CK_USE_CC_BUILTINS 51 #if defined(__x86_64__) 52 #include "gcc/x86_64/ck_pr.h" 53 #elif defined(__x86__) 54 #include "gcc/x86/ck_pr.h" 55 #elif defined(__sparcv9__) 56 #include "gcc/sparcv9/ck_pr.h" 57 #elif defined(__ppc64__) 58 #include "gcc/ppc64/ck_pr.h" 59 #elif defined(__s390x__) 60 #include "gcc/s390x/ck_pr.h" 61 #elif defined(__ppc__) 62 #include "gcc/ppc/ck_pr.h" 63 #elif defined(__arm__) 64 #include "gcc/arm/ck_pr.h" 65 #elif defined(__aarch64__) 66 #include "gcc/aarch64/ck_pr.h" 67 #elif !defined(__GNUC__) 68 #error Your platform is unsupported 69 #endif 70 #endif /* !CK_USE_CC_BUILTINS */ 71 72 #if defined(__GNUC__) 73 #include "gcc/ck_pr.h" 74 #endif 75 76 #define CK_PR_FENCE_EMIT(T) \ 77 CK_CC_INLINE static void \ 78 ck_pr_fence_##T(void) \ 79 { \ 80 ck_pr_fence_strict_##T(); \ 81 return; \ 82 } 83 #define CK_PR_FENCE_NOOP(T) \ 84 CK_CC_INLINE static void \ 85 ck_pr_fence_##T(void) \ 86 { \ 87 ck_pr_barrier(); \ 88 return; \ 89 } 90 91 /* 92 * None of the currently supported platforms allow for data-dependent 93 * load ordering. 94 */ 95 CK_PR_FENCE_NOOP(load_depends) 96 #define ck_pr_fence_strict_load_depends ck_pr_fence_load_depends 97 98 /* 99 * In memory models where atomic operations do not have serializing 100 * effects, atomic read-modify-write operations are modeled as stores. 101 */ 102 #if defined(CK_MD_RMO) 103 /* 104 * Only stores to the same location have a global 105 * ordering. 106 */ 107 CK_PR_FENCE_EMIT(atomic) 108 CK_PR_FENCE_EMIT(atomic_load) 109 CK_PR_FENCE_EMIT(atomic_store) 110 CK_PR_FENCE_EMIT(store_atomic) 111 CK_PR_FENCE_EMIT(load_atomic) 112 CK_PR_FENCE_EMIT(load_store) 113 CK_PR_FENCE_EMIT(store_load) 114 CK_PR_FENCE_EMIT(load) 115 CK_PR_FENCE_EMIT(store) 116 CK_PR_FENCE_EMIT(memory) 117 CK_PR_FENCE_EMIT(acquire) 118 CK_PR_FENCE_EMIT(release) 119 CK_PR_FENCE_EMIT(acqrel) 120 CK_PR_FENCE_EMIT(lock) 121 CK_PR_FENCE_EMIT(unlock) 122 #elif defined(CK_MD_PSO) 123 /* 124 * Anything can be re-ordered with respect to stores. 125 * Otherwise, loads are executed in-order. 126 */ 127 CK_PR_FENCE_EMIT(atomic) 128 CK_PR_FENCE_NOOP(atomic_load) 129 CK_PR_FENCE_EMIT(atomic_store) 130 CK_PR_FENCE_EMIT(store_atomic) 131 CK_PR_FENCE_NOOP(load_atomic) 132 CK_PR_FENCE_EMIT(load_store) 133 CK_PR_FENCE_EMIT(store_load) 134 CK_PR_FENCE_NOOP(load) 135 CK_PR_FENCE_EMIT(store) 136 CK_PR_FENCE_EMIT(memory) 137 CK_PR_FENCE_EMIT(acquire) 138 CK_PR_FENCE_EMIT(release) 139 CK_PR_FENCE_EMIT(acqrel) 140 CK_PR_FENCE_EMIT(lock) 141 CK_PR_FENCE_EMIT(unlock) 142 #elif defined(CK_MD_TSO) 143 /* 144 * Only loads are re-ordered and only with respect to 145 * prior stores. Atomic operations are serializing. 146 */ 147 CK_PR_FENCE_NOOP(atomic) 148 CK_PR_FENCE_NOOP(atomic_load) 149 CK_PR_FENCE_NOOP(atomic_store) 150 CK_PR_FENCE_NOOP(store_atomic) 151 CK_PR_FENCE_NOOP(load_atomic) 152 CK_PR_FENCE_NOOP(load_store) 153 CK_PR_FENCE_EMIT(store_load) 154 CK_PR_FENCE_NOOP(load) 155 CK_PR_FENCE_NOOP(store) 156 CK_PR_FENCE_EMIT(memory) 157 CK_PR_FENCE_NOOP(acquire) 158 CK_PR_FENCE_NOOP(release) 159 CK_PR_FENCE_NOOP(acqrel) 160 CK_PR_FENCE_NOOP(lock) 161 CK_PR_FENCE_NOOP(unlock) 162 #else 163 #error "No memory model has been defined." 164 #endif /* CK_MD_TSO */ 165 166 #undef CK_PR_FENCE_EMIT 167 #undef CK_PR_FENCE_NOOP 168 169 #ifndef CK_F_PR_RFO 170 #define CK_F_PR_RFO 171 CK_CC_INLINE static void 172 ck_pr_rfo(const void *m) 173 { 174 175 (void)m; 176 return; 177 } 178 #endif /* CK_F_PR_RFO */ 179 180 #define CK_PR_STORE_SAFE(DST, VAL, TYPE) \ 181 ck_pr_md_store_##TYPE( \ 182 ((void)sizeof(*(DST) = (VAL)), (DST)), \ 183 (VAL)) 184 185 #define ck_pr_store_ptr(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), ptr) 186 #define ck_pr_store_char(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), char) 187 #ifndef CK_PR_DISABLE_DOUBLE 188 #define ck_pr_store_double(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), double) 189 #endif 190 #define ck_pr_store_uint(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), uint) 191 #define ck_pr_store_int(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), int) 192 #define ck_pr_store_32(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 32) 193 #define ck_pr_store_16(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 16) 194 #define ck_pr_store_8(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 8) 195 196 #define ck_pr_store_ptr_unsafe(DST, VAL) ck_pr_md_store_ptr((DST), (VAL)) 197 198 #ifdef CK_F_PR_LOAD_64 199 #define ck_pr_store_64(DST, VAL) CK_PR_STORE_SAFE((DST), (VAL), 64) 200 #endif /* CK_F_PR_LOAD_64 */ 201 202 #define CK_PR_LOAD_PTR_SAFE(SRC) (CK_CC_TYPEOF(*(SRC), (void *)))ck_pr_md_load_ptr((SRC)) 203 #define ck_pr_load_ptr(SRC) CK_PR_LOAD_PTR_SAFE((SRC)) 204 205 #define CK_PR_LOAD_SAFE(SRC, TYPE) ck_pr_md_load_##TYPE((SRC)) 206 #define ck_pr_load_char(SRC) CK_PR_LOAD_SAFE((SRC), char) 207 #ifndef CK_PR_DISABLE_DOUBLE 208 #define ck_pr_load_double(SRC) CK_PR_LOAD_SAFE((SRC), double) 209 #endif 210 #define ck_pr_load_uint(SRC) CK_PR_LOAD_SAFE((SRC), uint) 211 #define ck_pr_load_int(SRC) CK_PR_LOAD_SAFE((SRC), int) 212 #define ck_pr_load_32(SRC) CK_PR_LOAD_SAFE((SRC), 32) 213 #define ck_pr_load_16(SRC) CK_PR_LOAD_SAFE((SRC), 16) 214 #define ck_pr_load_8(SRC) CK_PR_LOAD_SAFE((SRC), 8) 215 216 #ifdef CK_F_PR_LOAD_64 217 #define ck_pr_load_64(SRC) CK_PR_LOAD_SAFE((SRC), 64) 218 #endif /* CK_F_PR_LOAD_64 */ 219 220 #define CK_PR_BIN(K, S, M, T, P, C) \ 221 CK_CC_INLINE static void \ 222 ck_pr_##K##_##S(M *target, T value) \ 223 { \ 224 T previous; \ 225 C punt; \ 226 punt = ck_pr_md_load_##S(target); \ 227 previous = (T)punt; \ 228 while (ck_pr_cas_##S##_value(target, \ 229 (C)previous, \ 230 (C)(previous P value), \ 231 &previous) == false) \ 232 ck_pr_stall(); \ 233 \ 234 return; \ 235 } 236 237 #define CK_PR_BIN_S(K, S, T, P) CK_PR_BIN(K, S, T, T, P, T) 238 239 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 240 241 #ifndef CK_F_PR_ADD_CHAR 242 #define CK_F_PR_ADD_CHAR 243 CK_PR_BIN_S(add, char, char, +) 244 #endif /* CK_F_PR_ADD_CHAR */ 245 246 #ifndef CK_F_PR_SUB_CHAR 247 #define CK_F_PR_SUB_CHAR 248 CK_PR_BIN_S(sub, char, char, -) 249 #endif /* CK_F_PR_SUB_CHAR */ 250 251 #ifndef CK_F_PR_AND_CHAR 252 #define CK_F_PR_AND_CHAR 253 CK_PR_BIN_S(and, char, char, &) 254 #endif /* CK_F_PR_AND_CHAR */ 255 256 #ifndef CK_F_PR_XOR_CHAR 257 #define CK_F_PR_XOR_CHAR 258 CK_PR_BIN_S(xor, char, char, ^) 259 #endif /* CK_F_PR_XOR_CHAR */ 260 261 #ifndef CK_F_PR_OR_CHAR 262 #define CK_F_PR_OR_CHAR 263 CK_PR_BIN_S(or, char, char, |) 264 #endif /* CK_F_PR_OR_CHAR */ 265 266 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 267 268 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 269 270 #ifndef CK_F_PR_ADD_INT 271 #define CK_F_PR_ADD_INT 272 CK_PR_BIN_S(add, int, int, +) 273 #endif /* CK_F_PR_ADD_INT */ 274 275 #ifndef CK_F_PR_SUB_INT 276 #define CK_F_PR_SUB_INT 277 CK_PR_BIN_S(sub, int, int, -) 278 #endif /* CK_F_PR_SUB_INT */ 279 280 #ifndef CK_F_PR_AND_INT 281 #define CK_F_PR_AND_INT 282 CK_PR_BIN_S(and, int, int, &) 283 #endif /* CK_F_PR_AND_INT */ 284 285 #ifndef CK_F_PR_XOR_INT 286 #define CK_F_PR_XOR_INT 287 CK_PR_BIN_S(xor, int, int, ^) 288 #endif /* CK_F_PR_XOR_INT */ 289 290 #ifndef CK_F_PR_OR_INT 291 #define CK_F_PR_OR_INT 292 CK_PR_BIN_S(or, int, int, |) 293 #endif /* CK_F_PR_OR_INT */ 294 295 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 296 297 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 298 !defined(CK_PR_DISABLE_DOUBLE) 299 300 #ifndef CK_F_PR_ADD_DOUBLE 301 #define CK_F_PR_ADD_DOUBLE 302 CK_PR_BIN_S(add, double, double, +) 303 #endif /* CK_F_PR_ADD_DOUBLE */ 304 305 #ifndef CK_F_PR_SUB_DOUBLE 306 #define CK_F_PR_SUB_DOUBLE 307 CK_PR_BIN_S(sub, double, double, -) 308 #endif /* CK_F_PR_SUB_DOUBLE */ 309 310 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 311 312 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 313 314 #ifndef CK_F_PR_ADD_UINT 315 #define CK_F_PR_ADD_UINT 316 CK_PR_BIN_S(add, uint, unsigned int, +) 317 #endif /* CK_F_PR_ADD_UINT */ 318 319 #ifndef CK_F_PR_SUB_UINT 320 #define CK_F_PR_SUB_UINT 321 CK_PR_BIN_S(sub, uint, unsigned int, -) 322 #endif /* CK_F_PR_SUB_UINT */ 323 324 #ifndef CK_F_PR_AND_UINT 325 #define CK_F_PR_AND_UINT 326 CK_PR_BIN_S(and, uint, unsigned int, &) 327 #endif /* CK_F_PR_AND_UINT */ 328 329 #ifndef CK_F_PR_XOR_UINT 330 #define CK_F_PR_XOR_UINT 331 CK_PR_BIN_S(xor, uint, unsigned int, ^) 332 #endif /* CK_F_PR_XOR_UINT */ 333 334 #ifndef CK_F_PR_OR_UINT 335 #define CK_F_PR_OR_UINT 336 CK_PR_BIN_S(or, uint, unsigned int, |) 337 #endif /* CK_F_PR_OR_UINT */ 338 339 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 340 341 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 342 343 #ifndef CK_F_PR_ADD_PTR 344 #define CK_F_PR_ADD_PTR 345 CK_PR_BIN(add, ptr, void, uintptr_t, +, void *) 346 #endif /* CK_F_PR_ADD_PTR */ 347 348 #ifndef CK_F_PR_SUB_PTR 349 #define CK_F_PR_SUB_PTR 350 CK_PR_BIN(sub, ptr, void, uintptr_t, -, void *) 351 #endif /* CK_F_PR_SUB_PTR */ 352 353 #ifndef CK_F_PR_AND_PTR 354 #define CK_F_PR_AND_PTR 355 CK_PR_BIN(and, ptr, void, uintptr_t, &, void *) 356 #endif /* CK_F_PR_AND_PTR */ 357 358 #ifndef CK_F_PR_XOR_PTR 359 #define CK_F_PR_XOR_PTR 360 CK_PR_BIN(xor, ptr, void, uintptr_t, ^, void *) 361 #endif /* CK_F_PR_XOR_PTR */ 362 363 #ifndef CK_F_PR_OR_PTR 364 #define CK_F_PR_OR_PTR 365 CK_PR_BIN(or, ptr, void, uintptr_t, |, void *) 366 #endif /* CK_F_PR_OR_PTR */ 367 368 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 369 370 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 371 372 #ifndef CK_F_PR_ADD_64 373 #define CK_F_PR_ADD_64 374 CK_PR_BIN_S(add, 64, uint64_t, +) 375 #endif /* CK_F_PR_ADD_64 */ 376 377 #ifndef CK_F_PR_SUB_64 378 #define CK_F_PR_SUB_64 379 CK_PR_BIN_S(sub, 64, uint64_t, -) 380 #endif /* CK_F_PR_SUB_64 */ 381 382 #ifndef CK_F_PR_AND_64 383 #define CK_F_PR_AND_64 384 CK_PR_BIN_S(and, 64, uint64_t, &) 385 #endif /* CK_F_PR_AND_64 */ 386 387 #ifndef CK_F_PR_XOR_64 388 #define CK_F_PR_XOR_64 389 CK_PR_BIN_S(xor, 64, uint64_t, ^) 390 #endif /* CK_F_PR_XOR_64 */ 391 392 #ifndef CK_F_PR_OR_64 393 #define CK_F_PR_OR_64 394 CK_PR_BIN_S(or, 64, uint64_t, |) 395 #endif /* CK_F_PR_OR_64 */ 396 397 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 398 399 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 400 401 #ifndef CK_F_PR_ADD_32 402 #define CK_F_PR_ADD_32 403 CK_PR_BIN_S(add, 32, uint32_t, +) 404 #endif /* CK_F_PR_ADD_32 */ 405 406 #ifndef CK_F_PR_SUB_32 407 #define CK_F_PR_SUB_32 408 CK_PR_BIN_S(sub, 32, uint32_t, -) 409 #endif /* CK_F_PR_SUB_32 */ 410 411 #ifndef CK_F_PR_AND_32 412 #define CK_F_PR_AND_32 413 CK_PR_BIN_S(and, 32, uint32_t, &) 414 #endif /* CK_F_PR_AND_32 */ 415 416 #ifndef CK_F_PR_XOR_32 417 #define CK_F_PR_XOR_32 418 CK_PR_BIN_S(xor, 32, uint32_t, ^) 419 #endif /* CK_F_PR_XOR_32 */ 420 421 #ifndef CK_F_PR_OR_32 422 #define CK_F_PR_OR_32 423 CK_PR_BIN_S(or, 32, uint32_t, |) 424 #endif /* CK_F_PR_OR_32 */ 425 426 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 427 428 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 429 430 #ifndef CK_F_PR_ADD_16 431 #define CK_F_PR_ADD_16 432 CK_PR_BIN_S(add, 16, uint16_t, +) 433 #endif /* CK_F_PR_ADD_16 */ 434 435 #ifndef CK_F_PR_SUB_16 436 #define CK_F_PR_SUB_16 437 CK_PR_BIN_S(sub, 16, uint16_t, -) 438 #endif /* CK_F_PR_SUB_16 */ 439 440 #ifndef CK_F_PR_AND_16 441 #define CK_F_PR_AND_16 442 CK_PR_BIN_S(and, 16, uint16_t, &) 443 #endif /* CK_F_PR_AND_16 */ 444 445 #ifndef CK_F_PR_XOR_16 446 #define CK_F_PR_XOR_16 447 CK_PR_BIN_S(xor, 16, uint16_t, ^) 448 #endif /* CK_F_PR_XOR_16 */ 449 450 #ifndef CK_F_PR_OR_16 451 #define CK_F_PR_OR_16 452 CK_PR_BIN_S(or, 16, uint16_t, |) 453 #endif /* CK_F_PR_OR_16 */ 454 455 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 456 457 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 458 459 #ifndef CK_F_PR_ADD_8 460 #define CK_F_PR_ADD_8 461 CK_PR_BIN_S(add, 8, uint8_t, +) 462 #endif /* CK_F_PR_ADD_8 */ 463 464 #ifndef CK_F_PR_SUB_8 465 #define CK_F_PR_SUB_8 466 CK_PR_BIN_S(sub, 8, uint8_t, -) 467 #endif /* CK_F_PR_SUB_8 */ 468 469 #ifndef CK_F_PR_AND_8 470 #define CK_F_PR_AND_8 471 CK_PR_BIN_S(and, 8, uint8_t, &) 472 #endif /* CK_F_PR_AND_8 */ 473 474 #ifndef CK_F_PR_XOR_8 475 #define CK_F_PR_XOR_8 476 CK_PR_BIN_S(xor, 8, uint8_t, ^) 477 #endif /* CK_F_PR_XOR_8 */ 478 479 #ifndef CK_F_PR_OR_8 480 #define CK_F_PR_OR_8 481 CK_PR_BIN_S(or, 8, uint8_t, |) 482 #endif /* CK_F_PR_OR_8 */ 483 484 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 485 486 #undef CK_PR_BIN_S 487 #undef CK_PR_BIN 488 489 #define CK_PR_BTX(K, S, M, T, P, C, R) \ 490 CK_CC_INLINE static bool \ 491 ck_pr_##K##_##S(M *target, unsigned int offset) \ 492 { \ 493 T previous; \ 494 C punt; \ 495 punt = ck_pr_md_load_##S(target); \ 496 previous = (T)punt; \ 497 while (ck_pr_cas_##S##_value(target, (C)previous, \ 498 (C)(previous P (R ((T)1 << offset))), &previous) == false) \ 499 ck_pr_stall(); \ 500 return ((previous >> offset) & 1); \ 501 } 502 503 #define CK_PR_BTX_S(K, S, T, P, R) CK_PR_BTX(K, S, T, T, P, T, R) 504 505 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 506 507 #ifndef CK_F_PR_BTC_INT 508 #define CK_F_PR_BTC_INT 509 CK_PR_BTX_S(btc, int, int, ^,) 510 #endif /* CK_F_PR_BTC_INT */ 511 512 #ifndef CK_F_PR_BTR_INT 513 #define CK_F_PR_BTR_INT 514 CK_PR_BTX_S(btr, int, int, &, ~) 515 #endif /* CK_F_PR_BTR_INT */ 516 517 #ifndef CK_F_PR_BTS_INT 518 #define CK_F_PR_BTS_INT 519 CK_PR_BTX_S(bts, int, int, |,) 520 #endif /* CK_F_PR_BTS_INT */ 521 522 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 523 524 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 525 526 #ifndef CK_F_PR_BTC_UINT 527 #define CK_F_PR_BTC_UINT 528 CK_PR_BTX_S(btc, uint, unsigned int, ^,) 529 #endif /* CK_F_PR_BTC_UINT */ 530 531 #ifndef CK_F_PR_BTR_UINT 532 #define CK_F_PR_BTR_UINT 533 CK_PR_BTX_S(btr, uint, unsigned int, &, ~) 534 #endif /* CK_F_PR_BTR_UINT */ 535 536 #ifndef CK_F_PR_BTS_UINT 537 #define CK_F_PR_BTS_UINT 538 CK_PR_BTX_S(bts, uint, unsigned int, |,) 539 #endif /* CK_F_PR_BTS_UINT */ 540 541 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 542 543 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 544 545 #ifndef CK_F_PR_BTC_PTR 546 #define CK_F_PR_BTC_PTR 547 CK_PR_BTX(btc, ptr, void, uintptr_t, ^, void *,) 548 #endif /* CK_F_PR_BTC_PTR */ 549 550 #ifndef CK_F_PR_BTR_PTR 551 #define CK_F_PR_BTR_PTR 552 CK_PR_BTX(btr, ptr, void, uintptr_t, &, void *, ~) 553 #endif /* CK_F_PR_BTR_PTR */ 554 555 #ifndef CK_F_PR_BTS_PTR 556 #define CK_F_PR_BTS_PTR 557 CK_PR_BTX(bts, ptr, void, uintptr_t, |, void *,) 558 #endif /* CK_F_PR_BTS_PTR */ 559 560 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 561 562 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 563 564 #ifndef CK_F_PR_BTC_64 565 #define CK_F_PR_BTC_64 566 CK_PR_BTX_S(btc, 64, uint64_t, ^,) 567 #endif /* CK_F_PR_BTC_64 */ 568 569 #ifndef CK_F_PR_BTR_64 570 #define CK_F_PR_BTR_64 571 CK_PR_BTX_S(btr, 64, uint64_t, &, ~) 572 #endif /* CK_F_PR_BTR_64 */ 573 574 #ifndef CK_F_PR_BTS_64 575 #define CK_F_PR_BTS_64 576 CK_PR_BTX_S(bts, 64, uint64_t, |,) 577 #endif /* CK_F_PR_BTS_64 */ 578 579 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 580 581 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 582 583 #ifndef CK_F_PR_BTC_32 584 #define CK_F_PR_BTC_32 585 CK_PR_BTX_S(btc, 32, uint32_t, ^,) 586 #endif /* CK_F_PR_BTC_32 */ 587 588 #ifndef CK_F_PR_BTR_32 589 #define CK_F_PR_BTR_32 590 CK_PR_BTX_S(btr, 32, uint32_t, &, ~) 591 #endif /* CK_F_PR_BTR_32 */ 592 593 #ifndef CK_F_PR_BTS_32 594 #define CK_F_PR_BTS_32 595 CK_PR_BTX_S(bts, 32, uint32_t, |,) 596 #endif /* CK_F_PR_BTS_32 */ 597 598 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 599 600 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 601 602 #ifndef CK_F_PR_BTC_16 603 #define CK_F_PR_BTC_16 604 CK_PR_BTX_S(btc, 16, uint16_t, ^,) 605 #endif /* CK_F_PR_BTC_16 */ 606 607 #ifndef CK_F_PR_BTR_16 608 #define CK_F_PR_BTR_16 609 CK_PR_BTX_S(btr, 16, uint16_t, &, ~) 610 #endif /* CK_F_PR_BTR_16 */ 611 612 #ifndef CK_F_PR_BTS_16 613 #define CK_F_PR_BTS_16 614 CK_PR_BTX_S(bts, 16, uint16_t, |,) 615 #endif /* CK_F_PR_BTS_16 */ 616 617 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 618 619 #undef CK_PR_BTX_S 620 #undef CK_PR_BTX 621 622 #define CK_PR_UNARY(K, X, S, M, T) \ 623 CK_CC_INLINE static void \ 624 ck_pr_##K##_##S(M *target) \ 625 { \ 626 ck_pr_##X##_##S(target, (T)1); \ 627 return; \ 628 } 629 630 #define CK_PR_UNARY_Z(K, S, M, T, P, C, Z) \ 631 CK_CC_INLINE static bool \ 632 ck_pr_##K##_##S##_is_zero(M *target) \ 633 { \ 634 T previous; \ 635 C punt; \ 636 punt = (C)ck_pr_md_load_##S(target); \ 637 previous = (T)punt; \ 638 while (ck_pr_cas_##S##_value(target, \ 639 (C)previous, \ 640 (C)(previous P 1), \ 641 &previous) == false) \ 642 ck_pr_stall(); \ 643 return previous == (T)Z; \ 644 } 645 646 #define CK_PR_UNARY_Z_STUB(K, S, M) \ 647 CK_CC_INLINE static void \ 648 ck_pr_##K##_##S##_zero(M *target, bool *zero) \ 649 { \ 650 *zero = ck_pr_##K##_##S##_is_zero(target); \ 651 return; \ 652 } 653 654 #define CK_PR_UNARY_S(K, X, S, M) CK_PR_UNARY(K, X, S, M, M) 655 #define CK_PR_UNARY_Z_S(K, S, M, P, Z) \ 656 CK_PR_UNARY_Z(K, S, M, M, P, M, Z) \ 657 CK_PR_UNARY_Z_STUB(K, S, M) 658 659 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 660 661 #ifndef CK_F_PR_INC_CHAR 662 #define CK_F_PR_INC_CHAR 663 CK_PR_UNARY_S(inc, add, char, char) 664 #endif /* CK_F_PR_INC_CHAR */ 665 666 #ifndef CK_F_PR_INC_CHAR_ZERO 667 #define CK_F_PR_INC_CHAR_ZERO 668 CK_PR_UNARY_Z_S(inc, char, char, +, -1) 669 #else 670 CK_PR_UNARY_Z_STUB(inc, char, char) 671 #endif /* CK_F_PR_INC_CHAR_ZERO */ 672 673 #ifndef CK_F_PR_DEC_CHAR 674 #define CK_F_PR_DEC_CHAR 675 CK_PR_UNARY_S(dec, sub, char, char) 676 #endif /* CK_F_PR_DEC_CHAR */ 677 678 #ifndef CK_F_PR_DEC_CHAR_ZERO 679 #define CK_F_PR_DEC_CHAR_ZERO 680 CK_PR_UNARY_Z_S(dec, char, char, -, 1) 681 #else 682 CK_PR_UNARY_Z_STUB(dec, char, char) 683 #endif /* CK_F_PR_DEC_CHAR_ZERO */ 684 685 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 686 687 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 688 689 #ifndef CK_F_PR_INC_INT 690 #define CK_F_PR_INC_INT 691 CK_PR_UNARY_S(inc, add, int, int) 692 #endif /* CK_F_PR_INC_INT */ 693 694 #ifndef CK_F_PR_INC_INT_ZERO 695 #define CK_F_PR_INC_INT_ZERO 696 CK_PR_UNARY_Z_S(inc, int, int, +, -1) 697 #else 698 CK_PR_UNARY_Z_STUB(inc, int, int) 699 #endif /* CK_F_PR_INC_INT_ZERO */ 700 701 #ifndef CK_F_PR_DEC_INT 702 #define CK_F_PR_DEC_INT 703 CK_PR_UNARY_S(dec, sub, int, int) 704 #endif /* CK_F_PR_DEC_INT */ 705 706 #ifndef CK_F_PR_DEC_INT_ZERO 707 #define CK_F_PR_DEC_INT_ZERO 708 CK_PR_UNARY_Z_S(dec, int, int, -, 1) 709 #else 710 CK_PR_UNARY_Z_STUB(dec, int, int) 711 #endif /* CK_F_PR_DEC_INT_ZERO */ 712 713 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 714 715 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 716 !defined(CK_PR_DISABLE_DOUBLE) 717 718 #ifndef CK_F_PR_INC_DOUBLE 719 #define CK_F_PR_INC_DOUBLE 720 CK_PR_UNARY_S(inc, add, double, double) 721 #endif /* CK_F_PR_INC_DOUBLE */ 722 723 #ifndef CK_F_PR_DEC_DOUBLE 724 #define CK_F_PR_DEC_DOUBLE 725 CK_PR_UNARY_S(dec, sub, double, double) 726 #endif /* CK_F_PR_DEC_DOUBLE */ 727 728 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 729 730 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 731 732 #ifndef CK_F_PR_INC_UINT 733 #define CK_F_PR_INC_UINT 734 CK_PR_UNARY_S(inc, add, uint, unsigned int) 735 #endif /* CK_F_PR_INC_UINT */ 736 737 #ifndef CK_F_PR_INC_UINT_ZERO 738 #define CK_F_PR_INC_UINT_ZERO 739 CK_PR_UNARY_Z_S(inc, uint, unsigned int, +, UINT_MAX) 740 #else 741 CK_PR_UNARY_Z_STUB(inc, uint, unsigned int) 742 #endif /* CK_F_PR_INC_UINT_ZERO */ 743 744 #ifndef CK_F_PR_DEC_UINT 745 #define CK_F_PR_DEC_UINT 746 CK_PR_UNARY_S(dec, sub, uint, unsigned int) 747 #endif /* CK_F_PR_DEC_UINT */ 748 749 #ifndef CK_F_PR_DEC_UINT_ZERO 750 #define CK_F_PR_DEC_UINT_ZERO 751 CK_PR_UNARY_Z_S(dec, uint, unsigned int, -, 1) 752 #else 753 CK_PR_UNARY_Z_STUB(dec, uint, unsigned int) 754 #endif /* CK_F_PR_DEC_UINT_ZERO */ 755 756 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 757 758 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 759 760 #ifndef CK_F_PR_INC_PTR 761 #define CK_F_PR_INC_PTR 762 CK_PR_UNARY(inc, add, ptr, void, uintptr_t) 763 #endif /* CK_F_PR_INC_PTR */ 764 765 #ifndef CK_F_PR_INC_PTR_ZERO 766 #define CK_F_PR_INC_PTR_ZERO 767 CK_PR_UNARY_Z(inc, ptr, void, uintptr_t, +, void *, UINT_MAX) 768 #else 769 CK_PR_UNARY_Z_STUB(inc, ptr, void) 770 #endif /* CK_F_PR_INC_PTR_ZERO */ 771 772 #ifndef CK_F_PR_DEC_PTR 773 #define CK_F_PR_DEC_PTR 774 CK_PR_UNARY(dec, sub, ptr, void, uintptr_t) 775 #endif /* CK_F_PR_DEC_PTR */ 776 777 #ifndef CK_F_PR_DEC_PTR_ZERO 778 #define CK_F_PR_DEC_PTR_ZERO 779 CK_PR_UNARY_Z(dec, ptr, void, uintptr_t, -, void *, 1) 780 #else 781 CK_PR_UNARY_Z_STUB(dec, ptr, void) 782 #endif /* CK_F_PR_DEC_PTR_ZERO */ 783 784 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 785 786 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 787 788 #ifndef CK_F_PR_INC_64 789 #define CK_F_PR_INC_64 790 CK_PR_UNARY_S(inc, add, 64, uint64_t) 791 #endif /* CK_F_PR_INC_64 */ 792 793 #ifndef CK_F_PR_INC_64_ZERO 794 #define CK_F_PR_INC_64_ZERO 795 CK_PR_UNARY_Z_S(inc, 64, uint64_t, +, UINT64_MAX) 796 #else 797 CK_PR_UNARY_Z_STUB(inc, 64, uint64_t) 798 #endif /* CK_F_PR_INC_64_ZERO */ 799 800 #ifndef CK_F_PR_DEC_64 801 #define CK_F_PR_DEC_64 802 CK_PR_UNARY_S(dec, sub, 64, uint64_t) 803 #endif /* CK_F_PR_DEC_64 */ 804 805 #ifndef CK_F_PR_DEC_64_ZERO 806 #define CK_F_PR_DEC_64_ZERO 807 CK_PR_UNARY_Z_S(dec, 64, uint64_t, -, 1) 808 #else 809 CK_PR_UNARY_Z_STUB(dec, 64, uint64_t) 810 #endif /* CK_F_PR_DEC_64_ZERO */ 811 812 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 813 814 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 815 816 #ifndef CK_F_PR_INC_32 817 #define CK_F_PR_INC_32 818 CK_PR_UNARY_S(inc, add, 32, uint32_t) 819 #endif /* CK_F_PR_INC_32 */ 820 821 #ifndef CK_F_PR_INC_32_ZERO 822 #define CK_F_PR_INC_32_ZERO 823 CK_PR_UNARY_Z_S(inc, 32, uint32_t, +, UINT32_MAX) 824 #else 825 CK_PR_UNARY_Z_STUB(inc, 32, uint32_t) 826 #endif /* CK_F_PR_INC_32_ZERO */ 827 828 #ifndef CK_F_PR_DEC_32 829 #define CK_F_PR_DEC_32 830 CK_PR_UNARY_S(dec, sub, 32, uint32_t) 831 #endif /* CK_F_PR_DEC_32 */ 832 833 #ifndef CK_F_PR_DEC_32_ZERO 834 #define CK_F_PR_DEC_32_ZERO 835 CK_PR_UNARY_Z_S(dec, 32, uint32_t, -, 1) 836 #else 837 CK_PR_UNARY_Z_STUB(dec, 32, uint32_t) 838 #endif /* CK_F_PR_DEC_32_ZERO */ 839 840 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 841 842 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 843 844 #ifndef CK_F_PR_INC_16 845 #define CK_F_PR_INC_16 846 CK_PR_UNARY_S(inc, add, 16, uint16_t) 847 #endif /* CK_F_PR_INC_16 */ 848 849 #ifndef CK_F_PR_INC_16_ZERO 850 #define CK_F_PR_INC_16_ZERO 851 CK_PR_UNARY_Z_S(inc, 16, uint16_t, +, UINT16_MAX) 852 #else 853 CK_PR_UNARY_Z_STUB(inc, 16, uint16_t) 854 #endif /* CK_F_PR_INC_16_ZERO */ 855 856 #ifndef CK_F_PR_DEC_16 857 #define CK_F_PR_DEC_16 858 CK_PR_UNARY_S(dec, sub, 16, uint16_t) 859 #endif /* CK_F_PR_DEC_16 */ 860 861 #ifndef CK_F_PR_DEC_16_ZERO 862 #define CK_F_PR_DEC_16_ZERO 863 CK_PR_UNARY_Z_S(dec, 16, uint16_t, -, 1) 864 #else 865 CK_PR_UNARY_Z_STUB(dec, 16, uint16_t) 866 #endif /* CK_F_PR_DEC_16_ZERO */ 867 868 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 869 870 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 871 872 #ifndef CK_F_PR_INC_8 873 #define CK_F_PR_INC_8 874 CK_PR_UNARY_S(inc, add, 8, uint8_t) 875 #endif /* CK_F_PR_INC_8 */ 876 877 #ifndef CK_F_PR_INC_8_ZERO 878 #define CK_F_PR_INC_8_ZERO 879 CK_PR_UNARY_Z_S(inc, 8, uint8_t, +, UINT8_MAX) 880 #else 881 CK_PR_UNARY_Z_STUB(inc, 8, uint8_t) 882 #endif /* CK_F_PR_INC_8_ZERO */ 883 884 #ifndef CK_F_PR_DEC_8 885 #define CK_F_PR_DEC_8 886 CK_PR_UNARY_S(dec, sub, 8, uint8_t) 887 #endif /* CK_F_PR_DEC_8 */ 888 889 #ifndef CK_F_PR_DEC_8_ZERO 890 #define CK_F_PR_DEC_8_ZERO 891 CK_PR_UNARY_Z_S(dec, 8, uint8_t, -, 1) 892 #else 893 CK_PR_UNARY_Z_STUB(dec, 8, uint8_t) 894 #endif /* CK_F_PR_DEC_8_ZERO */ 895 896 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 897 898 #undef CK_PR_UNARY_Z_S 899 #undef CK_PR_UNARY_S 900 #undef CK_PR_UNARY_Z 901 #undef CK_PR_UNARY 902 903 #define CK_PR_N(K, S, M, T, P, C) \ 904 CK_CC_INLINE static void \ 905 ck_pr_##K##_##S(M *target) \ 906 { \ 907 T previous; \ 908 C punt; \ 909 punt = (C)ck_pr_md_load_##S(target); \ 910 previous = (T)punt; \ 911 while (ck_pr_cas_##S##_value(target, \ 912 (C)previous, \ 913 (C)(P previous), \ 914 &previous) == false) \ 915 ck_pr_stall(); \ 916 \ 917 return; \ 918 } 919 920 #define CK_PR_N_Z(S, M, T, C) \ 921 CK_CC_INLINE static void \ 922 ck_pr_neg_##S##_zero(M *target, bool *zero) \ 923 { \ 924 T previous; \ 925 C punt; \ 926 punt = (C)ck_pr_md_load_##S(target); \ 927 previous = (T)punt; \ 928 while (ck_pr_cas_##S##_value(target, \ 929 (C)previous, \ 930 (C)(-previous), \ 931 &previous) == false) \ 932 ck_pr_stall(); \ 933 \ 934 *zero = previous == 0; \ 935 return; \ 936 } 937 938 #define CK_PR_N_S(K, S, M, P) CK_PR_N(K, S, M, M, P, M) 939 #define CK_PR_N_Z_S(S, M) CK_PR_N_Z(S, M, M, M) 940 941 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 942 943 #ifndef CK_F_PR_NOT_CHAR 944 #define CK_F_PR_NOT_CHAR 945 CK_PR_N_S(not, char, char, ~) 946 #endif /* CK_F_PR_NOT_CHAR */ 947 948 #ifndef CK_F_PR_NEG_CHAR 949 #define CK_F_PR_NEG_CHAR 950 CK_PR_N_S(neg, char, char, -) 951 #endif /* CK_F_PR_NEG_CHAR */ 952 953 #ifndef CK_F_PR_NEG_CHAR_ZERO 954 #define CK_F_PR_NEG_CHAR_ZERO 955 CK_PR_N_Z_S(char, char) 956 #endif /* CK_F_PR_NEG_CHAR_ZERO */ 957 958 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 959 960 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 961 962 #ifndef CK_F_PR_NOT_INT 963 #define CK_F_PR_NOT_INT 964 CK_PR_N_S(not, int, int, ~) 965 #endif /* CK_F_PR_NOT_INT */ 966 967 #ifndef CK_F_PR_NEG_INT 968 #define CK_F_PR_NEG_INT 969 CK_PR_N_S(neg, int, int, -) 970 #endif /* CK_F_PR_NEG_INT */ 971 972 #ifndef CK_F_PR_NEG_INT_ZERO 973 #define CK_F_PR_NEG_INT_ZERO 974 CK_PR_N_Z_S(int, int) 975 #endif /* CK_F_PR_NEG_INT_ZERO */ 976 977 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 978 979 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 980 !defined(CK_PR_DISABLE_DOUBLE) 981 982 #ifndef CK_F_PR_NEG_DOUBLE 983 #define CK_F_PR_NEG_DOUBLE 984 CK_PR_N_S(neg, double, double, -) 985 #endif /* CK_F_PR_NEG_DOUBLE */ 986 987 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 988 989 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 990 991 #ifndef CK_F_PR_NOT_UINT 992 #define CK_F_PR_NOT_UINT 993 CK_PR_N_S(not, uint, unsigned int, ~) 994 #endif /* CK_F_PR_NOT_UINT */ 995 996 #ifndef CK_F_PR_NEG_UINT 997 #define CK_F_PR_NEG_UINT 998 CK_PR_N_S(neg, uint, unsigned int, -) 999 #endif /* CK_F_PR_NEG_UINT */ 1000 1001 #ifndef CK_F_PR_NEG_UINT_ZERO 1002 #define CK_F_PR_NEG_UINT_ZERO 1003 CK_PR_N_Z_S(uint, unsigned int) 1004 #endif /* CK_F_PR_NEG_UINT_ZERO */ 1005 1006 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1007 1008 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1009 1010 #ifndef CK_F_PR_NOT_PTR 1011 #define CK_F_PR_NOT_PTR 1012 CK_PR_N(not, ptr, void, uintptr_t, ~, void *) 1013 #endif /* CK_F_PR_NOT_PTR */ 1014 1015 #ifndef CK_F_PR_NEG_PTR 1016 #define CK_F_PR_NEG_PTR 1017 CK_PR_N(neg, ptr, void, uintptr_t, -, void *) 1018 #endif /* CK_F_PR_NEG_PTR */ 1019 1020 #ifndef CK_F_PR_NEG_PTR_ZERO 1021 #define CK_F_PR_NEG_PTR_ZERO 1022 CK_PR_N_Z(ptr, void, uintptr_t, void *) 1023 #endif /* CK_F_PR_NEG_PTR_ZERO */ 1024 1025 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1026 1027 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1028 1029 #ifndef CK_F_PR_NOT_64 1030 #define CK_F_PR_NOT_64 1031 CK_PR_N_S(not, 64, uint64_t, ~) 1032 #endif /* CK_F_PR_NOT_64 */ 1033 1034 #ifndef CK_F_PR_NEG_64 1035 #define CK_F_PR_NEG_64 1036 CK_PR_N_S(neg, 64, uint64_t, -) 1037 #endif /* CK_F_PR_NEG_64 */ 1038 1039 #ifndef CK_F_PR_NEG_64_ZERO 1040 #define CK_F_PR_NEG_64_ZERO 1041 CK_PR_N_Z_S(64, uint64_t) 1042 #endif /* CK_F_PR_NEG_64_ZERO */ 1043 1044 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1045 1046 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1047 1048 #ifndef CK_F_PR_NOT_32 1049 #define CK_F_PR_NOT_32 1050 CK_PR_N_S(not, 32, uint32_t, ~) 1051 #endif /* CK_F_PR_NOT_32 */ 1052 1053 #ifndef CK_F_PR_NEG_32 1054 #define CK_F_PR_NEG_32 1055 CK_PR_N_S(neg, 32, uint32_t, -) 1056 #endif /* CK_F_PR_NEG_32 */ 1057 1058 #ifndef CK_F_PR_NEG_32_ZERO 1059 #define CK_F_PR_NEG_32_ZERO 1060 CK_PR_N_Z_S(32, uint32_t) 1061 #endif /* CK_F_PR_NEG_32_ZERO */ 1062 1063 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1064 1065 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1066 1067 #ifndef CK_F_PR_NOT_16 1068 #define CK_F_PR_NOT_16 1069 CK_PR_N_S(not, 16, uint16_t, ~) 1070 #endif /* CK_F_PR_NOT_16 */ 1071 1072 #ifndef CK_F_PR_NEG_16 1073 #define CK_F_PR_NEG_16 1074 CK_PR_N_S(neg, 16, uint16_t, -) 1075 #endif /* CK_F_PR_NEG_16 */ 1076 1077 #ifndef CK_F_PR_NEG_16_ZERO 1078 #define CK_F_PR_NEG_16_ZERO 1079 CK_PR_N_Z_S(16, uint16_t) 1080 #endif /* CK_F_PR_NEG_16_ZERO */ 1081 1082 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1083 1084 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1085 1086 #ifndef CK_F_PR_NOT_8 1087 #define CK_F_PR_NOT_8 1088 CK_PR_N_S(not, 8, uint8_t, ~) 1089 #endif /* CK_F_PR_NOT_8 */ 1090 1091 #ifndef CK_F_PR_NEG_8 1092 #define CK_F_PR_NEG_8 1093 CK_PR_N_S(neg, 8, uint8_t, -) 1094 #endif /* CK_F_PR_NEG_8 */ 1095 1096 #ifndef CK_F_PR_NEG_8_ZERO 1097 #define CK_F_PR_NEG_8_ZERO 1098 CK_PR_N_Z_S(8, uint8_t) 1099 #endif /* CK_F_PR_NEG_8_ZERO */ 1100 1101 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1102 1103 #undef CK_PR_N_Z_S 1104 #undef CK_PR_N_S 1105 #undef CK_PR_N_Z 1106 #undef CK_PR_N 1107 1108 #define CK_PR_FAA(S, M, T, C) \ 1109 CK_CC_INLINE static C \ 1110 ck_pr_faa_##S(M *target, T delta) \ 1111 { \ 1112 T previous; \ 1113 C punt; \ 1114 punt = (C)ck_pr_md_load_##S(target); \ 1115 previous = (T)punt; \ 1116 while (ck_pr_cas_##S##_value(target, \ 1117 (C)previous, \ 1118 (C)(previous + delta), \ 1119 &previous) == false) \ 1120 ck_pr_stall(); \ 1121 \ 1122 return ((C)previous); \ 1123 } 1124 1125 #define CK_PR_FAS(S, M, C) \ 1126 CK_CC_INLINE static C \ 1127 ck_pr_fas_##S(M *target, C update) \ 1128 { \ 1129 C previous; \ 1130 previous = ck_pr_md_load_##S(target); \ 1131 while (ck_pr_cas_##S##_value(target, \ 1132 previous, \ 1133 update, \ 1134 &previous) == false) \ 1135 ck_pr_stall(); \ 1136 \ 1137 return (previous); \ 1138 } 1139 1140 #define CK_PR_FAA_S(S, M) CK_PR_FAA(S, M, M, M) 1141 #define CK_PR_FAS_S(S, M) CK_PR_FAS(S, M, M) 1142 1143 #if defined(CK_F_PR_LOAD_CHAR) && defined(CK_F_PR_CAS_CHAR_VALUE) 1144 1145 #ifndef CK_F_PR_FAA_CHAR 1146 #define CK_F_PR_FAA_CHAR 1147 CK_PR_FAA_S(char, char) 1148 #endif /* CK_F_PR_FAA_CHAR */ 1149 1150 #ifndef CK_F_PR_FAS_CHAR 1151 #define CK_F_PR_FAS_CHAR 1152 CK_PR_FAS_S(char, char) 1153 #endif /* CK_F_PR_FAS_CHAR */ 1154 1155 #endif /* CK_F_PR_LOAD_CHAR && CK_F_PR_CAS_CHAR_VALUE */ 1156 1157 #if defined(CK_F_PR_LOAD_INT) && defined(CK_F_PR_CAS_INT_VALUE) 1158 1159 #ifndef CK_F_PR_FAA_INT 1160 #define CK_F_PR_FAA_INT 1161 CK_PR_FAA_S(int, int) 1162 #endif /* CK_F_PR_FAA_INT */ 1163 1164 #ifndef CK_F_PR_FAS_INT 1165 #define CK_F_PR_FAS_INT 1166 CK_PR_FAS_S(int, int) 1167 #endif /* CK_F_PR_FAS_INT */ 1168 1169 #endif /* CK_F_PR_LOAD_INT && CK_F_PR_CAS_INT_VALUE */ 1170 1171 #if defined(CK_F_PR_LOAD_DOUBLE) && defined(CK_F_PR_CAS_DOUBLE_VALUE) && \ 1172 !defined(CK_PR_DISABLE_DOUBLE) 1173 1174 #ifndef CK_F_PR_FAA_DOUBLE 1175 #define CK_F_PR_FAA_DOUBLE 1176 CK_PR_FAA_S(double, double) 1177 #endif /* CK_F_PR_FAA_DOUBLE */ 1178 1179 #ifndef CK_F_PR_FAS_DOUBLE 1180 #define CK_F_PR_FAS_DOUBLE 1181 CK_PR_FAS_S(double, double) 1182 #endif /* CK_F_PR_FAS_DOUBLE */ 1183 1184 #endif /* CK_F_PR_LOAD_DOUBLE && CK_F_PR_CAS_DOUBLE_VALUE && !CK_PR_DISABLE_DOUBLE */ 1185 1186 #if defined(CK_F_PR_LOAD_UINT) && defined(CK_F_PR_CAS_UINT_VALUE) 1187 1188 #ifndef CK_F_PR_FAA_UINT 1189 #define CK_F_PR_FAA_UINT 1190 CK_PR_FAA_S(uint, unsigned int) 1191 #endif /* CK_F_PR_FAA_UINT */ 1192 1193 #ifndef CK_F_PR_FAS_UINT 1194 #define CK_F_PR_FAS_UINT 1195 CK_PR_FAS_S(uint, unsigned int) 1196 #endif /* CK_F_PR_FAS_UINT */ 1197 1198 #endif /* CK_F_PR_LOAD_UINT && CK_F_PR_CAS_UINT_VALUE */ 1199 1200 #if defined(CK_F_PR_LOAD_PTR) && defined(CK_F_PR_CAS_PTR_VALUE) 1201 1202 #ifndef CK_F_PR_FAA_PTR 1203 #define CK_F_PR_FAA_PTR 1204 CK_PR_FAA(ptr, void, uintptr_t, void *) 1205 #endif /* CK_F_PR_FAA_PTR */ 1206 1207 #ifndef CK_F_PR_FAS_PTR 1208 #define CK_F_PR_FAS_PTR 1209 CK_PR_FAS(ptr, void, void *) 1210 #endif /* CK_F_PR_FAS_PTR */ 1211 1212 #endif /* CK_F_PR_LOAD_PTR && CK_F_PR_CAS_PTR_VALUE */ 1213 1214 #if defined(CK_F_PR_LOAD_64) && defined(CK_F_PR_CAS_64_VALUE) 1215 1216 #ifndef CK_F_PR_FAA_64 1217 #define CK_F_PR_FAA_64 1218 CK_PR_FAA_S(64, uint64_t) 1219 #endif /* CK_F_PR_FAA_64 */ 1220 1221 #ifndef CK_F_PR_FAS_64 1222 #define CK_F_PR_FAS_64 1223 CK_PR_FAS_S(64, uint64_t) 1224 #endif /* CK_F_PR_FAS_64 */ 1225 1226 #endif /* CK_F_PR_LOAD_64 && CK_F_PR_CAS_64_VALUE */ 1227 1228 #if defined(CK_F_PR_LOAD_32) && defined(CK_F_PR_CAS_32_VALUE) 1229 1230 #ifndef CK_F_PR_FAA_32 1231 #define CK_F_PR_FAA_32 1232 CK_PR_FAA_S(32, uint32_t) 1233 #endif /* CK_F_PR_FAA_32 */ 1234 1235 #ifndef CK_F_PR_FAS_32 1236 #define CK_F_PR_FAS_32 1237 CK_PR_FAS_S(32, uint32_t) 1238 #endif /* CK_F_PR_FAS_32 */ 1239 1240 #endif /* CK_F_PR_LOAD_32 && CK_F_PR_CAS_32_VALUE */ 1241 1242 #if defined(CK_F_PR_LOAD_16) && defined(CK_F_PR_CAS_16_VALUE) 1243 1244 #ifndef CK_F_PR_FAA_16 1245 #define CK_F_PR_FAA_16 1246 CK_PR_FAA_S(16, uint16_t) 1247 #endif /* CK_F_PR_FAA_16 */ 1248 1249 #ifndef CK_F_PR_FAS_16 1250 #define CK_F_PR_FAS_16 1251 CK_PR_FAS_S(16, uint16_t) 1252 #endif /* CK_F_PR_FAS_16 */ 1253 1254 #endif /* CK_F_PR_LOAD_16 && CK_F_PR_CAS_16_VALUE */ 1255 1256 #if defined(CK_F_PR_LOAD_8) && defined(CK_F_PR_CAS_8_VALUE) 1257 1258 #ifndef CK_F_PR_FAA_8 1259 #define CK_F_PR_FAA_8 1260 CK_PR_FAA_S(8, uint8_t) 1261 #endif /* CK_F_PR_FAA_8 */ 1262 1263 #ifndef CK_F_PR_FAS_8 1264 #define CK_F_PR_FAS_8 1265 CK_PR_FAS_S(8, uint8_t) 1266 #endif /* CK_F_PR_FAS_8 */ 1267 1268 #endif /* CK_F_PR_LOAD_8 && CK_F_PR_CAS_8_VALUE */ 1269 1270 #undef CK_PR_FAA_S 1271 #undef CK_PR_FAS_S 1272 #undef CK_PR_FAA 1273 #undef CK_PR_FAS 1274 1275 #endif /* CK_PR_H */ 1276